From f20d4dddc2fe66e036d7cc1e14d06fa7be76e585 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:31:49 -0700 Subject: [PATCH 01/26] feat(repo): time to first useful result per run A task.json `useful` key (`{"command": regex}` for a task graded by its target tests, `{"edit": glob}` for one graded by an edit to a target file) names the event that is the run's first useful result. result.json `useful` records its seconds (the event's issue time) and steps (events up to and including it), or why there is none (never, unobserved, untimed, not_declared). repo report shows the observed count and medians per row; repo compare pairs the per-task medians as first_useful_s and first_useful_steps. A malformed value is refused by repo validate, tasks check and repo run. Batches recorded before this have no such rows. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 13 + dokimi/docs/repo-evaluation-contract.md | 27 +- dokimi/dokimi/repos/compare.py | 17 +- dokimi/dokimi/repos/host.py | 4 +- dokimi/dokimi/repos/manifest.py | 11 + dokimi/dokimi/repos/process_counts.py | 6 + dokimi/dokimi/repos/report.py | 8 + dokimi/dokimi/repos/runner.py | 3 +- dokimi/dokimi/repos/useful.py | 113 ++++++ dokimi/dokimi/repos/useful_spec.py | 54 +++ dokimi/dokimi/repos/workspace.py | 6 + dokimi/tests/test_batch_ledger_golden.py | 1 + dokimi/tests/test_repo_manifest.py | 2 + .../tests/test_repo_report_unreadable_cell.py | 2 +- dokimi/tests/test_repo_useful.py | 381 ++++++++++++++++++ 15 files changed, 633 insertions(+), 15 deletions(-) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md create mode 100644 dokimi/dokimi/repos/useful.py create mode 100644 dokimi/dokimi/repos/useful_spec.py create mode 100644 dokimi/tests/test_repo_useful.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md new file mode 100644 index 000000000..36c443698 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -0,0 +1,13 @@ +### Added + +- **Time to first useful result, per repo-eval run** — a `task.json` can declare + `"useful": {"command": ""}` for a task graded by its target tests, or + `"useful": {"edit": ""}` (or a list of globs) for one graded by an edit to a target + file. The first event that hits it (a command whose text matches and whose `rc` is 0, or an + edit whose path matches) is recorded in `result.json` `useful` with its `seconds` (the + event's issue time) and `steps` (events up to and including it). A run with no hit says + why (`never`, `unobserved`, `untimed`); a task with no declaration is `not_declared`. + `repo report` shows a row's observed count, medians and reasons under the row, and + `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`. + A malformed `useful` is refused by `repo validate`, `repo tasks check` and `repo run`. + Batches run before this change have no `useful` and no such rows. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c2b873922..47dbe638c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -279,6 +279,7 @@ belongs to the repository's grader and is passed through unread, except `grader` | `host` | no | framework | `{"min_cpus": , "cpu_vendors": [], "cpu_flags": []}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not | | `egress` | no | framework | extra hosts (DNS names, optional `:port`; `*` stands for one whole label and the last two labels are literal, so `*.pythonhosted.org` passes and `*` or `*.org` does not) this task's agent and entry points reach through the proxy (§2), for example `["pypi.org", "files.pythonhosted.org"]` | | `expected_skill` | no | framework | the name (directory name) of the skill the agent should use; a `skill-catalog:` arm must install it, or the arm is refused before anything runs. `result.json` `skills` records whether the agent used it and whether it used it first | +| `useful` | no | framework | the definition of the run's first useful result, reported as `useful` in `result.json` (§4.1): `{"command": ""}` for a task whose grade runs the target tests, or `{"edit": ""}` (or a list of globs) for one that grades an edit to a target file. Omit it when no event is the useful result. A malformed value is refused by `repo validate`, `tasks check` and `repo run` | | anything else | — | repository | for example `hidden_files`, `targets`, `regression_targets`, `manifest` | `setup_patch` (a single string) is accepted as a synonym for a one-element @@ -443,6 +444,27 @@ parent's and subagents' together, so a grader need not: `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). +`result.json` `useful` is the run's first useful result: the first event that +hits the task's `useful` key (§3). The definition depends on the grader: + +| Grader | Declare | Hit | +|---|---|---| +| test: the grade runs the target tests | `{"command": ""}` | the first `command` event whose text matches the regex (`re.search`) and whose `rc` is `0`. `rc` 0 says that command passed, not that every target test did, so the regex should name the target tests | +| edit or diff: the grade reads a target file | `{"edit": ""}` or a list | the first `edit` event whose path matches a glob (`match_glob`, as the guidance globs) | +| review (`dokimi:review_*`) | none | none: the answer is the product, so no event is the useful result, and the reason is `not_declared` | +| any other repository grader | either form, or none | none declared, `not_declared` | + +A refused event (`denied`) never hits. `steps` counts every event up to and +including the hit, the parent's and subagents' (as `process.tool_calls` does). +`seconds` is the hit's `t`: the time the call was issued, since a call's result +is not timed. `useful` is `{"kind": "command"|"edit"|null, "reason": ..., +"seconds": number|null, "steps": int|null}`. `reason` is `null` when the hit was +found and timed; otherwise it says why `seconds` (and `steps`, with no hit) is +null: `not_declared` (no definition), `unobserved` (the trace was not read: the +harness never started or it did not parse), `never` (no event hit) or `untimed` +(the trace format has no timestamps, so `steps` is set and `seconds` is null). +A cell of an older batch has no `useful`. + A cell of an arm that installs skills also has a `skills` object in `result.json`: `measured`, `skills_installed` (the installed skill names), `skills_used` (the installed skills the agent called with the `Skill` tool or @@ -1454,13 +1476,14 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, - cost, turns, `first_prompt_tokens`, each `--metric` from `grade.json`, and each `process` count a + cost, turns, `first_prompt_tokens`, `first_useful_s` and `first_useful_steps` (the median over a + task's cells that observed the first useful result, §4.1), each `--metric` from `grade.json`, and each `process` count a compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for gate and score, a t interval for the descriptive measures, `-` for a single task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). - A `process` count or `first_prompt_tokens` only one arm reports for the shared tasks of a + A `process` count, `first_prompt_tokens` or `first_useful_*` only one arm reports for the shared tasks of a (treatment, model) is not compared; a warning names it. When neither arm reports it (a batch run before `repo run` recorded it), the row is absent. Holm's correction runs over the gate and score comparisons of one call; the diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 308370da3..c82890e17 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -15,6 +15,8 @@ - **first_prompt_tokens**: the mean of ``result.json`` ``agent.first_prompt_tokens`` (what the first main-thread call was sent: start-up plus the task) over the cells that have it. Like a process count, it is compared only when both arms report it for some shared task: a batch run before it was recorded has none; +- **first_useful_s**, **first_useful_steps**: the median seconds and steps to the first useful result (``useful`` + in ``result.json``, :mod:`.useful`) over the cells that observed it; compared like ``first_prompt_tokens``; - **metric:**: the mean of ``grade.json`` ``metrics.`` over the cells that report it (``true`` is 1), for ``--metric``; - **pass^k**: for ``--pass-k K``, the chance that K cells drawn from the @@ -111,7 +113,8 @@ from .compare_reach import ALPHA, holm_floor, min_detectable from .compare_tasks import CompareError, select_tasks -from .process_counts import as_measure, process_order, process_values +from .process_counts import as_measure, process_lists, process_order, process_values +from .useful import USEFUL_MEASURES, useful_task_values from .report import (load_cells, mismatch_warnings, model_mismatch, reading_cells, resolved_models, unmeasured_note, uptake_counts) @@ -124,7 +127,7 @@ #: Measures Holm's correction runs over. PRIMARY = ("gate", "score") #: Descriptive measures compared only when both arms report them for some shared task (an older batch has none). -REPORTED = ("process:", "counter:", "first_prompt_tokens") +REPORTED = ("process:", "counter:", "first_prompt_tokens", "first_useful") #: Batch stamp keys that must agree between the arms of a comparison; ``tree`` is shown but may differ (another #: revision of the repository is a treatment of its own). STAMP_KEYS = ("eval_dir", "image", "graders") @@ -192,7 +195,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None treatments = _treatments(cells, baseline, base_sel, treatments, scope) measures = ["gate", "score", *([f"pass^{pass_k}"] if pass_k else []), *(["path_distance"] if path_stability else []), "cost_usd", "turns", "first_prompt_tokens", - *(f"metric:{m}" for m in metrics)] + *USEFUL_MEASURES, *(f"metric:{m}" for m in metrics)] base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics, pass_k, path_stability) rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] @@ -696,7 +699,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...], pass_k: "turns": [r["agent"].get("num_turns") for r in recs], "first_prompt_tokens": [r["agent"].get("first_prompt_tokens") for r in recs], **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}} - found.update(_process_lists(recs)) + found.update({**process_lists(recs), **useful_task_values(recs)}) if score_task: found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0) for r in recs] @@ -777,12 +780,6 @@ def _finite(values: list[Any]) -> list[float]: return [num for v in values if (num := as_measure(v)) is not None] -def _process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]: - """Each process measure any of ``recs`` reports, with every cell's value (``None`` where it has none).""" - values = [process_values(r) for r in recs] - return {name: [v.get(name) for v in values] for name in {k for v in values for k in v}} - - def _mean(values: list[Any]) -> float | None: nums = _finite(values) return sum(nums) / len(nums) if nums else None diff --git a/dokimi/dokimi/repos/host.py b/dokimi/dokimi/repos/host.py index d504e7eda..d6b98cb46 100644 --- a/dokimi/dokimi/repos/host.py +++ b/dokimi/dokimi/repos/host.py @@ -33,6 +33,7 @@ from .container import ContainerError, ContainerSpec from .grading import EvalContext from .host_spec import HOST_KEYS, KNOWN_VENDORS, HostRequirements, HostSpecError, parse_host # noqa: F401 +from .useful_spec import parse_useful from .workspace import Task PROBE_TIMEOUT_S = 120 @@ -136,9 +137,10 @@ def requirements(task: Task) -> HostRequirements | None: def validate_tasks(tasks: list[Task]) -> None: - """Refuse a bad ``host`` value before anything runs, not cell by cell.""" + """Refuse a bad ``host`` or ``useful`` value before anything runs, not cell by cell.""" for task in tasks: requirements(task) + parse_useful(task.raw.get("useful")) # raises UsefulSpecError on a bad shape def cgroup_cpus(raw: dict[str, Any]) -> int | None: diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 57aa5ce6a..3f2a767f4 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -35,6 +35,7 @@ from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter from .source import RepoTree +from .useful_spec import UsefulSpecError, parse_useful SCHEMA_VERSION = "repo_manifest.v1" _RUN_KEYS = ("eval_dir", "tasks", "entry_points") @@ -559,6 +560,7 @@ def _check_task(tree: RepoTree, task_dir: str, report: RepoCheckReport) -> None: parse_host(raw.get("host"), name) except HostSpecError as exc: problems.append(str(exc).split("task.json ", 1)[-1]) + problems += _useful_problems(raw) report.errors.extend(f"{where}: {p}" for p in problems) if not problems: report.tasks.append(TaskSummary( @@ -591,6 +593,15 @@ def _positive(value: Any) -> bool: return num is not None and num > 0 +def _useful_problems(raw: dict[str, Any]) -> list[str]: + """The ``useful`` key's shape problem, if it has one (contract §3).""" + try: + parse_useful(raw.get("useful")) + except UsefulSpecError as exc: + return [str(exc)] + return [] + + def _builtin_problems(tree: RepoTree, task_dir: str, raw: dict[str, Any], name: str) -> list[str]: """A ``dokimi:`` grader must be one the framework has, and its data file must be in the task directory.""" if name not in BUILTIN_NAMES: diff --git a/dokimi/dokimi/repos/process_counts.py b/dokimi/dokimi/repos/process_counts.py index e83d49425..8ec2a173d 100644 --- a/dokimi/dokimi/repos/process_counts.py +++ b/dokimi/dokimi/repos/process_counts.py @@ -25,6 +25,12 @@ PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read") +def process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]: + """Each process measure any of ``recs`` reports, with every cell's value (``None`` where it has none).""" + values = [process_values(r) for r in recs] + return {name: [v.get(name) for v in values] for name in {k for v in values for k in v}} + + def process_values(rec: dict[str, Any]) -> dict[str, float]: """A cell's ``process`` counters as numbers: ``process:`` and ``counter:``; ``{}`` without one.""" found = rec.get("process") diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 074656cb4..801611eeb 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -19,6 +19,9 @@ - **uptake**: for an arm that added a file or an MCP server, the cells whose agent used it (``result.json`` ``uptake``, see :mod:`.uptake`) over the cells with a trace to count from. +- **first useful**: for a task that declares a ``useful`` definition, the cells that observed the first useful + result (``result.json`` ``useful``, see :mod:`.useful`): how many, their median seconds and steps, and why the + others did not (``never``, ``unobserved``, ``untimed``). - skill arms add ``skill_invoked`` (cells whose agent used the expected skill, or any installed one when the task names none) and, when the task has an ``expected_skill``, ``skill_selected_correct`` (cells that used it first; ``skill_expected`` @@ -61,6 +64,7 @@ from .grading import SCORE_BOUND, read_grade_json from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug +from .useful import useful_line, useful_summary _TOKENS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") @@ -239,6 +243,9 @@ def summarize(batch_dir: Path) -> dict[str, Any]: uptake = uptake_counts(recs) if uptake is not None: row["uptake"] = uptake + first = useful_summary(recs) + if first is not None: + row["first_useful"] = first if score_only: scores = [0.0 if r["grade"]["status"] == "grader_failure" else r["grade"]["score"] for r in recs] row.update(kind="score", score=_mean([0.0 if s is None else s for s in scores])) @@ -286,6 +293,7 @@ def format_report(summary: dict[str, Any]) -> str: if notes: lines.append(" " * (width + 2) + " " + ", ".join(notes)) lines += process_line(r.get("process"), width) + lines += useful_line(r.get("first_useful"), width) if summary["missing"]: shown = summary["missing"][:10] lines.append("missing (no result.json; rerun the batch): " + ", ".join(shown) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a0a122128..39f3680c5 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -71,6 +71,7 @@ from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert from .uptake import arm_uptake +from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry from .harness_profile import HarnessProfile from .host import probe_host, unmet, validate_tasks @@ -288,7 +289,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "harness": opts.profile.name, "harness_version": meta["harness_version"], "base_date": meta["base_date"], "execution": execution, "score_task": bool(task.score), "score_max": task.score["max"] if task.score else None, "agent": agent, - "process": process, "grade": grade} + "process": process, "useful": first_useful(task.useful, events), "grade": grade} _add_arm_use(record, meta, task, skills, uptake_of, events) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py new file mode 100644 index 000000000..080e625a4 --- /dev/null +++ b/dokimi/dokimi/repos/useful.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The first useful result of a run, for ``result.json`` ``useful`` (contract §4.1). + +The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text +matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never +hits. ``steps`` counts every event up to and including the hit, as ``process.tool_calls`` does. ``seconds`` is the +hit's ``t``: the time the call was issued, since its result is not timed, so a test run's pass is not timed. +""" + +from __future__ import annotations + +import statistics +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +from .manifest import match_glob +from .useful_spec import Useful + +#: The ``repo compare`` measures of the first useful result, one value per task (see :func:`useful_task_values`). +USEFUL_MEASURES = ("first_useful_s", "first_useful_steps") + + +def hits(useful: Useful, event: Mapping[str, Any]) -> bool: + """Whether ``event`` satisfies ``useful``: a passing command matching the regex, or an edit matching a glob.""" + if event.get("denied"): # a refused call never ran, or never changed the file + return False + if useful.kind == "command": + text = str(event.get("text") or "") + matched = bool(useful.command and useful.command.search(text)) + return event.get("kind") == "command" and event.get("rc") == 0 and matched + path = str(event.get("path") or "") + return event.get("kind") == "edit" and any(match_glob(g, path) for g in useful.globs) + + +def first_useful( + useful: Useful | None, events: Sequence[Mapping[str, Any]] | None +) -> dict[str, Any]: + """``result.json`` ``useful`` of one run; ``events`` is None when its trace could not be read.""" + if useful is None: + return _record(None, "not_declared") + if events is None: + return _record(useful.kind, "unobserved") + for steps, event in enumerate(events, 1): + if hits(useful, event): + t = event.get("t") + if isinstance(t, (int, float)) and not isinstance(t, bool): + return _record(useful.kind, None, seconds=float(t), steps=steps) + return _record(useful.kind, "untimed", steps=steps) # a trace without timestamps + return _record(useful.kind, "never") + + +def _record( + kind: str | None, reason: str | None, seconds: float | None = None, steps: int | None = None +) -> dict[str, Any]: + """``reason`` says why ``seconds`` or ``steps`` is null; it is null when the hit was found and timed.""" + return {"kind": kind, "reason": reason, "seconds": seconds, "steps": steps} + + +def _declared(recs: Iterable[Mapping[str, Any]]) -> list[Mapping[str, Any]]: + """The ``useful`` objects of the cells that record a definition: an older batch has none.""" + return [u for r in recs if isinstance(u := r.get("useful"), dict) and u.get("kind")] + + +def _nums(values: Iterable[Any]) -> list[float]: + return [float(v) for v in values if isinstance(v, (int, float)) and not isinstance(v, bool)] + + +def _median(values: list[float]) -> float | None: + return statistics.median(values) if values else None + + +def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: + """``repo report``'s ``first_useful`` for one row: the cells observed, their medians and why the rest are not. + + None when no cell records a definition.""" + have = _declared(recs) + if not have: + return None + seconds = _nums(u.get("seconds") for u in have) + steps = _nums(u.get("steps") for u in have) + return { + "kind": have[0]["kind"], + "n": len(have), + "observed": len(steps), + "median_seconds": _median(seconds), + "median_steps": _median(steps), + "reasons": dict(Counter(u["reason"] for u in have if u.get("reason"))), + } + + +def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[float | None]]: + """``repo compare``'s per-task values: the median over the cells that observed it, one value (None if none).""" + have = _declared(recs) + return { + "first_useful_s": [_median(_nums(u.get("seconds") for u in have))], + "first_useful_steps": [_median(_nums(u.get("steps") for u in have))], + } + + +def useful_line(summary: Mapping[str, Any] | None, width: int) -> list[str]: + """The row's first-useful summary on one indented line; none for a row without a definition.""" + if not summary: + return [] + parts = [f"{summary['observed']} of {summary['n']} observed"] + if summary["median_seconds"] is not None: + parts.append(f"median {summary['median_seconds']:.1f} s") + if summary["median_steps"] is not None: + parts.append(f"{summary['median_steps']:g} steps") + parts += [f"{reason} {count}" for reason, count in sorted(summary["reasons"].items())] + return [" " * (width + 4) + f"first useful ({summary['kind']}): " + ", ".join(parts)] diff --git a/dokimi/dokimi/repos/useful_spec.py b/dokimi/dokimi/repos/useful_spec.py new file mode 100644 index 000000000..047f1d1a0 --- /dev/null +++ b/dokimi/dokimi/repos/useful_spec.py @@ -0,0 +1,54 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The ``useful`` key of ``task.json``: the event that is a run's first useful result (contract §3, §4.1). + +``{"command": ""}`` names the test run that passes the target tests; ``{"edit": ""}`` (or a list +of globs) the first edit of a target file. :mod:`.useful` reads it. No parser imports :mod:`.manifest`. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from .errors import PrepError + +SHAPE = 'useful must be {"command": } or {"edit": } (or a list of globs)' + + +class UsefulSpecError(PrepError): + """A ``task.json`` ``useful`` value of the wrong shape.""" + + +@dataclass(frozen=True) +class Useful: + """One task's definition: ``command`` (a regex over command text) or ``edit`` (globs over edited paths).""" + + kind: str + command: re.Pattern[str] | None = None + globs: tuple[str, ...] = () + + +def parse_useful(value: Any) -> Useful | None: + """The task's definition, or None when it names none; :class:`UsefulSpecError` when the shape is wrong.""" + if value is None: + return None + if not (isinstance(value, dict) and len(value) == 1): + raise UsefulSpecError(SHAPE) + ((kind, pattern),) = value.items() + if kind == "command" and isinstance(pattern, str) and pattern: + try: + return Useful("command", command=re.compile(pattern)) + except re.error as exc: + raise UsefulSpecError(f"useful command is not a valid regex ({exc})") from exc + globs = [pattern] if isinstance(pattern, str) else pattern + if ( + kind == "edit" + and isinstance(globs, list) + and globs + and all(isinstance(g, str) and g for g in globs) + ): + return Useful("edit", globs=tuple(globs)) + raise UsefulSpecError(SHAPE) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index b5e0dba74..127a8f5fc 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -55,6 +55,7 @@ from .manifest import RepoManifest, _is_within, match_glob from .source import GitTree, RepoSource, RepoTree, SourceError, cache_path, default_cache_dir, resolve_rev from .uptake import MCP_CONFIG, base_servers, mcp_servers +from .useful_spec import Useful, parse_useful ARMS = ("guidance:none", "guidance:raw", "guidance:bridge") # the guidance parts REV_ARM, PATCH_ARM = "guidance:rev:", "patch:" @@ -235,6 +236,11 @@ def expected_skill(self) -> str | None: def score(self) -> dict[str, Any] | None: return self.raw.get("score") + @property + def useful(self) -> Useful | None: + """The ``useful`` definition (contract §3); :class:`UsefulSpecError` on a bad shape.""" + return parse_useful(self.raw.get("useful")) + def load_tasks(content_dir: Path, manifest: RepoManifest, ids: list[str] | None = None) -> list[Task]: """Tasks of the exported eval dir; ``ids`` selects (and must name existing tasks). diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index df3069a53..942bc29d7 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -110,6 +110,7 @@ "score_max", "score_task", "task", + "useful", ] diff --git a/dokimi/tests/test_repo_manifest.py b/dokimi/tests/test_repo_manifest.py index cdc2e9cfc..d88eef3fd 100644 --- a/dokimi/tests/test_repo_manifest.py +++ b/dokimi/tests/test_repo_manifest.py @@ -187,6 +187,8 @@ def test_missing_and_non_yaml_manifest(tmp_path): ({"egress": ["py*.org"]}, "egress must be a list"), ({"expected_skill": "skills/lint"}, "expected_skill must be a skill name"), # a path never matches usage ({"expected_skill": 3}, "expected_skill must be a skill name"), + ({"useful": {"command": "("}}, "useful command is not a valid regex"), + ({"useful": {"edit": []}}, "useful must be"), # a definition names one kind and at least one target ], ) def test_bad_task_is_reported(tmp_path, task_extra, needle): diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py index d67264347..17695312a 100644 --- a/dokimi/tests/test_repo_report_unreadable_cell.py +++ b/dokimi/tests/test_repo_report_unreadable_cell.py @@ -347,7 +347,7 @@ def boom(recs): raise exc monkeypatch.setattr(report, "process_means", boom) - monkeypatch.setattr(compare, "_process_lists", boom) + monkeypatch.setattr(compare, "process_lists", boom) batch = _two_arm_batch(tmp_path, _GOOD) with pytest.raises(SystemExit) as raised: _run(cmd[0], str(batch), *cmd[1:]) diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py new file mode 100644 index 000000000..53e8a5589 --- /dev/null +++ b/dokimi/tests/test_repo_useful.py @@ -0,0 +1,381 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The first useful result of a run: the task's ``useful`` definition, the event that hits it, and its report/compare rows.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.repos.events import convert +from dokimi.repos.host import validate_tasks +from dokimi.repos.report import format_report, summarize +from dokimi.repos.useful import first_useful, useful_line, useful_summary, useful_task_values +from dokimi.repos.useful_spec import UsefulSpecError, parse_useful +from dokimi.repos.workspace import Task + +from dokimi.repos.compare import compare +from test_repo_compare import _cell, _row +from test_repo_run import _only_cell, _run, setup # noqa: F401 # the run fixture, with its helpers + +TEST_CMD = r"pytest tests/test_x\.py" + + +def _command(text: str, rc: int | None, t: float | None = None, **extra) -> dict: + return {"t": t, "kind": "command", "text": text, "rc": rc, **extra} + + +def _edit(path: str, t: float | None = None, **extra) -> dict: + return {"t": t, "kind": "edit", "path": path, **extra} + + +# -- the definition ---------------------------------------------------------------------------------------------- + + +def test_a_task_without_the_key_names_no_definition(): + assert parse_useful(None) is None + + +def test_the_two_kinds_parse_to_their_pattern_and_globs(): + command = parse_useful({"command": TEST_CMD}) + assert command.kind == "command" and command.command.search("pytest tests/test_x.py -q") + assert parse_useful({"edit": "src/a.py"}).globs == ("src/a.py",) + assert parse_useful({"edit": ["src/a.py", "**/*.h"]}).globs == ("src/a.py", "**/*.h") + + +@pytest.mark.parametrize( + "value,needle", + [ + ({"command": ""}, "useful must be"), + ({"command": "("}, "not a valid regex"), + ({"edit": []}, "useful must be"), + ({"edit": ["src/a.py", ""]}, "useful must be"), + ({"edit": 3}, "useful must be"), + ({"command": "pytest", "edit": "src/a.py"}, "useful must be"), # one kind per task + ("pytest", "useful must be"), + ({"test": "pytest"}, "useful must be"), + ], +) +def test_a_malformed_definition_is_refused(value, needle): + with pytest.raises(UsefulSpecError, match=needle): + parse_useful(value) + + +def test_validate_tasks_refuses_a_malformed_definition_before_a_run(): + with pytest.raises(UsefulSpecError, match="useful must be"): + validate_tasks([Task("t", {"id": "t", "useful": {"edit": 3}}, Path("."))]) + + +# -- the first hit ----------------------------------------------------------------------------------------------- + + +def test_a_passing_matching_command_is_the_first_useful_result_counted_in_steps_and_seconds(): + events = [ + _command("ls", 0, t=1.0), + _command("pytest tests/test_x.py", 1, t=2.0), # runs, fails: not useful + _edit("src/x.py", t=3.0), + _command("pytest tests/test_x.py", None, t=4.0), # no result came back: not useful + _command("pytest tests/test_x.py", 0, t=5.0, denied=True), # refused: never ran + _command("pytest tests/test_x.py", 0, t=6.0), + _command("pytest tests/test_x.py", 0, t=7.0), + ] + assert first_useful(parse_useful({"command": TEST_CMD}), events) == { + "kind": "command", + "reason": None, + "seconds": 6.0, + "steps": 6, + } + + +def test_the_first_edit_of_a_target_file_counts_steps_across_every_event_kind(): + events = [ + _command("cat README.md", 0, t=0.5), + _edit("docs/x.md", t=1.0), + _edit("src/other.py", t=1.5, denied=True), # the hook refused the edit + _edit("src/sub/lib.cpp", t=2.0), + _edit("src/sub/lib.cpp", t=3.0), + ] + assert first_useful(parse_useful({"edit": ["src/**/*.cpp", "src/x.py"]}), events) == { + "kind": "edit", + "reason": None, + "seconds": 2.0, + "steps": 4, + } + + +def test_a_run_with_no_hit_is_never_and_names_no_steps(): + events = [_command(TEST_CMD, 1, t=1.0), _edit("docs/x.md", t=2.0)] + assert first_useful(parse_useful({"command": TEST_CMD}), events) == { + "kind": "command", + "reason": "never", + "seconds": None, + "steps": None, + } + + +def test_a_run_whose_trace_was_not_read_is_unobserved_not_never(): + assert first_useful(parse_useful({"edit": "src/x.py"}), None) == { + "kind": "edit", + "reason": "unobserved", + "seconds": None, + "steps": None, + } + + +def test_a_task_without_a_definition_is_not_declared(): + assert first_useful(None, [_edit("src/x.py", t=1.0)]) == { + "kind": None, + "reason": "not_declared", + "seconds": None, + "steps": None, + } + + +def test_a_trace_without_timestamps_keeps_the_steps_and_says_untimed(): + events = [_edit("docs/x.md"), _edit("src/x.py")] + assert first_useful(parse_useful({"edit": "src/x.py"}), events) == { + "kind": "edit", + "reason": "untimed", + "seconds": None, + "steps": 2, + } + + +def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): + lines = [ + { + "type": "assistant", + "timestamp": "2026-10-31T10:00:00.000Z", + "message": { + "content": [ + { + "type": "tool_use", + "id": "a", + "name": "Bash", + "input": {"command": "pytest tests/test_x.py"}, + } + ] + }, + }, + { + "type": "user", + "timestamp": "2026-10-31T10:00:03.000Z", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "a", + "is_error": True, + "content": "1 failed", + } + ] + }, + }, + { + "type": "assistant", + "timestamp": "2026-10-31T10:00:10.000Z", + "message": { + "content": [ + { + "type": "tool_use", + "id": "b", + "name": "Edit", + "input": {"file_path": "/ws/src/x.py"}, + } + ] + }, + }, + { + "type": "assistant", + "timestamp": "2026-10-31T10:00:42.000Z", + "message": { + "content": [ + { + "type": "tool_use", + "id": "c", + "name": "Bash", + "input": {"command": "pytest tests/test_x.py"}, + } + ] + }, + }, + { + "type": "user", + "timestamp": "2026-10-31T10:00:45.000Z", + "message": { + "content": [ + {"type": "tool_result", "tool_use_id": "c", "is_error": False, "content": "ok"} + ] + }, + }, + ] + raw = "\n".join(json.dumps(x) for x in lines) + events = convert("claude-stream-json", raw, "/ws").events + assert first_useful(parse_useful({"command": TEST_CMD}), events) == { + "kind": "command", + "reason": None, + "seconds": 42.0, + "steps": 3, + } + assert first_useful(parse_useful({"edit": "src/*.py"}), events) == { + "kind": "edit", + "reason": None, + "seconds": 10.0, + "steps": 2, + } + + +# -- report and compare rows ------------------------------------------------------------------------------------- + + +def _timed(batch: Path, task: str, arm: str, rep: int, useful: dict | None) -> None: + """A finished gate cell whose ``result.json`` carries ``useful`` (left out for ``None``: an older batch).""" + _cell(batch, task, arm, rep) + path = batch / f"{task}__{arm.replace(':', '-')}__m-1__r{rep}" / "result.json" + rec = json.loads(path.read_text()) + if useful is not None: + rec["useful"] = useful + path.write_text(json.dumps(rec)) + + +def _hit(seconds: float, steps: int) -> dict: + return {"kind": "command", "reason": None, "seconds": seconds, "steps": steps} + + +def test_useful_summary_is_the_observed_cells_medians_and_why_the_rest_are_not(): + recs = [ + {"useful": _hit(10.0, 4)}, + {"useful": _hit(20.0, 6)}, + {"useful": {"kind": "command", "reason": "never", "seconds": None, "steps": None}}, + ] + assert useful_summary(recs) == { + "kind": "command", + "n": 3, + "observed": 2, + "median_seconds": 15.0, + "median_steps": 5.0, + "reasons": {"never": 1}, + } + assert useful_summary([{}, {"useful": None}]) is None # an older batch declares nothing + assert useful_line(useful_summary(recs), 4) == [ + " " * 8 + "first useful (command): 2 of 3 observed, median 15.0 s, 5 steps, never 1" + ] + assert useful_line(None, 4) == [] + + +def test_a_task_value_is_the_median_of_the_cells_that_observed_it(): + recs = [ + {"useful": _hit(10.0, 4)}, + {"useful": _hit(30.0, 8)}, + {"useful": _hit(50.0, 6)}, + {"useful": {"kind": "command", "reason": "never", "seconds": None, "steps": None}}, + ] + assert useful_task_values(recs) == {"first_useful_s": [30.0], "first_useful_steps": [6.0]} + assert useful_task_values([{}]) == {"first_useful_s": [None], "first_useful_steps": [None]} + + +def test_repo_report_shows_the_first_useful_line_under_its_row_only_for_a_declared_task(tmp_path): + batch = tmp_path / "b" + _timed(batch, "t1", "guidance:raw", 1, _hit(10.0, 4)) + _timed(batch, "t1", "guidance:raw", 2, _hit(20.0, 6)) + _timed( + batch, + "t1", + "guidance:raw", + 3, + {"kind": "command", "reason": "never", "seconds": None, "steps": None}, + ) + _timed(batch, "t2", "guidance:raw", 1, None) + summary = summarize(batch) + rows = {r["task"]: r for r in summary["rows"]} + assert rows["t1"]["first_useful"] == { + "kind": "command", + "n": 3, + "observed": 2, + "median_seconds": 15.0, + "median_steps": 5.0, + "reasons": {"never": 1}, + } + assert "first_useful" not in rows["t2"] + text = format_report(summary) + assert "first useful (command): 2 of 3 observed, median 15.0 s, 5 steps, never 1" in text + assert text.count("first useful") == 1 + + +def test_repo_compare_pairs_the_first_useful_medians_per_task_and_says_so_for_a_partial_batch( + tmp_path, +): + batch = tmp_path / "b" + base = {"t1": ((10.0, 30.0), (4, 6)), "t2": ((None, None), (None, None))} + treat = {"t1": ((40.0, 50.0), (8, 10))} + for task, (secs, steps) in base.items(): + for rep, (s, n) in enumerate(zip(secs, steps, strict=True), 1): + _timed(batch, task, "guidance:none", rep, None if s is None else _hit(s, n)) + for rep, (s, n) in enumerate(zip(*treat["t1"], strict=True), 1): + _timed(batch, "t1", "guidance:raw", rep, _hit(s, n)) + s = compare([batch], "guidance:none") + row = _row(s, "first_useful_s") + by_task = {t["task"]: t for t in row["tasks"]} + assert by_task["t1"]["baseline"] == 20.0 and by_task["t1"]["treatment"] == 45.0 + assert row["delta"]["mean"] == 25.0 + assert _row(s, "first_useful_steps")["delta"]["mean"] == 4.0 # medians 5 and 9 + assert "t2" not in by_task # no treatment cell of t2 observed one: t2 is left out, and said so + assert {"task": "t2", "measure": None, "reason": "no cells in treatment"}.items() <= next( + u for u in s["unpaired"] if u["task"] == "t2" + ).items() + + +def test_an_older_batch_without_the_field_has_no_first_useful_row(tmp_path): + batch = tmp_path / "old" + for rep in (1, 2): + _timed(batch, "t1", "guidance:none", rep, None) + _timed(batch, "t1", "guidance:raw", rep, None) + s = compare([batch], "guidance:none") + assert not {r["measure"] for r in s["comparisons"]} & {"first_useful_s", "first_useful_steps"} + assert not s["unpaired"] + + +# -- the run itself ---------------------------------------------------------------------------------------------- + + +def test_result_json_records_the_first_edit_of_a_declared_target(setup): # noqa: F811 + task_json = setup["root"] / ".agent-eval/tasks/fix_add/task.json" + task = json.loads(task_json.read_text()) + task["useful"] = {"edit": "src/calc.py"} + task_json.write_text(json.dumps(task)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) + # the fake agent's one Edit, with no timestamps: a hit that is untimed, at step 1 + assert _only_cell(setup)["useful"] == { + "kind": "edit", + "reason": "untimed", + "seconds": None, + "steps": 1, + } + + +def test_result_json_says_a_definition_that_no_command_hits_is_never(setup, monkeypatch): # noqa: F811 + monkeypatch.setenv("FAKE_PROCESS", "1") # a `bazel build` the fake agent runs and fails + task_json = setup["root"] / ".agent-eval/tasks/fix_add/task.json" + task = json.loads(task_json.read_text()) + task["useful"] = {"command": "bazel build"} + task_json.write_text(json.dumps(task)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) + assert _only_cell(setup)["useful"] == { + "kind": "command", + "reason": "never", + "seconds": None, + "steps": None, + } + + +def test_result_json_says_a_task_with_no_definition_is_not_declared(setup): # noqa: F811 + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) + assert _only_cell(setup)["useful"] == { + "kind": None, + "reason": "not_declared", + "seconds": None, + "steps": None, + } From 00be2abd39cac85765a62002b12f0a49a0b1a19b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:51:30 -0700 Subject: [PATCH 02/26] fix(repo): name the first-useful result first_useful, issued_s and median_s Review of the first-useful change before its first push: result.json names the field first_useful (the task.json key is the definition, the result is the hit), its time field is issued_s (the call's issue time, as the repo's other times end in _s), and the report summary's median is median_s. A refusal test drives repo run with a malformed value. The design note says what the medians cover and that the compare means pool the tasks of both kinds. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 4 +- dokimi/docs/repo-evaluation-contract.md | 22 ++++--- dokimi/dokimi/repos/compare.py | 2 +- dokimi/dokimi/repos/report.py | 2 +- dokimi/dokimi/repos/runner.py | 2 +- dokimi/dokimi/repos/useful.py | 22 +++---- dokimi/tests/test_batch_ledger_golden.py | 2 +- dokimi/tests/test_repo_useful.py | 66 +++++++++++-------- 8 files changed, 69 insertions(+), 53 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md index 36c443698..4bbe3e424 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -4,10 +4,10 @@ `"useful": {"command": ""}` for a task graded by its target tests, or `"useful": {"edit": ""}` (or a list of globs) for one graded by an edit to a target file. The first event that hits it (a command whose text matches and whose `rc` is 0, or an - edit whose path matches) is recorded in `result.json` `useful` with its `seconds` (the + edit whose path matches) is recorded in `result.json` `first_useful` with its `issued_s` (the event's issue time) and `steps` (events up to and including it). A run with no hit says why (`never`, `unobserved`, `untimed`); a task with no declaration is `not_declared`. `repo report` shows a row's observed count, medians and reasons under the row, and `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`. A malformed `useful` is refused by `repo validate`, `repo tasks check` and `repo run`. - Batches run before this change have no `useful` and no such rows. + Batches run before this change have no `first_useful` and no such rows. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 47dbe638c..77cf42a80 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -279,7 +279,7 @@ belongs to the repository's grader and is passed through unread, except `grader` | `host` | no | framework | `{"min_cpus": , "cpu_vendors": [], "cpu_flags": []}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not | | `egress` | no | framework | extra hosts (DNS names, optional `:port`; `*` stands for one whole label and the last two labels are literal, so `*.pythonhosted.org` passes and `*` or `*.org` does not) this task's agent and entry points reach through the proxy (§2), for example `["pypi.org", "files.pythonhosted.org"]` | | `expected_skill` | no | framework | the name (directory name) of the skill the agent should use; a `skill-catalog:` arm must install it, or the arm is refused before anything runs. `result.json` `skills` records whether the agent used it and whether it used it first | -| `useful` | no | framework | the definition of the run's first useful result, reported as `useful` in `result.json` (§4.1): `{"command": ""}` for a task whose grade runs the target tests, or `{"edit": ""}` (or a list of globs) for one that grades an edit to a target file. Omit it when no event is the useful result. A malformed value is refused by `repo validate`, `tasks check` and `repo run` | +| `useful` | no | framework | the definition of the run's first useful result, reported as `first_useful` in `result.json` (§4.1): `{"command": ""}` for a task whose grade runs the target tests, or `{"edit": ""}` (or a list of globs) for one that grades an edit to a target file. Omit it when no event is the useful result. A malformed value is refused by `repo validate`, `tasks check` and `repo run` | | anything else | — | repository | for example `hidden_files`, `targets`, `regression_targets`, `manifest` | `setup_patch` (a single string) is accepted as a synonym for a one-element @@ -444,26 +444,30 @@ parent's and subagents' together, so a grader need not: `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). -`result.json` `useful` is the run's first useful result: the first event that +`result.json` `first_useful` is the run's first useful result: the first event that hits the task's `useful` key (§3). The definition depends on the grader: | Grader | Declare | Hit | |---|---|---| | test: the grade runs the target tests | `{"command": ""}` | the first `command` event whose text matches the regex (`re.search`) and whose `rc` is `0`. `rc` 0 says that command passed, not that every target test did, so the regex should name the target tests | | edit or diff: the grade reads a target file | `{"edit": ""}` or a list | the first `edit` event whose path matches a glob (`match_glob`, as the guidance globs) | -| review (`dokimi:review_*`) | none | none: the answer is the product, so no event is the useful result, and the reason is `not_declared` | +| review (`dokimi:review_*`) | none | none, so `not_declared`: the answer is the product, so no event is the useful result. A declared key is measured all the same | | any other repository grader | either form, or none | none declared, `not_declared` | A refused event (`denied`) never hits. `steps` counts every event up to and including the hit, the parent's and subagents' (as `process.tool_calls` does). -`seconds` is the hit's `t`: the time the call was issued, since a call's result -is not timed. `useful` is `{"kind": "command"|"edit"|null, "reason": ..., -"seconds": number|null, "steps": int|null}`. `reason` is `null` when the hit was -found and timed; otherwise it says why `seconds` (and `steps`, with no hit) is +`issued_s` is the hit's `t`: the time the call was issued, since a call's result +is not timed. `first_useful` is `{"kind": "command"|"edit"|null, "reason": ..., +"issued_s": number|null, "steps": int|null}`. `reason` is `null` when the hit was +found and timed; otherwise it says why `issued_s` (and `steps`, with no hit) is null: `not_declared` (no definition), `unobserved` (the trace was not read: the harness never started or it did not parse), `never` (no event hit) or `untimed` -(the trace format has no timestamps, so `steps` is set and `seconds` is null). -A cell of an older batch has no `useful`. +(the trace format has no timestamps, so `steps` is set and `issued_s` is null). +A cell of an older batch has no `first_useful`. Each cell that declares a definition +is counted in `repo report`: the row says how many observed a hit, and the medians +cover only those; `repo compare` has no rate, so read its `first_useful_*` with the +row's observed count. A compared task has one kind, but the mean delta pools the +tasks of both kinds: a per-task row shows which. A cell of an arm that installs skills also has a `skills` object in `result.json`: `measured`, `skills_installed` (the installed skill names), diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index c82890e17..9da0865a6 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -15,7 +15,7 @@ - **first_prompt_tokens**: the mean of ``result.json`` ``agent.first_prompt_tokens`` (what the first main-thread call was sent: start-up plus the task) over the cells that have it. Like a process count, it is compared only when both arms report it for some shared task: a batch run before it was recorded has none; -- **first_useful_s**, **first_useful_steps**: the median seconds and steps to the first useful result (``useful`` +- **first_useful_s**, **first_useful_steps**: the median seconds and steps to the first useful result (``first_useful`` in ``result.json``, :mod:`.useful`) over the cells that observed it; compared like ``first_prompt_tokens``; - **metric:**: the mean of ``grade.json`` ``metrics.`` over the cells that report it (``true`` is 1), for ``--metric``; diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 801611eeb..df1666a11 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -20,7 +20,7 @@ agent used it (``result.json`` ``uptake``, see :mod:`.uptake`) over the cells with a trace to count from. - **first useful**: for a task that declares a ``useful`` definition, the cells that observed the first useful - result (``result.json`` ``useful``, see :mod:`.useful`): how many, their median seconds and steps, and why the + result (``result.json`` ``first_useful``, see :mod:`.useful`): how many, their median seconds and steps, and why the others did not (``never``, ``unobserved``, ``untimed``). - skill arms add ``skill_invoked`` (cells whose agent used the expected skill, or any installed one when the task names none) and, when the task has an diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 39f3680c5..d0d3ac374 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -289,7 +289,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "harness": opts.profile.name, "harness_version": meta["harness_version"], "base_date": meta["base_date"], "execution": execution, "score_task": bool(task.score), "score_max": task.score["max"] if task.score else None, "agent": agent, - "process": process, "useful": first_useful(task.useful, events), "grade": grade} + "process": process, "first_useful": first_useful(task.useful, events), "grade": grade} _add_arm_use(record, meta, task, skills, uptake_of, events) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 080e625a4..bd58f0000 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -5,7 +5,7 @@ The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never -hits. ``steps`` counts every event up to and including the hit, as ``process.tool_calls`` does. ``seconds`` is the +hits. ``steps`` counts every event up to and including the hit, as ``process.tool_calls`` does. ``issued_s`` is the hit's ``t``: the time the call was issued, since its result is not timed, so a test run's pass is not timed. """ @@ -47,21 +47,21 @@ def first_useful( if hits(useful, event): t = event.get("t") if isinstance(t, (int, float)) and not isinstance(t, bool): - return _record(useful.kind, None, seconds=float(t), steps=steps) + return _record(useful.kind, None, issued_s=float(t), steps=steps) return _record(useful.kind, "untimed", steps=steps) # a trace without timestamps return _record(useful.kind, "never") def _record( - kind: str | None, reason: str | None, seconds: float | None = None, steps: int | None = None + kind: str | None, reason: str | None, issued_s: float | None = None, steps: int | None = None ) -> dict[str, Any]: - """``reason`` says why ``seconds`` or ``steps`` is null; it is null when the hit was found and timed.""" - return {"kind": kind, "reason": reason, "seconds": seconds, "steps": steps} + """``reason`` says why ``issued_s`` or ``steps`` is null; it is null when the hit was found and timed.""" + return {"kind": kind, "reason": reason, "issued_s": issued_s, "steps": steps} def _declared(recs: Iterable[Mapping[str, Any]]) -> list[Mapping[str, Any]]: """The ``useful`` objects of the cells that record a definition: an older batch has none.""" - return [u for r in recs if isinstance(u := r.get("useful"), dict) and u.get("kind")] + return [u for r in recs if isinstance(u := r.get("first_useful"), dict) and u.get("kind")] def _nums(values: Iterable[Any]) -> list[float]: @@ -79,13 +79,13 @@ def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: have = _declared(recs) if not have: return None - seconds = _nums(u.get("seconds") for u in have) + seconds = _nums(u.get("issued_s") for u in have) steps = _nums(u.get("steps") for u in have) return { "kind": have[0]["kind"], "n": len(have), "observed": len(steps), - "median_seconds": _median(seconds), + "median_s": _median(seconds), "median_steps": _median(steps), "reasons": dict(Counter(u["reason"] for u in have if u.get("reason"))), } @@ -95,7 +95,7 @@ def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[floa """``repo compare``'s per-task values: the median over the cells that observed it, one value (None if none).""" have = _declared(recs) return { - "first_useful_s": [_median(_nums(u.get("seconds") for u in have))], + "first_useful_s": [_median(_nums(u.get("issued_s") for u in have))], "first_useful_steps": [_median(_nums(u.get("steps") for u in have))], } @@ -105,8 +105,8 @@ def useful_line(summary: Mapping[str, Any] | None, width: int) -> list[str]: if not summary: return [] parts = [f"{summary['observed']} of {summary['n']} observed"] - if summary["median_seconds"] is not None: - parts.append(f"median {summary['median_seconds']:.1f} s") + if summary["median_s"] is not None: + parts.append(f"median {summary['median_s']:.1f} s") if summary["median_steps"] is not None: parts.append(f"{summary['median_steps']:g} steps") parts += [f"{reason} {count}" for reason, count in sorted(summary["reasons"].items())] diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index 942bc29d7..f54c64a81 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -100,6 +100,7 @@ "base_date", "execution", "family", + "first_useful", "grade", "harness", "harness_version", @@ -110,7 +111,6 @@ "score_max", "score_task", "task", - "useful", ] diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 53e8a5589..c1bf2c7f2 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -85,7 +85,7 @@ def test_a_passing_matching_command_is_the_first_useful_result_counted_in_steps_ assert first_useful(parse_useful({"command": TEST_CMD}), events) == { "kind": "command", "reason": None, - "seconds": 6.0, + "issued_s": 6.0, "steps": 6, } @@ -101,7 +101,7 @@ def test_the_first_edit_of_a_target_file_counts_steps_across_every_event_kind(): assert first_useful(parse_useful({"edit": ["src/**/*.cpp", "src/x.py"]}), events) == { "kind": "edit", "reason": None, - "seconds": 2.0, + "issued_s": 2.0, "steps": 4, } @@ -111,7 +111,7 @@ def test_a_run_with_no_hit_is_never_and_names_no_steps(): assert first_useful(parse_useful({"command": TEST_CMD}), events) == { "kind": "command", "reason": "never", - "seconds": None, + "issued_s": None, "steps": None, } @@ -120,7 +120,7 @@ def test_a_run_whose_trace_was_not_read_is_unobserved_not_never(): assert first_useful(parse_useful({"edit": "src/x.py"}), None) == { "kind": "edit", "reason": "unobserved", - "seconds": None, + "issued_s": None, "steps": None, } @@ -129,7 +129,7 @@ def test_a_task_without_a_definition_is_not_declared(): assert first_useful(None, [_edit("src/x.py", t=1.0)]) == { "kind": None, "reason": "not_declared", - "seconds": None, + "issued_s": None, "steps": None, } @@ -139,7 +139,7 @@ def test_a_trace_without_timestamps_keeps_the_steps_and_says_untimed(): assert first_useful(parse_useful({"edit": "src/x.py"}), events) == { "kind": "edit", "reason": "untimed", - "seconds": None, + "issued_s": None, "steps": 2, } @@ -217,13 +217,13 @@ def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): assert first_useful(parse_useful({"command": TEST_CMD}), events) == { "kind": "command", "reason": None, - "seconds": 42.0, + "issued_s": 42.0, "steps": 3, } assert first_useful(parse_useful({"edit": "src/*.py"}), events) == { "kind": "edit", "reason": None, - "seconds": 10.0, + "issued_s": 10.0, "steps": 2, } @@ -237,29 +237,29 @@ def _timed(batch: Path, task: str, arm: str, rep: int, useful: dict | None) -> N path = batch / f"{task}__{arm.replace(':', '-')}__m-1__r{rep}" / "result.json" rec = json.loads(path.read_text()) if useful is not None: - rec["useful"] = useful + rec["first_useful"] = useful path.write_text(json.dumps(rec)) def _hit(seconds: float, steps: int) -> dict: - return {"kind": "command", "reason": None, "seconds": seconds, "steps": steps} + return {"kind": "command", "reason": None, "issued_s": seconds, "steps": steps} def test_useful_summary_is_the_observed_cells_medians_and_why_the_rest_are_not(): recs = [ - {"useful": _hit(10.0, 4)}, - {"useful": _hit(20.0, 6)}, - {"useful": {"kind": "command", "reason": "never", "seconds": None, "steps": None}}, + {"first_useful": _hit(10.0, 4)}, + {"first_useful": _hit(20.0, 6)}, + {"first_useful": {"kind": "command", "reason": "never", "issued_s": None, "steps": None}}, ] assert useful_summary(recs) == { "kind": "command", "n": 3, "observed": 2, - "median_seconds": 15.0, + "median_s": 15.0, "median_steps": 5.0, "reasons": {"never": 1}, } - assert useful_summary([{}, {"useful": None}]) is None # an older batch declares nothing + assert useful_summary([{}, {"first_useful": None}]) is None # an older batch declares nothing assert useful_line(useful_summary(recs), 4) == [ " " * 8 + "first useful (command): 2 of 3 observed, median 15.0 s, 5 steps, never 1" ] @@ -268,10 +268,10 @@ def test_useful_summary_is_the_observed_cells_medians_and_why_the_rest_are_not() def test_a_task_value_is_the_median_of_the_cells_that_observed_it(): recs = [ - {"useful": _hit(10.0, 4)}, - {"useful": _hit(30.0, 8)}, - {"useful": _hit(50.0, 6)}, - {"useful": {"kind": "command", "reason": "never", "seconds": None, "steps": None}}, + {"first_useful": _hit(10.0, 4)}, + {"first_useful": _hit(30.0, 8)}, + {"first_useful": _hit(50.0, 6)}, + {"first_useful": {"kind": "command", "reason": "never", "issued_s": None, "steps": None}}, ] assert useful_task_values(recs) == {"first_useful_s": [30.0], "first_useful_steps": [6.0]} assert useful_task_values([{}]) == {"first_useful_s": [None], "first_useful_steps": [None]} @@ -286,7 +286,7 @@ def test_repo_report_shows_the_first_useful_line_under_its_row_only_for_a_declar "t1", "guidance:raw", 3, - {"kind": "command", "reason": "never", "seconds": None, "steps": None}, + {"kind": "command", "reason": "never", "issued_s": None, "steps": None}, ) _timed(batch, "t2", "guidance:raw", 1, None) summary = summarize(batch) @@ -295,7 +295,7 @@ def test_repo_report_shows_the_first_useful_line_under_its_row_only_for_a_declar "kind": "command", "n": 3, "observed": 2, - "median_seconds": 15.0, + "median_s": 15.0, "median_steps": 5.0, "reasons": {"never": 1}, } @@ -348,10 +348,10 @@ def test_result_json_records_the_first_edit_of_a_declared_target(setup): # noqa task_json.write_text(json.dumps(task)) _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) # the fake agent's one Edit, with no timestamps: a hit that is untimed, at step 1 - assert _only_cell(setup)["useful"] == { + assert _only_cell(setup)["first_useful"] == { "kind": "edit", "reason": "untimed", - "seconds": None, + "issued_s": None, "steps": 1, } @@ -363,19 +363,31 @@ def test_result_json_says_a_definition_that_no_command_hits_is_never(setup, monk task["useful"] = {"command": "bazel build"} task_json.write_text(json.dumps(task)) _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) - assert _only_cell(setup)["useful"] == { + assert _only_cell(setup)["first_useful"] == { "kind": "command", "reason": "never", - "seconds": None, + "issued_s": None, "steps": None, } def test_result_json_says_a_task_with_no_definition_is_not_declared(setup): # noqa: F811 _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) - assert _only_cell(setup)["useful"] == { + assert _only_cell(setup)["first_useful"] == { "kind": None, "reason": "not_declared", - "seconds": None, + "issued_s": None, "steps": None, } + + +def test_repo_run_refuses_a_malformed_definition_before_any_cell_runs(setup): # noqa: F811 + task_json = setup["root"] / ".agent-eval/tasks/fix_add/task.json" + task = json.loads(task_json.read_text()) + task["useful"] = {"edit": 3} + task_json.write_text(json.dumps(task)) + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) + assert not list( + (setup["out"] / "b1").glob("fix_add__*") + ) # refused by validate_tasks: no cell was made From c0773d65fb6efdf3bab618bd566fadc4dded92f7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 14:40:40 -0700 Subject: [PATCH 03/26] feat(repo): first_useful_rate in repo compare, per-arm counts The medians of the first useful result cover only the cells that hit it, so an arm that rarely reaches it looked no slower. repo compare now pairs first_useful_rate per task (observed over declared cells) and its header counts observed of declared cells per arm. The per-task values move to compare_values.py to keep compare.py under its 800-line cap. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 4 +- dokimi/docs/repo-evaluation-contract.md | 11 +- dokimi/dokimi/repos/compare.py | 86 +++------------ dokimi/dokimi/repos/compare_values.py | 104 ++++++++++++++++++ dokimi/dokimi/repos/useful.py | 23 +++- .../tests/test_repo_report_unreadable_cell.py | 8 +- dokimi/tests/test_repo_useful.py | 52 ++++++++- 7 files changed, 203 insertions(+), 85 deletions(-) create mode 100644 dokimi/dokimi/repos/compare_values.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md index 4bbe3e424..bbee7a43c 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -8,6 +8,8 @@ event's issue time) and `steps` (events up to and including it). A run with no hit says why (`never`, `unobserved`, `untimed`); a task with no declaration is `not_declared`. `repo report` shows a row's observed count, medians and reasons under the row, and - `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`. + `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`, and + the share of declared cells that observed a hit as `first_useful_rate`; its header counts + observed of declared cells per arm. A malformed `useful` is refused by `repo validate`, `repo tasks check` and `repo run`. Batches run before this change have no `first_useful` and no such rows. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 77cf42a80..46e90e890 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -465,9 +465,11 @@ harness never started or it did not parse), `never` (no event hit) or `untimed` (the trace format has no timestamps, so `steps` is set and `issued_s` is null). A cell of an older batch has no `first_useful`. Each cell that declares a definition is counted in `repo report`: the row says how many observed a hit, and the medians -cover only those; `repo compare` has no rate, so read its `first_useful_*` with the -row's observed count. A compared task has one kind, but the mean delta pools the -tasks of both kinds: a per-task row shows which. +cover only those. `repo compare` pairs `first_useful_rate` per task (the share of its +declared cells that observed a hit, so a task where most runs never hit it is not read +through its medians alone) and its header counts observed of declared cells per arm. A +compared task has one kind, but the mean delta pools the tasks of both kinds: a per-task +row shows which. A cell of an arm that installs skills also has a `skills` object in `result.json`: `measured`, `skills_installed` (the installed skill names), @@ -1481,7 +1483,8 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, cost, turns, `first_prompt_tokens`, `first_useful_s` and `first_useful_steps` (the median over a - task's cells that observed the first useful result, §4.1), each `--metric` from `grade.json`, and each `process` count a + task's cells that observed the first useful result, §4.1), `first_useful_rate` (the share of a task's + declared cells that observed one), each `--metric` from `grade.json`, and each `process` count a compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for gate and score, a t interval for the descriptive measures, `-` for a single diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 9da0865a6..db76a8533 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -17,6 +17,8 @@ compared only when both arms report it for some shared task: a batch run before it was recorded has none; - **first_useful_s**, **first_useful_steps**: the median seconds and steps to the first useful result (``first_useful`` in ``result.json``, :mod:`.useful`) over the cells that observed it; compared like ``first_prompt_tokens``; +- **first_useful_rate**: the share of the cells that declare a first useful result and observed one, per task; the + header counts observed of declared cells per arm; - **metric:**: the mean of ``grade.json`` ``metrics.`` over the cells that report it (``true`` is 1), for ``--metric``; - **pass^k**: for ``--pass-k K``, the chance that K cells drawn from the @@ -107,16 +109,17 @@ from pathlib import Path from typing import Any -from dokimi.context.consistency import Run, order_distance, pass_hat, run_from_events +from dokimi.context.consistency import Run, run_from_events from dokimi.eval.stats import bootstrap_ci, correct_family, mean_ci from dokimi.utils import same_model from .compare_reach import ALPHA, holm_floor, min_detectable from .compare_tasks import CompareError, select_tasks -from .process_counts import as_measure, process_lists, process_order, process_values -from .useful import USEFUL_MEASURES, useful_task_values -from .report import (load_cells, mismatch_warnings, model_mismatch, reading_cells, resolved_models, +from .compare_values import finite_values, task_values +from .process_counts import as_measure, process_order, process_values +from .report import (load_cells, mismatch_warnings, model_mismatch, resolved_models, unmeasured_note, uptake_counts) +from .useful import USEFUL_MEASURES, useful_arms_line, useful_counts #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns). EXACT_MAX = 16 @@ -196,14 +199,17 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None measures = ["gate", "score", *([f"pass^{pass_k}"] if pass_k else []), *(["path_distance"] if path_stability else []), "cost_usd", "turns", "first_prompt_tokens", *USEFUL_MEASURES, *(f"metric:{m}" for m in metrics)] - base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics, pass_k, path_stability) + base_cells = [c for c in cells if selector_matches(base_sel, c)] + base_vals = task_values(base_cells, metrics, pass_k, path_stability) + useful = {baseline: useful_counts(base_cells)} rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] one_arm: list[str] = [] measures += _process_measures(cells, [baseline, *treatments]) for text in treatments: - treat_vals = _task_values([c for c in cells if selector_matches(parse_selector(text), c)], metrics, - pass_k, path_stability) + treat_cells = [c for c in cells if selector_matches(parse_selector(text), c)] + treat_vals = task_values(treat_cells, metrics, pass_k, path_stability) + useful[text] = useful_counts(treat_cells) unpaired += _one_arm_models(text, base_vals, treat_vals, models) shared_models = sorted({m for m, _ in base_vals} & {m for m, _ in treat_vals}) for model in [m for m in shared_models if not models or m in models]: @@ -217,7 +223,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None provenance["warnings"] += one_arm return {"batches": [str(d) for d in batch_dirs], "baseline": baseline, "treatments": treatments, "task_filter": task_filter, "cells": len(cells), "missing": missing, **provenance, - "comparisons": rows, "unpaired": unpaired} + "first_useful": useful, "comparisons": rows, "unpaired": unpaired} def _one_arm_models(treatment: str, base_vals: dict[tuple[str, str], dict[str, Any]], @@ -679,68 +685,14 @@ def _format_header(summary: dict[str, Any]) -> list[str]: f"{', '.join(summary['task_filter']['dropped_arms'])}") for text, batches in summary["sources"].items(): lines.append(f" {text}: {', '.join(batches)}") + if (useful := useful_arms_line(summary.get("first_useful", {}))): + lines.append(" " + useful) for name, stamp in summary["stamps"].items(): shown = [f"{k} {stamp[k][:12] if k in ('eval_dir', 'graders') else stamp[k]}" for k in ("tree", *STAMP_KEYS) if k in stamp] lines.append(f" batch {name}: " + ", ".join([*shown, "harness " + ", ".join(stamp["harness_versions"])])) return lines -def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...], pass_k: int | None = None, - path_stability: bool = False) -> dict[tuple[str, str], dict[str, Any]]: - """One value per measure for each (model, task) of one arm.""" - groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) - for rec in cells: - groups[(rec["model"], rec["task"])].append(rec) - out = {} - for key, recs in groups.items(): - with reading_cells([f"{r['batch']}/{r['cell']}/result.json" for r in recs]): - score_task = any(r["score_task"] for r in recs) - found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs], - "turns": [r["agent"].get("num_turns") for r in recs], - "first_prompt_tokens": [r["agent"].get("first_prompt_tokens") for r in recs], - **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}} - found.update({**process_lists(recs), **useful_task_values(recs)}) - if score_task: - found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0) - for r in recs] - else: - found["gate"] = [float(bool(r["grade"]["success"])) for r in recs] - # unmeasured: a grader failure, or an agent outage (see _agent_outage); either way its 0 is not a measurement - grader_failed = any(r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs) - nums = {measure: _finite(vals) for measure, vals in found.items()} - score_max_vals = [r.get("score_max") for r in recs] - maxes = set(_finite(score_max_vals)) # cells of two eval trees may differ - # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value - has_unknown_max = any(v is None for v in score_max_vals) - top = (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) if score_task else 1.0 - n = {measure: len(v) for measure, v in nums.items()} - values = {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()} - if pass_k and not score_task: - gates = nums["gate"] - values[f"pass^{pass_k}"] = pass_hat(len(gates), int(sum(gates)), pass_k) if len(gates) >= pass_k else None - n[f"pass^{pass_k}"] = len(gates) - if path_stability: - runs: list[Run] = [r["path"] for r in recs if r.get("path") is not None] - values["path_distance"] = order_distance(runs) - n["path_distance"] = len(runs) - out[key] = {"kind": "score" if score_task else "gate", "grader_failed": grader_failed, - "top": top, "n": n, "values": values} - return out - - -def _agent_outage(rec: dict[str, Any]) -> bool: - """A cell whose agent never ran: an error result with no turn count or at most one turn, or a harness that - exited non-zero before it reported any turn (no result record, so no ``is_error``) and was not killed for - its timeout: an agent killed at its timeout did attempt the task.""" - agent = rec.get("agent") or {} - turns = _finite([agent.get("num_turns")]) # a count that is not a number is no count, as in the turns row - if agent.get("is_error") and (not turns or turns[0] <= 1): - return True - rc = agent.get("rc") - crashed = isinstance(rc, int) and not isinstance(rc, bool) and rc != 0 - return crashed and not turns and not agent.get("timed_out") - - def _cell_path(path: Path) -> Run | None: """A cell's main-thread tool path from its ``events.jsonl``; ``None`` when it has none.""" try: @@ -776,12 +728,8 @@ def _grade_metrics(path: Path, names: tuple[str, ...]) -> dict[str, float]: return out -def _finite(values: list[Any]) -> list[float]: - return [num for v in values if (num := as_measure(v)) is not None] - - def _mean(values: list[Any]) -> float | None: - nums = _finite(values) + nums = finite_values(values) return sum(nums) / len(nums) if nums else None diff --git a/dokimi/dokimi/repos/compare_values.py b/dokimi/dokimi/repos/compare_values.py new file mode 100644 index 000000000..fc3b329db --- /dev/null +++ b/dokimi/dokimi/repos/compare_values.py @@ -0,0 +1,104 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The per-task values ``repo compare`` pairs: one value per measure for each (model, task) of one arm. + +Split out of :mod:`.compare` to keep that module under its line cap. The values are the cells' means, medians +and rates, the process counts and the first-useful measures (:mod:`.useful`). +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any + +from dokimi.context.consistency import Run, order_distance, pass_hat + +from .process_counts import as_measure, process_lists +from .report import reading_cells +from .useful import useful_task_values + + +def finite_values(values: list[Any]) -> list[float]: + return [num for v in values if (num := as_measure(v)) is not None] + + +def task_values( + cells: list[dict[str, Any]], + metrics: tuple[str, ...], + pass_k: int | None = None, + path_stability: bool = False, +) -> dict[tuple[str, str], dict[str, Any]]: + """One value per measure for each (model, task) of one arm.""" + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for rec in cells: + groups[(rec["model"], rec["task"])].append(rec) + out = {} + for key, recs in groups.items(): + with reading_cells([f"{r['batch']}/{r['cell']}/result.json" for r in recs]): + score_task = any(r["score_task"] for r in recs) + found = { + "cost_usd": [r["agent"].get("cost_usd") for r in recs], + "turns": [r["agent"].get("num_turns") for r in recs], + "first_prompt_tokens": [r["agent"].get("first_prompt_tokens") for r in recs], + **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}, + } + found.update({**process_lists(recs), **useful_task_values(recs)}) + if score_task: + found["score"] = [ + 0.0 + if r["grade"]["status"] == "grader_failure" + else (r["grade"]["score"] or 0.0) + for r in recs + ] + else: + found["gate"] = [float(bool(r["grade"]["success"])) for r in recs] + # unmeasured: a grader failure, or an agent outage (see _agent_outage); either way its 0 is not a measurement + grader_failed = any( + r["grade"]["status"] == "grader_failure" or _agent_outage(r) for r in recs + ) + nums = {measure: finite_values(vals) for measure, vals in found.items()} + score_max_vals = [r.get("score_max") for r in recs] + maxes = set(finite_values(score_max_vals)) # cells of two eval trees may differ + # If any cell has unknown max, or maxes differ, top is None; otherwise take the single value + has_unknown_max = any(v is None for v in score_max_vals) + top = ( + (maxes.pop() if len(maxes) == 1 and not has_unknown_max else None) + if score_task + else 1.0 + ) + n = {measure: len(v) for measure, v in nums.items()} + values = {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()} + if pass_k and not score_task: + gates = nums["gate"] + values[f"pass^{pass_k}"] = ( + pass_hat(len(gates), int(sum(gates)), pass_k) if len(gates) >= pass_k else None + ) + n[f"pass^{pass_k}"] = len(gates) + if path_stability: + runs: list[Run] = [r["path"] for r in recs if r.get("path") is not None] + values["path_distance"] = order_distance(runs) + n["path_distance"] = len(runs) + out[key] = { + "kind": "score" if score_task else "gate", + "grader_failed": grader_failed, + "top": top, + "n": n, + "values": values, + } + return out + + +def _agent_outage(rec: dict[str, Any]) -> bool: + """A cell whose agent never ran: an error result with no turn count or at most one turn, or a harness that + exited non-zero before it reported any turn (no result record, so no ``is_error``) and was not killed for + its timeout: an agent killed at its timeout did attempt the task.""" + agent = rec.get("agent") or {} + turns = finite_values( + [agent.get("num_turns")] + ) # a count that is not a number is no count, as in the turns row + if agent.get("is_error") and (not turns or turns[0] <= 1): + return True + rc = agent.get("rc") + crashed = isinstance(rc, int) and not isinstance(rc, bool) and rc != 0 + return crashed and not turns and not agent.get("timed_out") diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index bd58f0000..9f467f899 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -20,7 +20,7 @@ from .useful_spec import Useful #: The ``repo compare`` measures of the first useful result, one value per task (see :func:`useful_task_values`). -USEFUL_MEASURES = ("first_useful_s", "first_useful_steps") +USEFUL_MEASURES = ("first_useful_s", "first_useful_steps", "first_useful_rate") def hits(useful: Useful, event: Mapping[str, Any]) -> bool: @@ -92,14 +92,31 @@ def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[float | None]]: - """``repo compare``'s per-task values: the median over the cells that observed it, one value (None if none).""" + """``repo compare``'s per-task values: the medians over the cells that observed it, and the share of the declared + cells that did (None if no cell declares it). The medians alone see only the observed cells.""" have = _declared(recs) + observed = _nums(u.get("steps") for u in have) return { "first_useful_s": [_median(_nums(u.get("issued_s") for u in have))], - "first_useful_steps": [_median(_nums(u.get("steps") for u in have))], + "first_useful_steps": [_median(observed)], + "first_useful_rate": [len(observed) / len(have) if have else None], } +def useful_counts(recs: Sequence[Mapping[str, Any]]) -> dict[str, int] | None: + """One arm's cells that observed a first useful result, of those that declare one; None if none declare.""" + have = _declared(recs) + if not have: + return None + return {"observed": len(_nums(u.get("steps") for u in have)), "declared": len(have)} + + +def useful_arms_line(counts: Mapping[str, Mapping[str, int] | None]) -> str | None: + """The compare header's line of observed-of-declared cells per arm; None when no arm declares one.""" + shown = [f"{arm} {c['observed']} of {c['declared']}" for arm, c in counts.items() if c] + return f"first useful observed: {', '.join(shown)}" if shown else None + + def useful_line(summary: Mapping[str, Any] | None, width: int) -> list[str]: """The row's first-useful summary on one indented line; none for a row without a definition.""" if not summary: diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py index 17695312a..86ccf4ff5 100644 --- a/dokimi/tests/test_repo_report_unreadable_cell.py +++ b/dokimi/tests/test_repo_report_unreadable_cell.py @@ -341,13 +341,13 @@ def test_a_cell_shape_the_checks_miss_exits_2_naming_the_cell(tmp_path, capsys, @pytest.mark.parametrize("exc", [KeyError("k"), TypeError("t"), AttributeError("a"), OverflowError("o"), ValueError("v")], ids=lambda e: type(e).__name__) def test_each_cell_shape_error_exits_2_naming_the_cell(tmp_path, capsys, monkeypatch, cmd, exc): - from dokimi.repos import compare, report + from dokimi.repos import compare_values, report def boom(recs): raise exc monkeypatch.setattr(report, "process_means", boom) - monkeypatch.setattr(compare, "process_lists", boom) + monkeypatch.setattr(compare_values, "process_lists", boom) # the compare path reads process counts here batch = _two_arm_batch(tmp_path, _GOOD) with pytest.raises(SystemExit) as raised: _run(cmd[0], str(batch), *cmd[1:]) @@ -358,12 +358,12 @@ def boom(recs): @pytest.mark.parametrize("target", ["pass_hat", "order_distance"]) def test_an_error_in_compare_statistics_is_not_reported_as_a_bad_cell(tmp_path, monkeypatch, target): - from dokimi.repos import compare + from dokimi.repos import compare_values def boom(*args): raise KeyError("a defect, not a cell") - monkeypatch.setattr(compare, target, boom) + monkeypatch.setattr(compare_values, target, boom) batch = _two_arm_batch(tmp_path, _GOOD) with pytest.raises(KeyError, match="a defect, not a cell"): _run("compare", str(batch), "--baseline", "a", "--pass-k", "1", "--path-stability") diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index c1bf2c7f2..2705dbdfd 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -13,11 +13,18 @@ from dokimi.repos.events import convert from dokimi.repos.host import validate_tasks from dokimi.repos.report import format_report, summarize -from dokimi.repos.useful import first_useful, useful_line, useful_summary, useful_task_values +from dokimi.repos.useful import ( + first_useful, + useful_arms_line, + useful_counts, + useful_line, + useful_summary, + useful_task_values, +) from dokimi.repos.useful_spec import UsefulSpecError, parse_useful from dokimi.repos.workspace import Task -from dokimi.repos.compare import compare +from dokimi.repos.compare import compare, format_compare from test_repo_compare import _cell, _row from test_repo_run import _only_cell, _run, setup # noqa: F401 # the run fixture, with its helpers @@ -273,8 +280,23 @@ def test_a_task_value_is_the_median_of_the_cells_that_observed_it(): {"first_useful": _hit(50.0, 6)}, {"first_useful": {"kind": "command", "reason": "never", "issued_s": None, "steps": None}}, ] - assert useful_task_values(recs) == {"first_useful_s": [30.0], "first_useful_steps": [6.0]} - assert useful_task_values([{}]) == {"first_useful_s": [None], "first_useful_steps": [None]} + assert useful_task_values(recs) == { + "first_useful_s": [30.0], + "first_useful_steps": [6.0], + "first_useful_rate": [0.75], + } + assert useful_task_values([{}]) == { + "first_useful_s": [None], + "first_useful_steps": [None], + "first_useful_rate": [None], + } + assert useful_counts(recs) == {"observed": 3, "declared": 4} + assert useful_counts([{}]) is None + assert ( + useful_arms_line({"a": {"observed": 1, "declared": 2}, "b": None}) + == "first useful observed: a 1 of 2" + ) + assert useful_arms_line({"a": None}) is None def test_repo_report_shows_the_first_useful_line_under_its_row_only_for_a_declared_task(tmp_path): @@ -338,6 +360,28 @@ def test_an_older_batch_without_the_field_has_no_first_useful_row(tmp_path): assert not s["unpaired"] +def test_the_rate_says_how_many_declared_cells_observed_a_hit_where_the_medians_see_only_those_that_did( + tmp_path, +): + batch = tmp_path / "b" + never = {"kind": "command", "reason": "never", "issued_s": None, "steps": None} + _timed(batch, "t1", "guidance:none", 1, _hit(10.0, 4)) + _timed(batch, "t1", "guidance:none", 2, never) + _timed(batch, "t1", "guidance:raw", 1, _hit(10.0, 4)) + _timed(batch, "t1", "guidance:raw", 2, _hit(12.0, 5)) + s = compare([batch], "guidance:none") + by_task = {t["task"]: t for t in _row(s, "first_useful_rate")["tasks"]} + assert by_task["t1"]["baseline"] == 0.5 and by_task["t1"]["treatment"] == 1.0 + assert _row(s, "first_useful_rate")["delta"]["mean"] == 0.5 + assert by_task["t1"]["baseline"] < 1.0 # the medians alone would read this arm as no slower + assert _row(s, "first_useful_s")["delta"]["mean"] == 1.0 # medians 10 and 11 + assert s["first_useful"] == { + "guidance:none": {"observed": 1, "declared": 2}, + "guidance:raw": {"observed": 2, "declared": 2}, + } + assert "first useful observed: guidance:none 1 of 2, guidance:raw 2 of 2" in format_compare(s) + + # -- the run itself ---------------------------------------------------------------------------------------------- From a221219729a0e36f8dc6024e4071bd589814832a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 14:48:13 -0700 Subject: [PATCH 04/26] fix(repo): a failed edit is no first useful result An Edit whose tool result is an error left the file unchanged, but it counted as the first useful edit at its issue time. The event now carries rc 1 on that error, as a command does, and hits() refuses it. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/events.py | 3 +++ dokimi/dokimi/repos/useful.py | 5 +++-- dokimi/tests/test_repo_useful.py | 13 ++++++++++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 46e90e890..7633af89e 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -454,7 +454,7 @@ hits the task's `useful` key (§3). The definition depends on the grader: | review (`dokimi:review_*`) | none | none, so `not_declared`: the answer is the product, so no event is the useful result. A declared key is measured all the same | | any other repository grader | either form, or none | none declared, `not_declared` | -A refused event (`denied`) never hits. `steps` counts every event up to and +A refused event (`denied`) never hits, nor does an `edit` whose tool result is an error (`rc` 1: the file was not changed). `steps` counts every event up to and including the hit, the parent's and subagents' (as `process.tool_calls` does). `issued_s` is the hit's `t`: the time the call was issued, since a call's result is not timed. `first_useful` is `{"kind": "command"|"edit"|null, "reason": ..., diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 12a0ceda0..3a424a5ec 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -101,6 +101,9 @@ def _apply_result(block: dict[str, Any], by_id: dict[str, dict[str, Any]]) -> No if result_of["kind"] == "read" and not denied: # A read that failed (the file does not exist) shows nothing of the file. result_of["rc"] = 1 if block.get("is_error") else 0 + if result_of["kind"] == "edit" and not denied: + # A failed edit (old_string not found) left the file as it was: rc 1, so it is no useful result. + result_of["rc"] = 1 if block.get("is_error") else 0 if result_of["kind"] == "command": # Claude Code reports success or failure only, with stdout and stderr merged into one text. if not denied: diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 9f467f899..4d9e90839 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -5,7 +5,7 @@ The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never -hits. ``steps`` counts every event up to and including the hit, as ``process.tool_calls`` does. ``issued_s`` is the +hits, nor does an edit whose tool result is an error (``rc`` 1). ``steps`` counts every event up to and including the hit, as ``process.tool_calls`` does. ``issued_s`` is the hit's ``t``: the time the call was issued, since its result is not timed, so a test run's pass is not timed. """ @@ -32,7 +32,8 @@ def hits(useful: Useful, event: Mapping[str, Any]) -> bool: matched = bool(useful.command and useful.command.search(text)) return event.get("kind") == "command" and event.get("rc") == 0 and matched path = str(event.get("path") or "") - return event.get("kind") == "edit" and any(match_glob(g, path) for g in useful.globs) + failed = event.get("rc") == 1 # an edit whose tool result is an error changed nothing + return event.get("kind") == "edit" and not failed and any(match_glob(g, path) for g in useful.globs) def first_useful( diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 2705dbdfd..ccc2b478b 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -10,7 +10,7 @@ import pytest -from dokimi.repos.events import convert +from dokimi.repos.events import _apply_result, convert from dokimi.repos.host import validate_tasks from dokimi.repos.report import format_report, summarize from dokimi.repos.useful import ( @@ -238,6 +238,17 @@ def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): # -- report and compare rows ------------------------------------------------------------------------------------- +def test_a_failed_edit_changed_nothing_so_the_next_successful_one_is_the_first_useful_result(): + failed = {"t": 4.0, "kind": "edit", "path": "src/foo.py"} + _apply_result({"tool_use_id": "u1", "is_error": True, "content": "String to replace not found in file."}, + {"u1": failed}) + ok = {"t": 5.0, "kind": "edit", "path": "src/foo.py"} + _apply_result({"tool_use_id": "u2", "is_error": False, "content": "edited"}, {"u2": ok}) + u = parse_useful({"edit": "src/*.py"}) + assert first_useful(u, [failed]) == {"kind": "edit", "reason": "never", "issued_s": None, "steps": None} + assert first_useful(u, [failed, ok]) == {"kind": "edit", "reason": None, "issued_s": 5.0, "steps": 2} + + def _timed(batch: Path, task: str, arm: str, rep: int, useful: dict | None) -> None: """A finished gate cell whose ``result.json`` carries ``useful`` (left out for ``None``: an older batch).""" _cell(batch, task, arm, rep) From 4f38f1e68bcbcefec25b381a05302e58f8f85f4b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 17:45:25 -0700 Subject: [PATCH 05/26] fix(repo): a copilot edit takes its completion's success; a deep useful regex is refused Copilot edit events never read the completion, so a failed edit counted as a first useful result. A regex nested deeply enough to overflow the parser raised RecursionError out of parse_useful instead of UsefulSpecError. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 3 +++ dokimi/dokimi/repos/useful_spec.py | 2 +- dokimi/tests/test_repo_cli_traces.py | 12 ++++++++++++ dokimi/tests/test_repo_useful.py | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 8233e1f55..12ec8de34 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -752,6 +752,9 @@ def _copilot_result(out: Transcript, events: list[dict[str, Any]], data: dict[st reads = [e for e in events if e["kind"] == "read"] failed = data.get("success") is False for event in events: + if event["kind"] == "edit" and data.get("success") is not None: + # a failed edit changed nothing: rc 1, so it is no useful result + event["rc"] = 0 if data.get("success") else 1 if event["kind"] != "command": continue shell = data.get("shellExecution") or {} diff --git a/dokimi/dokimi/repos/useful_spec.py b/dokimi/dokimi/repos/useful_spec.py index 047f1d1a0..6fec09dd6 100644 --- a/dokimi/dokimi/repos/useful_spec.py +++ b/dokimi/dokimi/repos/useful_spec.py @@ -41,7 +41,7 @@ def parse_useful(value: Any) -> Useful | None: if kind == "command" and isinstance(pattern, str) and pattern: try: return Useful("command", command=re.compile(pattern)) - except re.error as exc: + except (re.error, RecursionError) as exc: # a deeply nested pattern overflows the parser raise UsefulSpecError(f"useful command is not a valid regex ({exc})") from exc globs = [pattern] if isinstance(pattern, str) else pattern if ( diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 50f20685e..a277b11e9 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -583,6 +583,18 @@ def test_copilot_failed_view_is_not_a_read(success, reads): assert convert("copilot-json", raw, "/ws").reads() == reads +@pytest.mark.parametrize(("success", "rc"), [(False, 1), (True, 0), (None, None)]) +def test_copilot_edit_rc_is_the_completion_success(success, rc): + done = {"toolCallId": "e"} if success is None else {"toolCallId": "e", "success": success} + raw = _jsonl( + {"type": "tool.execution_start", + "data": {"toolCallId": "e", "toolName": "edit", "arguments": {"path": "/ws/src/a.py"}}}, + {"type": "tool.execution_complete", "data": done}, + ) + edits = [e for e in convert("copilot-json", raw, "/ws").events if e["kind"] == "edit"] + assert [e.get("rc") for e in edits] == [rc] + + def test_copilot_result_without_an_exit_code_is_not_an_error(): t = convert("copilot-json", _jsonl({"type": "assistant.message", "data": {"content": "x"}}, {"type": "result"}), None) diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index ccc2b478b..05f69fae8 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -58,6 +58,7 @@ def test_the_two_kinds_parse_to_their_pattern_and_globs(): [ ({"command": ""}, "useful must be"), ({"command": "("}, "not a valid regex"), + ({"command": "(" * 2000 + ")" * 2000}, "not a valid regex"), # overflows, not re.error ({"edit": []}, "useful must be"), ({"edit": ["src/a.py", ""]}, "useful must be"), ({"edit": 3}, "useful must be"), From 4586da3dc1f0b0cf749475fdf5342e1faa9029ba Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:01:48 -0700 Subject: [PATCH 06/26] fix(repo): a huge useful repeat count is refused, not an OverflowError parse_useful caught re.error and RecursionError only, so a repeat count the regex parser cannot hold escaped as OverflowError and repo validate, tasks check and repo run crashed with a traceback instead of the refusal. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/useful_spec.py | 3 ++- dokimi/tests/test_repo_useful.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/useful_spec.py b/dokimi/dokimi/repos/useful_spec.py index 6fec09dd6..18f4662eb 100644 --- a/dokimi/dokimi/repos/useful_spec.py +++ b/dokimi/dokimi/repos/useful_spec.py @@ -39,9 +39,10 @@ def parse_useful(value: Any) -> Useful | None: raise UsefulSpecError(SHAPE) ((kind, pattern),) = value.items() if kind == "command" and isinstance(pattern, str) and pattern: + # a deep nesting or a huge repeat count overflows the parser, not only re.error try: return Useful("command", command=re.compile(pattern)) - except (re.error, RecursionError) as exc: # a deeply nested pattern overflows the parser + except (re.error, RecursionError, OverflowError) as exc: raise UsefulSpecError(f"useful command is not a valid regex ({exc})") from exc globs = [pattern] if isinstance(pattern, str) else pattern if ( diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 05f69fae8..e06af6096 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -59,6 +59,7 @@ def test_the_two_kinds_parse_to_their_pattern_and_globs(): ({"command": ""}, "useful must be"), ({"command": "("}, "not a valid regex"), ({"command": "(" * 2000 + ")" * 2000}, "not a valid regex"), # overflows, not re.error + ({"command": "a{99999999999}"}, "not a valid regex"), # a repeat count too large for the parser ({"edit": []}, "useful must be"), ({"edit": ["src/a.py", ""]}, "useful must be"), ({"edit": 3}, "useful must be"), From c36bc805a207cc5aa0d937273f5ec99e5b33f070 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 23:12:31 -0700 Subject: [PATCH 07/26] fix(repo): count useful results per task and per model, keep kinds and paths - header counts (first_useful observed/declared) only for the models compared - useful_task_counts: timed, observed and declared counts per task, not a single aggregate; a kind mismatch between arms is unpaired, not compared - hits() normalizes the path into the workspace and refuses paths outside it - a useful_spec repeat count too large for int is refused, not an OverflowError - NaN and huge numbers go through as_measure - a run whose trace had dropped lines reports the result as unobserved - trace lines that are valid JSON but not an object count as dropped lines Thread: PRRT_kwDOVBg7g86yWRo0 Thread: PRRT_kwDOVBg7g86yWRo4 Thread: PRRT_kwDOVBg7g86yWRo_ Thread: PRRT_kwDOVBg7g86yWRpJ Thread: PRRT_kwDOVBg7g86yWRpM Thread: PRRT_kwDOVBg7g86yWRt8 Thread: PRRT_kwDOVBg7g86yWRuM Thread: PRRT_kwDOVBg7g86yWRuT Thread: PRRT_kwDOVBg7g86yWRuc Thread: PRRT_kwDOVBg7g86yWRuk Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 6 + dokimi/dokimi/repos/compare.py | 31 ++++- dokimi/dokimi/repos/compare_values.py | 4 +- dokimi/dokimi/repos/events.py | 3 + dokimi/dokimi/repos/runner.py | 4 +- dokimi/dokimi/repos/transcript.py | 1 + dokimi/dokimi/repos/useful.py | 38 +++++- dokimi/dokimi/repos/useful_spec.py | 2 +- dokimi/tests/test_repo_useful.py | 162 +++++++++++++++++++++++++- 9 files changed, 233 insertions(+), 18 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 12ec8de34..422a20e54 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -531,13 +531,16 @@ def codex_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: try: rec = json.loads(line) except json.JSONDecodeError: + out.dropped_lines += 1 out.errors.append(f"line {n}: not JSON") continue if not isinstance(rec, dict): + out.dropped_lines += 1 continue try: _codex_apply(out, rec, messages, workdir, by_id, cwd, event_reads) except (AttributeError, TypeError, ValueError) as exc: + out.dropped_lines += 1 out.errors.append(f"line {n}: unexpected shape ({exc})") out.answer = messages[-1].strip() if messages else "" out.agent_messages = messages @@ -712,14 +715,17 @@ def copilot_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: try: rec = json.loads(line) except json.JSONDecodeError: + out.dropped_lines += 1 out.errors.append(f"line {n}: not JSON") continue if not isinstance(rec, dict): + out.dropped_lines += 1 continue t = run.offset(rec) try: run.apply(out, rec, t, workdir, cwd) except (AttributeError, TypeError, ValueError) as exc: + out.dropped_lines += 1 out.errors.append(f"line {n}: unexpected shape ({exc})") out.answer = (run.final or run.last).strip() out.num_turns = run.turns or None diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index db76a8533..e6c714e5e 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -201,7 +201,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None *USEFUL_MEASURES, *(f"metric:{m}" for m in metrics)] base_cells = [c for c in cells if selector_matches(base_sel, c)] base_vals = task_values(base_cells, metrics, pass_k, path_stability) - useful = {baseline: useful_counts(base_cells)} + useful = {baseline: useful_counts(_in_models(base_cells, models))} rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] one_arm: list[str] = [] @@ -209,7 +209,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None for text in treatments: treat_cells = [c for c in cells if selector_matches(parse_selector(text), c)] treat_vals = task_values(treat_cells, metrics, pass_k, path_stability) - useful[text] = useful_counts(treat_cells) + useful[text] = useful_counts(_in_models(treat_cells, models)) unpaired += _one_arm_models(text, base_vals, treat_vals, models) shared_models = sorted({m for m, _ in base_vals} & {m for m, _ in treat_vals}) for model in [m for m in shared_models if not models or m in models]: @@ -366,6 +366,11 @@ def _holm(rows: list[dict[str, Any]]) -> None: row["mde"] = row["mde"] if reachable else None +def _in_models(cells: list[dict[str, Any]], models: list[str] | None) -> list[dict[str, Any]]: + """The cells of the models compared (all of them without ``--model``).""" + return [c for c in cells if not models or c["model"] in models] + + def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], measures: list[str], seed: int, rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> list[str]: @@ -379,20 +384,28 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], unpaired.append(_unpaired(treatment, model, task, None, "no cells in " + ("treatment" if task in base_tasks else "baseline"))) shared = sorted(base_tasks & treat_tasks) + useful_ok = [] # the tasks whose first useful results pair: one kind in both arms, or a declaration in one for task in shared: b, t = base_vals[(model, task)]["kind"], treat_vals[(model, task)]["kind"] if b != t: # batches of different eval trees: the primary result of this task has no pair unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment")) + bk, tk = base_vals[(model, task)]["useful_kind"], treat_vals[(model, task)]["useful_kind"] + if (bk and tk and bk != tk) or "mixed" in (bk, tk): # a command's timing and an edit's are not one measure + unpaired.append(_unpaired(treatment, model, task, None, + f"first useful kinds differ: {bk} in baseline, {tk} in treatment")) + else: + useful_ok.append(task) skipped = [] for measure in measures: + tasks = useful_ok if measure.startswith("first_useful") else shared if measure.startswith(REPORTED): - has = [any(vals[(model, t)]["values"].get(measure) is not None for t in shared) + has = [any(vals[(model, t)]["values"].get(measure) is not None for t in tasks) for vals in (base_vals, treat_vals)] if not all(has): # an arm that reports none of it at all (an older batch) leaves no task out of a pair if any(has): skipped.append(measure) continue - row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired) + row = _compare_measure(treatment, model, measure, tasks, base_vals, treat_vals, seed, unpaired) if row is not None: rows.append(row) return skipped @@ -412,6 +425,9 @@ def _measure_pairs(treatment: str, model: str, shared: list[str], base_vals: dic continue # a task of the other kind, or one listed above if measure.startswith("pass^") and "score" in (base["kind"], treat["kind"]): continue # pass^K is a gate measure: a score task has none to lack + if measure.startswith("first_useful") and base["values"].get(measure) is None \ + and treat["values"].get(measure) is None: + continue # a task neither arm declares a first useful result for: no pair, and no warning pair = _pair(task, base, treat, measure) if pair is not None: pairs.append(pair) @@ -521,8 +537,11 @@ def _pair(task: str, base: dict[str, Any], treat: dict[str, Any], measure: str) bv, tv = base["values"].get(measure), treat["values"].get(measure) if bv is None or tv is None: return None - return {"task": task, "baseline": bv, "treatment": tv, "delta": tv - bv, - "n_baseline": base["n"][measure], "n_treatment": treat["n"][measure]} + out = {"task": task, "baseline": bv, "treatment": tv, "delta": tv - bv, + "n_baseline": base["n"][measure], "n_treatment": treat["n"][measure]} + if measure.startswith("first_useful"): # a compared task has one kind: the row shows which + out["useful_kind"] = base["useful_kind"] + return out def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int = 0) -> dict[str, Any]: diff --git a/dokimi/dokimi/repos/compare_values.py b/dokimi/dokimi/repos/compare_values.py index fc3b329db..46b1f7786 100644 --- a/dokimi/dokimi/repos/compare_values.py +++ b/dokimi/dokimi/repos/compare_values.py @@ -16,7 +16,7 @@ from .process_counts import as_measure, process_lists from .report import reading_cells -from .useful import useful_task_values +from .useful import useful_kind, useful_task_counts, useful_task_values def finite_values(values: list[Any]) -> list[float]: @@ -68,6 +68,7 @@ def task_values( else 1.0 ) n = {measure: len(v) for measure, v in nums.items()} + n.update(useful_task_counts(recs)) # the cells behind each aggregate, not the one aggregate values = {measure: (sum(v) / len(v) if v else None) for measure, v in nums.items()} if pass_k and not score_task: gates = nums["gate"] @@ -83,6 +84,7 @@ def task_values( "kind": "score" if score_task else "gate", "grader_failed": grader_failed, "top": top, + "useful_kind": useful_kind(recs), "n": n, "values": values, } diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 3a424a5ec..643a456c3 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -69,9 +69,11 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: try: rec = json.loads(line) except json.JSONDecodeError: + out.dropped_lines += 1 out.errors.append(f"line {n}: not JSON") continue if not isinstance(rec, dict): + out.dropped_lines += 1 continue t = None stamp = _parse_time(rec.get("timestamp")) @@ -81,6 +83,7 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: try: _apply(out, rec, t, by_id, workdir) except (AttributeError, TypeError) as exc: # valid JSON of an unexpected shape + out.dropped_lines += 1 out.errors.append(f"line {n}: unexpected shape ({exc})") main = parse_trace(raw).main_calls() out.first_prompt_tokens = main[0]["prompt_tokens"] if main else None diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index d0d3ac374..b44ea6578 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -248,6 +248,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, added = meta.get("arm_patch") or {} uptake_of = (added.get("added") or [], added.get("mcp_servers") or []) events = None # the trace's events once it parsed; none: the uptake is unmeasured + useful_events = None # the same events when no line of the trace was dropped: a dropped line may hold the hit try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) @@ -263,6 +264,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, started = bool(transcript.events) or transcript.result_seen # else the agent never started if started: events = transcript.events + useful_events = None if transcript.dropped_lines else events cost = transcript.cost_usd agent.update( cost_usd=cost, charged_usd=cost if cost is not None else caps["budget_usd"], @@ -289,7 +291,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "harness": opts.profile.name, "harness_version": meta["harness_version"], "base_date": meta["base_date"], "execution": execution, "score_task": bool(task.score), "score_max": task.score["max"] if task.score else None, "agent": agent, - "process": process, "first_useful": first_useful(task.useful, events), "grade": grade} + "process": process, "first_useful": first_useful(task.useful, useful_events), "grade": grade} _add_arm_use(record, meta, task, skills, uptake_of, events) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 2411f8e72..972ed8a46 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -41,6 +41,7 @@ class Transcript: result_subtype: str | None = None plugins: list[str] = field(default_factory=list) # loaders the harness reported at start-up errors: list[str] = field(default_factory=list) + dropped_lines: int = 0 # trace lines that did not parse or were of no known shape: their events are missing init_model: str | None = None # the model the harness says it starts with (``system``/``init``) answered_by: set[str] = field(default_factory=set) # ``message.model`` of the main thread's replies diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 4d9e90839..5d707cc49 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -17,6 +17,8 @@ from typing import Any from .manifest import match_glob +from .process_counts import as_measure +from .process_metrics import _workspace_path from .useful_spec import Useful #: The ``repo compare`` measures of the first useful result, one value per task (see :func:`useful_task_values`). @@ -31,9 +33,13 @@ def hits(useful: Useful, event: Mapping[str, Any]) -> bool: text = str(event.get("text") or "") matched = bool(useful.command and useful.command.search(text)) return event.get("kind") == "command" and event.get("rc") == 0 and matched - path = str(event.get("path") or "") + path = _workspace_path( + str(event.get("path") or "") + ) # outside the workspace, or not a path: no target failed = event.get("rc") == 1 # an edit whose tool result is an error changed nothing - return event.get("kind") == "edit" and not failed and any(match_glob(g, path) for g in useful.globs) + if path is None or event.get("kind") != "edit" or failed: + return False + return any(match_glob(g, path) for g in useful.globs) def first_useful( @@ -66,7 +72,8 @@ def _declared(recs: Iterable[Mapping[str, Any]]) -> list[Mapping[str, Any]]: def _nums(values: Iterable[Any]) -> list[float]: - return [float(v) for v in values if isinstance(v, (int, float)) and not isinstance(v, bool)] + """The stored numbers that are measures: finite and within the report bound (a NaN or a 10**400 is no measure).""" + return [num for v in values if (num := as_measure(v)) is not None] def _median(values: list[float]) -> float | None: @@ -92,10 +99,22 @@ def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: } +def useful_kind(recs: Sequence[Mapping[str, Any]]) -> str | None: + """The kind the declared cells of ``recs`` define: ``command`` or ``edit``; ``mixed`` when they name both; None + when none declares one.""" + kinds = {u["kind"] for u in _declared(recs)} + if not kinds: + return None + return kinds.pop() if len(kinds) == 1 else "mixed" + + def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[float | None]]: """``repo compare``'s per-task values: the medians over the cells that observed it, and the share of the declared - cells that did (None if no cell declares it). The medians alone see only the observed cells.""" + cells that did (None if no cell declares it). The medians alone see only the observed cells. Cells that name two + kinds have no value: a command's timing and an edit's are not one measure.""" have = _declared(recs) + if useful_kind(recs) == "mixed": + return {measure: [None] for measure in USEFUL_MEASURES} observed = _nums(u.get("steps") for u in have) return { "first_useful_s": [_median(_nums(u.get("issued_s") for u in have))], @@ -104,6 +123,17 @@ def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[floa } +def useful_task_counts(recs: Sequence[Mapping[str, Any]]) -> dict[str, int]: + """The cells behind each :func:`useful_task_values` measure: the timed hits for seconds, the observed hits for + steps, and the declared cells for the rate (a median of one value is not one cell).""" + have = _declared(recs) + return { + "first_useful_s": len(_nums(u.get("issued_s") for u in have)), + "first_useful_steps": len(_nums(u.get("steps") for u in have)), + "first_useful_rate": len(have), + } + + def useful_counts(recs: Sequence[Mapping[str, Any]]) -> dict[str, int] | None: """One arm's cells that observed a first useful result, of those that declare one; None if none declare.""" have = _declared(recs) diff --git a/dokimi/dokimi/repos/useful_spec.py b/dokimi/dokimi/repos/useful_spec.py index 18f4662eb..7483935ae 100644 --- a/dokimi/dokimi/repos/useful_spec.py +++ b/dokimi/dokimi/repos/useful_spec.py @@ -42,7 +42,7 @@ def parse_useful(value: Any) -> Useful | None: # a deep nesting or a huge repeat count overflows the parser, not only re.error try: return Useful("command", command=re.compile(pattern)) - except (re.error, RecursionError, OverflowError) as exc: + except (re.error, RecursionError, OverflowError, ValueError) as exc: raise UsefulSpecError(f"useful command is not a valid regex ({exc})") from exc globs = [pattern] if isinstance(pattern, str) else pattern if ( diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index e06af6096..a6a816668 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -15,10 +15,13 @@ from dokimi.repos.report import format_report, summarize from dokimi.repos.useful import ( first_useful, + hits, useful_arms_line, useful_counts, + useful_kind, useful_line, useful_summary, + useful_task_counts, useful_task_values, ) from dokimi.repos.useful_spec import UsefulSpecError, parse_useful @@ -59,7 +62,10 @@ def test_the_two_kinds_parse_to_their_pattern_and_globs(): ({"command": ""}, "useful must be"), ({"command": "("}, "not a valid regex"), ({"command": "(" * 2000 + ")" * 2000}, "not a valid regex"), # overflows, not re.error - ({"command": "a{99999999999}"}, "not a valid regex"), # a repeat count too large for the parser + ( + {"command": "a{99999999999}"}, + "not a valid regex", + ), # a repeat count too large for the parser ({"edit": []}, "useful must be"), ({"edit": ["src/a.py", ""]}, "useful must be"), ({"edit": 3}, "useful must be"), @@ -242,13 +248,25 @@ def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): def test_a_failed_edit_changed_nothing_so_the_next_successful_one_is_the_first_useful_result(): failed = {"t": 4.0, "kind": "edit", "path": "src/foo.py"} - _apply_result({"tool_use_id": "u1", "is_error": True, "content": "String to replace not found in file."}, - {"u1": failed}) + _apply_result( + {"tool_use_id": "u1", "is_error": True, "content": "String to replace not found in file."}, + {"u1": failed}, + ) ok = {"t": 5.0, "kind": "edit", "path": "src/foo.py"} _apply_result({"tool_use_id": "u2", "is_error": False, "content": "edited"}, {"u2": ok}) u = parse_useful({"edit": "src/*.py"}) - assert first_useful(u, [failed]) == {"kind": "edit", "reason": "never", "issued_s": None, "steps": None} - assert first_useful(u, [failed, ok]) == {"kind": "edit", "reason": None, "issued_s": 5.0, "steps": 2} + assert first_useful(u, [failed]) == { + "kind": "edit", + "reason": "never", + "issued_s": None, + "steps": None, + } + assert first_useful(u, [failed, ok]) == { + "kind": "edit", + "reason": None, + "issued_s": 5.0, + "steps": 2, + } def _timed(batch: Path, task: str, arm: str, rep: int, useful: dict | None) -> None: @@ -448,3 +466,137 @@ def test_repo_run_refuses_a_malformed_definition_before_any_cell_runs(setup): # assert not list( (setup["out"] / "b1").glob("fix_add__*") ) # refused by validate_tasks: no cell was made + + +# -- review of #332: the counts behind a value, the kinds, the paths, the models ------------------------------- + + +def _set_useful(batch: Path, name: str, useful: dict) -> None: + path = batch / name / "result.json" + rec = json.loads(path.read_text()) + rec["first_useful"] = useful + path.write_text(json.dumps(rec)) + + +def test_the_counts_behind_each_useful_value_are_the_cells_not_the_one_aggregate(): + recs = [ + {"first_useful": _hit(10.0, 4)}, + {"first_useful": _hit(30.0, 8)}, + {"first_useful": {"kind": "command", "reason": "untimed", "issued_s": None, "steps": 6}}, + ] + assert useful_task_counts(recs) == { + "first_useful_s": 2, + "first_useful_steps": 3, + "first_useful_rate": 3, + } + assert useful_task_values(recs)["first_useful_rate"] == [1.0] + + +def test_a_task_whose_cells_name_two_kinds_has_no_useful_value(): + recs = [ + {"first_useful": _hit(10.0, 4)}, + {"first_useful": {"kind": "edit", "reason": None, "issued_s": 5.0, "steps": 2}}, + ] + assert useful_kind(recs) == "mixed" + assert useful_task_values(recs) == { + "first_useful_s": [None], + "first_useful_steps": [None], + "first_useful_rate": [None], + } + assert useful_kind([{"first_useful": _hit(1.0, 1)}]) == "command" + assert useful_kind([{}]) is None + + +def test_an_edit_path_is_matched_after_normalization_and_one_outside_the_workspace_never_hits(): + assert hits(parse_useful({"edit": "src/*.py"}), _edit("./src/x.py")) + assert hits(parse_useful({"edit": "src/*.py"}), _edit("src/../src/x.py")) + assert not hits(parse_useful({"edit": "**/*.py"}), _edit("/tmp/x.py")) + assert not hits(parse_useful({"edit": "**/*.py"}), _edit("../x.py")) + assert not hits(parse_useful({"edit": "**/*.py"}), _edit("src/../../x.py")) + + +@pytest.mark.parametrize("steps", [float("nan"), 10**400]) +def test_a_stored_non_finite_or_huge_number_is_no_measure(steps): + recs = [{"first_useful": {"kind": "edit", "reason": None, "issued_s": 1.0, "steps": steps}}] + assert useful_task_values(recs)["first_useful_steps"] == [None] + assert useful_task_values(recs)["first_useful_rate"] == [0.0] + assert useful_counts(recs) == {"observed": 0, "declared": 1} + + +def test_a_command_regex_that_overflows_the_parser_is_a_task_error_not_a_crash(): + with pytest.raises(UsefulSpecError): + parse_useful({"command": "a{" + "9" * 5000 + "}"}) + + +def test_a_trace_line_that_does_not_parse_marks_the_trace_incomplete(): + assert convert("claude-stream-json", "{not json\n", None).dropped_lines == 1 + assert convert("claude-stream-json", "[1]\n", None).dropped_lines == 1 # valid JSON, not an object + assert convert("claude-stream-json", "", None).dropped_lines == 0 + + +def test_the_compare_counts_are_the_cells_behind_each_value(tmp_path): + batch = tmp_path / "b" + _timed(batch, "t1", "guidance:none", 1, _hit(10.0, 4)) + _timed(batch, "t1", "guidance:none", 2, _hit(30.0, 8)) + _timed( + batch, + "t1", + "guidance:none", + 3, + {"kind": "command", "reason": "untimed", "issued_s": None, "steps": 6}, + ) + _timed(batch, "t1", "guidance:raw", 1, _hit(20.0, 5)) + s = compare([batch], "guidance:none") + assert _row(s, "first_useful_s")["tasks"][0]["n_baseline"] == 2 + rate = _row(s, "first_useful_rate")["tasks"][0] + assert (rate["n_baseline"], rate["n_treatment"]) == (3, 1) + + +def test_the_header_counts_only_the_models_compared(tmp_path): + batch = tmp_path / "b" + for arm, rep_model in ( + ("guidance:none", "m1"), + ("guidance:none", "m2"), + ("guidance:raw", "m1"), + ): + _cell(batch, "t1", arm, 1, model=rep_model) + _set_useful( + batch, + "t1__guidance-none__m1__r1", + {"kind": "command", "reason": "never", "issued_s": None, "steps": None}, + ) + _set_useful(batch, "t1__guidance-none__m2__r1", _hit(5.0, 3)) + _set_useful(batch, "t1__guidance-raw__m1__r1", _hit(9.0, 4)) + s = compare([batch], "guidance:none", models=["m1"]) + assert s["first_useful"]["guidance:none"] == {"observed": 0, "declared": 1} + + +def test_a_task_whose_arms_define_different_kinds_of_first_useful_result_is_unpaired(tmp_path): + batch = tmp_path / "b" + _timed(batch, "t1", "guidance:none", 1, _hit(10.0, 4)) + _timed( + batch, + "t1", + "guidance:raw", + 1, + {"kind": "edit", "reason": None, "issued_s": 40.0, "steps": 9}, + ) + s = compare([batch], "guidance:none") + assert not [r for r in s["comparisons"] if r["measure"].startswith("first_useful")] + assert any( + u["task"] == "t1" and "first useful kinds differ" in u["reason"] for u in s["unpaired"] + ) + + +def test_a_review_task_with_no_definition_in_either_arm_is_not_reported_unpaired(tmp_path): + batch = tmp_path / "b" + _timed(batch, "t1", "guidance:none", 1, _hit(10.0, 4)) + _timed(batch, "t1", "guidance:raw", 1, _hit(20.0, 6)) + _timed(batch, "t2", "guidance:none", 1, None) + _timed(batch, "t2", "guidance:raw", 1, None) + s = compare([batch], "guidance:none") + assert not [ + u + for u in s["unpaired"] + if u["task"] == "t2" and (u["measure"] or "").startswith("first_useful") + ] From 2909150e97370ed13d6383836d5521491cdd99f7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 23:47:51 -0700 Subject: [PATCH 08/26] fix(repo): useful hits refuse Windows paths; failed Codex patches and unknown trace blocks count - A drive-qualified or backslash-rooted edit path (C:/x, \\server\share) is outside the workspace; the POSIX check took it for relative and the glob matched (CodeFactor: the test used /tmp, a B108 finding). - A Codex file_change with status failed is an edit with rc 1, as Copilot's is; completed is rc 0. - A Claude stream record whose content holds a block of no known shape, or a content that is not a list, counts as a dropped line, so the run reports unobserved rather than never. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 7 ++++++- dokimi/dokimi/repos/events.py | 17 ++++++++++++---- dokimi/dokimi/repos/useful.py | 11 +++++++--- dokimi/tests/test_repo_useful.py | 34 ++++++++++++++++++++++++++++++- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 422a20e54..f795589e2 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -620,9 +620,14 @@ def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[ else: _codex_command_unstarted(out, item, script, workdir, cwd) elif itype == "file_change": + status = item.get("status") # a failed patch changed nothing + rc = {"completed": 0, "failed": 1}.get(status) if isinstance(status, str) else None for change in item.get("changes") or []: if isinstance(change, dict) and change.get("path"): - out.events.append({"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)}) + event = {"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)} + if rc is not None: + event["rc"] = rc + out.events.append(event) elif itype in ("mcp_tool_call", "web_search"): name = f"{item.get('server')}.{item.get('tool')}" if itype == "mcp_tool_call" else "web_search" out.events.append({"t": None, "kind": "tool", "name": name}) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 643a456c3..5972a2e04 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -131,7 +131,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, out.plugins = sorted(str(p.get("name")) for p in rec.get("plugins") or [] if isinstance(p, dict) and p.get("name")) elif kind == "assistant": - for block in _content(rec): + for block in _content(rec, out): if block.get("type") == "tool_use": event = _tool_event(block, t, workdir) if rec.get("parent_tool_use_id"): @@ -146,7 +146,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, if model and model != "" and rec.get("parent_tool_use_id") is None: out.answered_by.add(str(model)) elif kind == "user": - for block in _content(rec): + for block in _content(rec, out): if block.get("type") == "tool_result": _apply_result(block, by_id) elif kind == "result": @@ -195,6 +195,15 @@ def _result_text(content: Any) -> str | None: return None -def _content(rec: dict[str, Any]) -> list[dict[str, Any]]: +def _content(rec: dict[str, Any], out: Transcript) -> list[dict[str, Any]]: + """The object blocks of a message; a record with a block or a content of another shape counts as dropped.""" content = (rec.get("message") or {}).get("content") - return [b for b in content if isinstance(b, dict)] if isinstance(content, list) else [] + if content is None or isinstance(content, str): # a user's prompt is a plain string: no block, nothing lost + return [] + if not isinstance(content, list): + out.dropped_lines += 1 + return [] + blocks = [b for b in content if isinstance(b, dict)] + if len(blocks) != len(content): + out.dropped_lines += 1 + return blocks diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 5d707cc49..1b0462043 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -12,6 +12,7 @@ from __future__ import annotations import statistics +import re from collections import Counter from collections.abc import Iterable, Mapping, Sequence from typing import Any @@ -24,6 +25,10 @@ #: The ``repo compare`` measures of the first useful result, one value per task (see :func:`useful_task_values`). USEFUL_MEASURES = ("first_useful_s", "first_useful_steps", "first_useful_rate") +#: A drive-qualified (``C:/x``, ``c:x``) or backslash-rooted (``\\server\share``, ``\\x``) path: not a POSIX +#: path, so ``_workspace_path`` would take it for relative; it is outside the workspace. +_NON_POSIX_ROOT = re.compile(r"^(?:[A-Za-z]:|\\)") + def hits(useful: Useful, event: Mapping[str, Any]) -> bool: """Whether ``event`` satisfies ``useful``: a passing command matching the regex, or an edit matching a glob.""" @@ -33,9 +38,9 @@ def hits(useful: Useful, event: Mapping[str, Any]) -> bool: text = str(event.get("text") or "") matched = bool(useful.command and useful.command.search(text)) return event.get("kind") == "command" and event.get("rc") == 0 and matched - path = _workspace_path( - str(event.get("path") or "") - ) # outside the workspace, or not a path: no target + raw = str(event.get("path") or "") + # outside the workspace, or not a path: no target + path = None if _NON_POSIX_ROOT.match(raw) else _workspace_path(raw) failed = event.get("rc") == 1 # an edit whose tool result is an error changed nothing if path is None or event.get("kind") != "edit" or failed: return False diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index a6a816668..04481b1a0 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -510,7 +510,7 @@ def test_a_task_whose_cells_name_two_kinds_has_no_useful_value(): def test_an_edit_path_is_matched_after_normalization_and_one_outside_the_workspace_never_hits(): assert hits(parse_useful({"edit": "src/*.py"}), _edit("./src/x.py")) assert hits(parse_useful({"edit": "src/*.py"}), _edit("src/../src/x.py")) - assert not hits(parse_useful({"edit": "**/*.py"}), _edit("/tmp/x.py")) + assert not hits(parse_useful({"edit": "**/*.py"}), _edit("/etc/x.py")) assert not hits(parse_useful({"edit": "**/*.py"}), _edit("../x.py")) assert not hits(parse_useful({"edit": "**/*.py"}), _edit("src/../../x.py")) @@ -600,3 +600,35 @@ def test_a_review_task_with_no_definition_in_either_arm_is_not_reported_unpaired for u in s["unpaired"] if u["task"] == "t2" and (u["measure"] or "").startswith("first_useful") ] + + +@pytest.mark.parametrize("path", ["C:/tmp/x.py", "c:x.py", "\\\\srv\\share\\x.py", "\\x.py"]) +def test_a_windows_absolute_or_drive_path_outside_the_workspace_never_hits(path): + assert not hits(parse_useful({"edit": "**/*.py"}), _edit(path)) + + +def test_a_codex_file_change_that_failed_is_no_useful_edit(): + def change(status: str) -> list[dict]: + item = {"id": "1", "type": "file_change", "status": status, "changes": [{"path": "/ws/src/x.py", "kind": "update"}]} + return convert("codex-json", json.dumps({"type": "item.completed", "item": item}), "/ws").events + + spec = parse_useful({"edit": "src/*.py"}) + assert first_useful(spec, change("failed"))["reason"] == "never" + assert first_useful(spec, change("completed"))["reason"] == "untimed" # a hit: the edit landed (no clock here) + + +def test_a_claude_trace_with_a_malformed_content_block_is_unobserved_not_never(): + lines = [ + {"type": "assistant", "message": {"content": [42]}}, # a block of no known shape + {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "e", "name": "Edit", + "input": {"file_path": "/ws/src/x.py"}}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "e", "is_error": False, + "content": "ok"}]}}, + {"type": "result", "subtype": "success", "is_error": False, "num_turns": 2, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}, + ] + transcript = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws") + assert transcript.dropped_lines == 1 + # the runner reports the uptake as unobserved when a line was dropped, as it would for a trace it could not read + events = None if transcript.dropped_lines else transcript.events + assert first_useful(parse_useful({"edit": "src/*.py"}), events)["reason"] == "unobserved" From 4a816d9dc0dc3f16096ef72dedce38c7391c4634 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:50:04 -0700 Subject: [PATCH 09/26] fix(cli-traces): type the codex edit event so mypy accepts its rc Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index f795589e2..9505063bb 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -624,7 +624,7 @@ def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[ rc = {"completed": 0, "failed": 1}.get(status) if isinstance(status, str) else None for change in item.get("changes") or []: if isinstance(change, dict) and change.get("path"): - event = {"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)} + event: dict[str, Any] = {"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)} if rc is not None: event["rc"] = rc out.events.append(event) From 3104341a717147ce7ebb7f85d490e700e0e394d4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:48:47 -0800 Subject: [PATCH 10/26] fix(useful): a hit needs a successful result; tests for the hit rules, summary and dropped lines An edit counts only with a success tool result (rc 0) and a real target path: an empty or '.' path is no target. Denied and failed calls never hit, and the kind and rc are checked before the pattern runs. Claude content blocks without a string type are dropped lines. Tests pin the hit rules, the median, the counts, the treatment header, the runner's dropped-line guard and the Claude block typing. The fixture binary gains a tool_result, so the sample-batch golden execution id moves with its digest. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/events.py | 2 +- dokimi/dokimi/repos/useful.py | 21 +++-- dokimi/tests/repo_eval_fixture.py | 2 + dokimi/tests/test_batch_ledger_golden.py | 9 +- dokimi/tests/test_repo_run.py | 2 +- dokimi/tests/test_repo_useful.py | 110 ++++++++++++++++++++++- 6 files changed, 130 insertions(+), 16 deletions(-) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 5972a2e04..87d09a3d2 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -203,7 +203,7 @@ def _content(rec: dict[str, Any], out: Transcript) -> list[dict[str, Any]]: if not isinstance(content, list): out.dropped_lines += 1 return [] - blocks = [b for b in content if isinstance(b, dict)] + blocks = [b for b in content if isinstance(b, dict) and isinstance(b.get("type"), str)] # a block says its type if len(blocks) != len(content): out.dropped_lines += 1 return blocks diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 1b0462043..647f2a499 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -5,8 +5,9 @@ The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never -hits, nor does an edit whose tool result is an error (``rc`` 1). ``steps`` counts every event up to and including the hit, as ``process.tool_calls`` does. ``issued_s`` is the -hit's ``t``: the time the call was issued, since its result is not timed, so a test run's pass is not timed. +hits, nor does an edit whose tool result is not a success (``rc`` 0 is the only success). ``steps`` counts every event +up to and including the hit, as ``process.tool_calls`` does. ``issued_s`` is the hit's ``t``: the time the call was +issued, since its result is not timed, so a test run's pass is not timed. """ from __future__ import annotations @@ -34,15 +35,17 @@ def hits(useful: Useful, event: Mapping[str, Any]) -> bool: """Whether ``event`` satisfies ``useful``: a passing command matching the regex, or an edit matching a glob.""" if event.get("denied"): # a refused call never ran, or never changed the file return False + # only a call whose result says it succeeded counts: a call with no result, or a failed one, changed nothing + if event.get("rc") != 0: + return False if useful.kind == "command": - text = str(event.get("text") or "") - matched = bool(useful.command and useful.command.search(text)) - return event.get("kind") == "command" and event.get("rc") == 0 and matched + return event.get("kind") == "command" and bool(useful.command and useful.command.search(str(event.get("text") or ""))) raw = str(event.get("path") or "") - # outside the workspace, or not a path: no target - path = None if _NON_POSIX_ROOT.match(raw) else _workspace_path(raw) - failed = event.get("rc") == 1 # an edit whose tool result is an error changed nothing - if path is None or event.get("kind") != "edit" or failed: + # no path, outside the workspace, or the workspace root itself: no target + if event.get("kind") != "edit" or not raw or _NON_POSIX_ROOT.match(raw): + return False + path = _workspace_path(raw) + if path is None or path == ".": return False return any(match_glob(g, path) for g in useful.globs) diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 5f0e7038d..4925cfa20 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -107,6 +107,8 @@ lines.append({"type": "assistant", "message": {"model": answered, "content": [ {"type": "tool_use", "id": "t1", "name": "Edit", "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) + lines.append({"type": "user", "message": {"content": [ # the edit succeeded: a hit needs its result (rc 0) + {"type": "tool_result", "tool_use_id": "t1", "is_error": False, "content": "ok"}]}}) if os.environ.get("FAKE_PROCESS"): # a guidance read and one build command, for the process metrics lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": "t3", "name": "Read", diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index f54c64a81..c7a152124 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -31,24 +31,25 @@ } # The sample batch (the fake profile and binary of the fixture): every cell's execution record, by cell name. +# The id moves whenever the fixture binary changes (its digest is in the id), e.g. when it gains a tool_result. GOLDEN_SAMPLE_CELLS = { "fix_add__guidance-none__m-1__r1": { - "id": "d8d2e00d02f7", + "id": "ac034e664259", "tools": "all", "delegation": "native", }, "fix_add__guidance-none__m-1__r2": { - "id": "d8d2e00d02f7", + "id": "ac034e664259", "tools": "all", "delegation": "native", }, "fix_add__guidance-raw__m-1__r1": { - "id": "d8d2e00d02f7", + "id": "ac034e664259", "tools": "all", "delegation": "native", }, "fix_add__guidance-raw__m-1__r2": { - "id": "d8d2e00d02f7", + "id": "ac034e664259", "tools": "all", "delegation": "native", }, diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 827496e88..e41dbd2b0 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -57,7 +57,7 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert meta["agent_network"] == "host" assert meta["egress_proxy"] is False and len(meta["start_sha"]) == 40 events = [json.loads(line) for line in (rd / "events.jsonl").read_text().splitlines()] - assert events == [{"kind": "edit", "path": "src/calc.py", "t": None}] # relative to /eval/run/workspace + assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None}] # relative to /eval/run/workspace assert (rd / "answer.txt").read_text() == "done: src/calc.py" assert (rd / "config/settings.json").read_text() == '{"hook": "python3 /eval/support/no_background.py"}' result = json.loads((rd / "result.json").read_text()) diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 04481b1a0..3a89e9036 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -39,7 +39,8 @@ def _command(text: str, rc: int | None, t: float | None = None, **extra) -> dict def _edit(path: str, t: float | None = None, **extra) -> dict: - return {"t": t, "kind": "edit", "path": path, **extra} + """An edit whose result is a success (rc 0), unless ``extra`` says otherwise.""" + return {"t": t, "kind": "edit", "path": path, "rc": 0, **extra} # -- the definition ---------------------------------------------------------------------------------------------- @@ -203,6 +204,15 @@ def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): ] }, }, + { + "type": "user", + "timestamp": "2026-10-31T10:00:11.000Z", + "message": { + "content": [ + {"type": "tool_result", "tool_use_id": "b", "is_error": False, "content": "ok"} + ] + }, + }, { "type": "assistant", "timestamp": "2026-10-31T10:00:42.000Z", @@ -632,3 +642,101 @@ def test_a_claude_trace_with_a_malformed_content_block_is_unobserved_not_never() # the runner reports the uptake as unobserved when a line was dropped, as it would for a trace it could not read events = None if transcript.dropped_lines else transcript.events assert first_useful(parse_useful({"edit": "src/*.py"}), events)["reason"] == "unobserved" + + +# -- the hit rules, the summary and the dropped lines, pinned ----------------------------------------------------- + + +@pytest.mark.parametrize("useful, event", [ + ({"edit": "src/*.py"}, _edit("src/x.py", t=1.0, rc=None)), # no tool result: the call never ran (killed) + ({"edit": "**"}, _edit("", t=1.0)), # an Edit with no file_path: no target, not the workspace root + ({"edit": "**"}, _edit("./", t=1.0)), + ({"edit": "*"}, _edit(".", t=1.0)), +]) +def test_an_edit_with_no_result_or_no_target_is_never_the_first_useful_result(useful, event): + assert first_useful(parse_useful(useful), [event])["reason"] == "never" + + +def test_a_command_pattern_is_searched_anywhere_in_the_command_text(): + events = [_command("cd x && pytest -q", 0, t=1.0)] + assert first_useful(parse_useful({"command": "pytest"}), events) == { + "kind": "command", "reason": None, "issued_s": 1.0, "steps": 1} + + +def test_a_read_of_a_matching_path_is_no_edit(): + events = [{"t": 1.0, "kind": "read", "path": "src/x.py", "rc": 0}, _edit("docs/y.md", t=2.0)] + assert first_useful(parse_useful({"edit": "src/*.py"}), events)["reason"] == "never" + + +def test_the_median_is_the_middle_cell_not_the_mean(): + recs = [{"first_useful": _hit(1.0, 1)}, {"first_useful": _hit(2.0, 2)}, {"first_useful": _hit(10.0, 10)}] + summary = useful_summary(recs) + assert (summary["median_s"], summary["median_steps"]) == (2.0, 2.0) + + +def test_a_cell_that_names_no_definition_is_not_in_the_summary(): + recs = [{"first_useful": {"kind": None, "reason": "not_declared", "issued_s": None, "steps": None}}, + {"first_useful": _hit(5.0, 3)}] + assert (useful_summary(recs)["n"], useful_summary(recs)["observed"]) == (1, 1) + + +def test_an_untimed_hit_is_observed_with_steps_but_no_seconds(): + recs = [{"first_useful": {"kind": "command", "reason": "untimed", "issued_s": None, "steps": 3}}] + summary = useful_summary(recs) + assert (summary["observed"], summary["median_s"], summary["median_steps"]) == (1, None, 3.0) + + +def test_the_rate_is_over_the_declared_cells_and_its_count_is_the_declared_cells(): + recs = [{"first_useful": _hit(1.0, 1)}, + {"first_useful": {"kind": "command", "reason": "never", "issued_s": None, "steps": None}}, + {"first_useful": _hit(3.0, 2)}] + assert useful_task_values(recs)["first_useful_rate"] == [2 / 3] + assert useful_task_counts(recs)["first_useful_rate"] == 3 + + +@pytest.mark.parametrize("dropped, reason", [(0, None), (1, "unobserved")]) +def test_result_json_says_a_trace_with_a_dropped_line_is_unobserved_not_never(setup, monkeypatch, dropped, reason): # noqa: F811 + """The runner's own guard: a dropped line may hold the hit, so the kept events are not used for it.""" + from dokimi.repos import runner + + real_convert = runner.convert + + def convert_with_a_hit_and_a_drop(trace, raw, workspace): + transcript = real_convert(trace, raw, workspace) + transcript.dropped_lines = dropped + transcript.events.insert(0, {"t": 1.0, "kind": "command", "text": "bazel build //x", "rc": 0}) + return transcript + + monkeypatch.setattr(runner, "convert", convert_with_a_hit_and_a_drop) + task_json = setup["root"] / ".agent-eval/tasks/fix_add/task.json" + task = json.loads(task_json.read_text()) + task["useful"] = {"command": "bazel build"} + task_json.write_text(json.dumps(task)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", source=str(setup["root"])) + first = _only_cell(setup)["first_useful"] + assert first["reason"] == reason + assert first["steps"] == (None if reason else 1) + + +def test_the_treatment_header_counts_only_the_models_compared(tmp_path): + batch = tmp_path / "b" + for arm, model in (("guidance:none", "m1"), ("guidance:raw", "m1"), ("guidance:raw", "m2")): + _cell(batch, "t1", arm, 1, model=model) + _set_useful(batch, "t1__guidance-none__m1__r1", _hit(5.0, 3)) + _set_useful(batch, "t1__guidance-raw__m1__r1", + {"kind": "command", "reason": "never", "issued_s": None, "steps": None}) + _set_useful(batch, "t1__guidance-raw__m2__r1", _hit(9.0, 4)) + s = compare([batch], "guidance:none", models=["m1"]) + assert s["first_useful"]["guidance:raw"] == {"observed": 0, "declared": 1} + + +@pytest.mark.parametrize("content", [[{}], [{"type": 7}], [{"type": None}], [{"text": "no type"}]]) +def test_a_claude_block_without_a_string_type_is_a_dropped_line(content): + line = {"type": "assistant", "message": {"content": content}} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 1 + + +def test_claude_text_and_thinking_blocks_are_not_dropped(): + line = {"type": "assistant", "message": {"content": [ + {"type": "text", "text": "hi"}, {"type": "thinking", "thinking": "t", "signature": "s"}]}} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 0 From 07c495a4e62b8deb38fde833871114143225911d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:30:51 -0800 Subject: [PATCH 11/26] fix(useful): time out a catastrophic command pattern; drop malformed Codex file_change entries A `command` pattern is now searched with the `regex` module under a 2 s budget per event (USEFUL_REGEX_TIMEOUT). A search past the budget ends the run with reason `timeout`: no verdict, and the rest of the trace is not searched. A Codex file_change whose changes list holds a non-object or path-less entry is one dropped line, with no partial edit list. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 5 ++- dokimi/docs/repo-evaluation-contract.md | 3 +- dokimi/dokimi/repos/cli_traces.py | 15 ++++--- dokimi/dokimi/repos/report.py | 2 +- dokimi/dokimi/repos/useful.py | 14 ++++++- dokimi/dokimi/repos/useful_spec.py | 11 ++--- dokimi/tests/test_repo_cli_traces.py | 8 ++++ dokimi/tests/test_repo_useful.py | 42 +++++++++++++++++++ 8 files changed, 84 insertions(+), 16 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md index bbee7a43c..ed224e735 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -6,10 +6,13 @@ file. The first event that hits it (a command whose text matches and whose `rc` is 0, or an edit whose path matches) is recorded in `result.json` `first_useful` with its `issued_s` (the event's issue time) and `steps` (events up to and including it). A run with no hit says - why (`never`, `unobserved`, `untimed`); a task with no declaration is `not_declared`. + why (`never`, `unobserved`, `untimed`, or `timeout`: a `command` pattern, searched with the `regex` module, ran past + 2 s on one event, so the run has no verdict); a task with no declaration is `not_declared`. `repo report` shows a row's observed count, medians and reasons under the row, and `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`, and the share of declared cells that observed a hit as `first_useful_rate`; its header counts observed of declared cells per arm. A malformed `useful` is refused by `repo validate`, `repo tasks check` and `repo run`. Batches run before this change have no `first_useful` and no such rows. +- A Codex `file_change` record with a malformed entry (not an object, or no path) is a dropped line, as the other + malformed records are; before, its valid entries were not read either, and the drop was silent. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 7633af89e..1f4ef646a 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -461,7 +461,8 @@ is not timed. `first_useful` is `{"kind": "command"|"edit"|null, "reason": ..., "issued_s": number|null, "steps": int|null}`. `reason` is `null` when the hit was found and timed; otherwise it says why `issued_s` (and `steps`, with no hit) is null: `not_declared` (no definition), `unobserved` (the trace was not read: the -harness never started or it did not parse), `never` (no event hit) or `untimed` +harness never started or it did not parse), `never` (no event hit), `timeout` (a `command` pattern ran past +2 s on one event: no verdict, and the rest of the trace is not searched) or `untimed` (the trace format has no timestamps, so `steps` is set and `issued_s` is null). A cell of an older batch has no `first_useful`. Each cell that declares a definition is counted in `repo report`: the row says how many observed a hit, and the medians diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 9505063bb..97fc0ef08 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -622,12 +622,15 @@ def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[ elif itype == "file_change": status = item.get("status") # a failed patch changed nothing rc = {"completed": 0, "failed": 1}.get(status) if isinstance(status, str) else None - for change in item.get("changes") or []: - if isinstance(change, dict) and change.get("path"): - event: dict[str, Any] = {"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)} - if rc is not None: - event["rc"] = rc - out.events.append(event) + changes = item.get("changes") or [] + # an entry that is not a change makes the record malformed: it is dropped whole (counted), not read in part + if not isinstance(changes, list) or not all(isinstance(c, dict) and c.get("path") for c in changes): + raise ValueError("file_change with a malformed changes entry") + for change in changes: + event: dict[str, Any] = {"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)} + if rc is not None: + event["rc"] = rc + out.events.append(event) elif itype in ("mcp_tool_call", "web_search"): name = f"{item.get('server')}.{item.get('tool')}" if itype == "mcp_tool_call" else "web_search" out.events.append({"t": None, "kind": "tool", "name": name}) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index df1666a11..089a1a99a 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -21,7 +21,7 @@ cells with a trace to count from. - **first useful**: for a task that declares a ``useful`` definition, the cells that observed the first useful result (``result.json`` ``first_useful``, see :mod:`.useful`): how many, their median seconds and steps, and why the - others did not (``never``, ``unobserved``, ``untimed``). + others did not (``never``, ``unobserved``, ``untimed``, ``timeout``). - skill arms add ``skill_invoked`` (cells whose agent used the expected skill, or any installed one when the task names none) and, when the task has an ``expected_skill``, ``skill_selected_correct`` (cells that used it first; ``skill_expected`` diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 647f2a499..ca2148719 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -23,6 +23,10 @@ from .process_metrics import _workspace_path from .useful_spec import Useful +#: Seconds one ``command`` pattern may search one event's text. Past it the run gets reason ``timeout``: no verdict, +#: and the rest of its trace is not searched (a catastrophic pattern never stalls the host-side result). +USEFUL_REGEX_TIMEOUT = 2.0 + #: The ``repo compare`` measures of the first useful result, one value per task (see :func:`useful_task_values`). USEFUL_MEASURES = ("first_useful_s", "first_useful_steps", "first_useful_rate") @@ -39,7 +43,9 @@ def hits(useful: Useful, event: Mapping[str, Any]) -> bool: if event.get("rc") != 0: return False if useful.kind == "command": - return event.get("kind") == "command" and bool(useful.command and useful.command.search(str(event.get("text") or ""))) + if event.get("kind") != "command" or useful.command is None: + return False + return useful.command.search(str(event.get("text") or ""), timeout=USEFUL_REGEX_TIMEOUT) is not None raw = str(event.get("path") or "") # no path, outside the workspace, or the workspace root itself: no target if event.get("kind") != "edit" or not raw or _NON_POSIX_ROOT.match(raw): @@ -59,7 +65,11 @@ def first_useful( if events is None: return _record(useful.kind, "unobserved") for steps, event in enumerate(events, 1): - if hits(useful, event): + try: + found = hits(useful, event) + except TimeoutError: # the pattern ran past its budget on this event: no verdict + return _record(useful.kind, "timeout") + if found: t = event.get("t") if isinstance(t, (int, float)) and not isinstance(t, bool): return _record(useful.kind, None, issued_s=float(t), steps=steps) diff --git a/dokimi/dokimi/repos/useful_spec.py b/dokimi/dokimi/repos/useful_spec.py index 7483935ae..386953907 100644 --- a/dokimi/dokimi/repos/useful_spec.py +++ b/dokimi/dokimi/repos/useful_spec.py @@ -9,7 +9,7 @@ from __future__ import annotations -import re +import regex from dataclasses import dataclass from typing import Any @@ -27,7 +27,7 @@ class Useful: """One task's definition: ``command`` (a regex over command text) or ``edit`` (globs over edited paths).""" kind: str - command: re.Pattern[str] | None = None + command: regex.Pattern[str] | None = None globs: tuple[str, ...] = () @@ -39,10 +39,11 @@ def parse_useful(value: Any) -> Useful | None: raise UsefulSpecError(SHAPE) ((kind, pattern),) = value.items() if kind == "command" and isinstance(pattern, str) and pattern: - # a deep nesting or a huge repeat count overflows the parser, not only re.error + # a deep nesting or a huge repeat count overflows the parser, not only regex.error. The regex module + # searches with a timeout (useful.hits); re has none, and a pattern like (a+)+$ takes exponential time in it. try: - return Useful("command", command=re.compile(pattern)) - except (re.error, RecursionError, OverflowError, ValueError) as exc: + return Useful("command", command=regex.compile(pattern)) + except (regex.error, RecursionError, OverflowError, ValueError) as exc: raise UsefulSpecError(f"useful command is not a valid regex ({exc})") from exc globs = [pattern] if isinstance(pattern, str) else pattern if ( diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index a277b11e9..adb5b3b51 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -315,6 +315,14 @@ def test_codex_file_change_without_a_usable_path_records_no_edit(): assert [e for e in convert("codex-json", line, "/ws").events if e["kind"] == "edit"] == [], changes +def test_a_codex_file_change_with_a_malformed_entry_is_one_dropped_line_and_no_edits(): + for changes in (42, [42], [{"path": None, "kind": "update"}], [{"path": "/ws/src/a.c"}, 42]): + line = json.dumps({"type": "item.completed", "item": {"id": "1", "type": "file_change", "changes": changes}}) + transcript = convert("codex-json", line, "/ws") + assert transcript.dropped_lines == 1, changes + assert [e for e in transcript.events if e["kind"] == "edit"] == [], changes + + def test_codex_json_to_events(): raw = _jsonl( {"type": "thread.started", "thread_id": "x"}, diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 3a89e9036..2579332d9 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -740,3 +740,45 @@ def test_claude_text_and_thinking_blocks_are_not_dropped(): line = {"type": "assistant", "message": {"content": [ {"type": "text", "text": "hi"}, {"type": "thinking", "thinking": "t", "signature": "s"}]}} assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 0 + + +def test_a_catastrophic_pattern_on_a_long_non_matching_command_does_not_hang(): + # (a+)+$ is exponential in re, which hung this test run before the regex module. The command text is long and + # does not match, so the run is a plain "never" and returns at once. + import time + + events = [_command("a" * 40 + "!", 0, t=1.0), _command("pytest tests/test_x.py", 0, t=2.0)] + started = time.monotonic() + assert first_useful(parse_useful({"command": "(a+)+$"}), events) == { + "kind": "command", "reason": "never", "issued_s": None, "steps": None} + assert time.monotonic() - started < 5 + + +def test_the_command_search_is_bounded_by_the_budget_constant(monkeypatch): + # (a|a)+$ is exponential in regex too, so only a search that really gets a timeout ends in "timeout", not "never". + from dokimi.repos import useful as useful_module + + monkeypatch.setattr(useful_module, "USEFUL_REGEX_TIMEOUT", 0.05) + events = [_command("a" * 40 + "!", 0, t=1.0)] + assert first_useful(parse_useful({"command": "(a|a)+$"}), events) == { + "kind": "command", "reason": "timeout", "issued_s": None, "steps": None} + + +def test_a_pattern_past_its_search_budget_is_a_timeout_reason_and_the_trace_is_not_searched(monkeypatch): + # The engine's own timing is not what is tested here: a search that raises TimeoutError maps to the reason + # "timeout", and the events after it are not searched. + from dokimi.repos import useful as useful_module + + searched = [] + + def slow(useful, event): + searched.append(event["text"]) + if len(searched) == 1: + raise TimeoutError + return False + + monkeypatch.setattr(useful_module, "hits", slow) + events = [_command("a" * 40 + "!", 0, t=1.0), _command("pytest tests/test_x.py", 0, t=2.0)] + assert first_useful(parse_useful({"command": "(a|a)+$"}), events) == { + "kind": "command", "reason": "timeout", "issued_s": None, "steps": None} + assert len(searched) == 1 From 5f404c826c81d1584ce13c20e0ebe9cd9b7a3771 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 20:07:15 +0000 Subject: [PATCH 12/26] Treat a falsy non-list Codex file_change changes as malformed Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/cli_traces.py | 2 +- dokimi/tests/test_repo_cli_traces.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 97fc0ef08..ea5de276b 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -622,7 +622,7 @@ def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[ elif itype == "file_change": status = item.get("status") # a failed patch changed nothing rc = {"completed": 0, "failed": 1}.get(status) if isinstance(status, str) else None - changes = item.get("changes") or [] + changes = [] if item.get("changes") is None else item["changes"] # an entry that is not a change makes the record malformed: it is dropped whole (counted), not read in part if not isinstance(changes, list) or not all(isinstance(c, dict) and c.get("path") for c in changes): raise ValueError("file_change with a malformed changes entry") diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index adb5b3b51..8dc85857e 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -316,7 +316,8 @@ def test_codex_file_change_without_a_usable_path_records_no_edit(): def test_a_codex_file_change_with_a_malformed_entry_is_one_dropped_line_and_no_edits(): - for changes in (42, [42], [{"path": None, "kind": "update"}], [{"path": "/ws/src/a.c"}, 42]): + for changes in (42, [42], [{"path": None, "kind": "update"}], [{"path": "/ws/src/a.c"}, 42], + {}, "", 0, False): line = json.dumps({"type": "item.completed", "item": {"id": "1", "type": "file_change", "changes": changes}}) transcript = convert("codex-json", line, "/ws") assert transcript.dropped_lines == 1, changes From bce5fc679243a55ef055534dd3fbc7eaf1c632e3 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 20:07:51 +0000 Subject: [PATCH 13/26] Fix the contract's unobserved, edit hit and command regex text Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 1f4ef646a..df8781601 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -449,19 +449,19 @@ hits the task's `useful` key (§3). The definition depends on the grader: | Grader | Declare | Hit | |---|---|---| -| test: the grade runs the target tests | `{"command": ""}` | the first `command` event whose text matches the regex (`re.search`) and whose `rc` is `0`. `rc` 0 says that command passed, not that every target test did, so the regex should name the target tests | +| test: the grade runs the target tests | `{"command": ""}` | the first `command` event whose text matches the regex (`regex.search`) and whose `rc` is `0`. `rc` 0 says that command passed, not that every target test did, so the regex should name the target tests | | edit or diff: the grade reads a target file | `{"edit": ""}` or a list | the first `edit` event whose path matches a glob (`match_glob`, as the guidance globs) | | review (`dokimi:review_*`) | none | none, so `not_declared`: the answer is the product, so no event is the useful result. A declared key is measured all the same | | any other repository grader | either form, or none | none declared, `not_declared` | -A refused event (`denied`) never hits, nor does an `edit` whose tool result is an error (`rc` 1: the file was not changed). `steps` counts every event up to and +A refused event (`denied`) never hits, nor does an `edit` without a success result (`rc` 0): an error result (`rc` 1) means the file was not changed, and no result at all (a killed run) proves nothing. `steps` counts every event up to and including the hit, the parent's and subagents' (as `process.tool_calls` does). `issued_s` is the hit's `t`: the time the call was issued, since a call's result is not timed. `first_useful` is `{"kind": "command"|"edit"|null, "reason": ..., "issued_s": number|null, "steps": int|null}`. `reason` is `null` when the hit was found and timed; otherwise it says why `issued_s` (and `steps`, with no hit) is null: `not_declared` (no definition), `unobserved` (the trace was not read: the -harness never started or it did not parse), `never` (no event hit), `timeout` (a `command` pattern ran past +harness never started, it did not parse, or a line of the trace was dropped), `never` (no event hit), `timeout` (a `command` pattern ran past 2 s on one event: no verdict, and the rest of the trace is not searched) or `untimed` (the trace format has no timestamps, so `steps` is set and `issued_s` is null). A cell of an older batch has no `first_useful`. Each cell that declares a definition From 24ab1d17b007aff9c3d503b0fc959f974a74ec1b Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 20:48:48 +0000 Subject: [PATCH 14/26] Count an assistant record without content as a dropped line Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/events.py | 2 +- dokimi/tests/test_repo_useful.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 87d09a3d2..38c897e5e 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -198,7 +198,7 @@ def _result_text(content: Any) -> str | None: def _content(rec: dict[str, Any], out: Transcript) -> list[dict[str, Any]]: """The object blocks of a message; a record with a block or a content of another shape counts as dropped.""" content = (rec.get("message") or {}).get("content") - if content is None or isinstance(content, str): # a user's prompt is a plain string: no block, nothing lost + if isinstance(content, str) or (content is None and rec.get("type") == "user"): # a user's prompt is a plain string: no block, nothing lost return [] if not isinstance(content, list): out.dropped_lines += 1 diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 2579332d9..dd2b05b1c 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -736,6 +736,18 @@ def test_a_claude_block_without_a_string_type_is_a_dropped_line(content): assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 1 +@pytest.mark.parametrize("message", [{"content": None}, {}]) +def test_an_assistant_record_without_content_is_a_dropped_line(message): + line = {"type": "assistant", "message": message} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 1 + assert convert("claude-stream-json", json.dumps({"type": "assistant"}), "/ws").dropped_lines == 1 + + +def test_a_user_record_without_content_is_not_dropped(): + line = {"type": "user", "message": {"content": None}} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 0 + + def test_claude_text_and_thinking_blocks_are_not_dropped(): line = {"type": "assistant", "message": {"content": [ {"type": "text", "text": "hi"}, {"type": "thinking", "thinking": "t", "signature": "s"}]}} From 8170f8fa46cafd77d37dc00deca8d00c177480bd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:08:10 -0800 Subject: [PATCH 15/26] ci(static): install regex in the static job's tools lock The static job installs only requirements-ci.lock.txt, but its registry-drift-check and guidance self-scan import dokimi.repos, which imports regex. Both failed with ModuleNotFoundError. Add regex to requirements-ci.txt and its hashed block to the tools lock, the same 2026.9.29 pin as requirements.lock.txt. Changelog: a Codex file_change with a malformed or non-list changes value drops the whole record (no partial read); an edit hit needs rc 0 and no denial. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 7 +- dokimi/requirements-ci.lock.txt | 134 ++++++++++++++++++ dokimi/requirements-ci.txt | 3 + 3 files changed, 141 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md index ed224e735..3e9695fd5 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -3,7 +3,7 @@ - **Time to first useful result, per repo-eval run** — a `task.json` can declare `"useful": {"command": ""}` for a task graded by its target tests, or `"useful": {"edit": ""}` (or a list of globs) for one graded by an edit to a target - file. The first event that hits it (a command whose text matches and whose `rc` is 0, or an + file. The first non-denied event with `rc` 0 that hits it (a command whose text matches, or an edit whose path matches) is recorded in `result.json` `first_useful` with its `issued_s` (the event's issue time) and `steps` (events up to and including it). A run with no hit says why (`never`, `unobserved`, `untimed`, or `timeout`: a `command` pattern, searched with the `regex` module, ran past @@ -14,5 +14,6 @@ observed of declared cells per arm. A malformed `useful` is refused by `repo validate`, `repo tasks check` and `repo run`. Batches run before this change have no `first_useful` and no such rows. -- A Codex `file_change` record with a malformed entry (not an object, or no path) is a dropped line, as the other - malformed records are; before, its valid entries were not read either, and the drop was silent. +- A Codex `file_change` record whose `changes` has a malformed entry (not an object, or no path), or is a + non-list value other than `null`, is one dropped line, as the other malformed records are. The whole record is dropped: its valid + entries are not read either, so no edit from it counts. diff --git a/dokimi/requirements-ci.lock.txt b/dokimi/requirements-ci.lock.txt index 9bf3e7a9e..807286bc4 100644 --- a/dokimi/requirements-ci.lock.txt +++ b/dokimi/requirements-ci.lock.txt @@ -1288,6 +1288,140 @@ referencing==0.37.0 \ # jsonschema # jsonschema-specifications # types-jsonschema +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via + # -c requirements.lock.txt + # -r requirements-ci.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed diff --git a/dokimi/requirements-ci.txt b/dokimi/requirements-ci.txt index 6a7b8926a..9a261d057 100644 --- a/dokimi/requirements-ci.txt +++ b/dokimi/requirements-ci.txt @@ -15,6 +15,9 @@ pip-audit==2.10.1 mypy==2.4.0 PyYAML jsonschema +# regex: the static job's registry-drift-check and guidance self-scan import dokimi.repos, +# which imports the `regex` module (the reference check's engine); the job installs no other lock. +regex types-PyYAML types-jsonschema deptry==0.25.1 From 65af232b2efc1e0562a5a3020260087285ecc45c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 16:57:18 -0800 Subject: [PATCH 16/26] fix(repo): dropped tool calls, untimed wording, timings of observed hits only A tool_use block with no id, name or object input is one dropped line, not an empty edit: a run whose edit call is unreadable reads unobserved, not never. A cell that observed no hit has no seconds in the summary, the task values or the counts. Docstrings name first_useful; the changelog says untimed is observed. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 5 +-- dokimi/dokimi/repos/events.py | 10 ++++++ dokimi/dokimi/repos/useful.py | 18 ++++++---- dokimi/tests/test_repo_reach.py | 1 - dokimi/tests/test_repo_useful.py | 36 +++++++++++++++++++ 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md index 3e9695fd5..9e252e804 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -6,8 +6,9 @@ file. The first non-denied event with `rc` 0 that hits it (a command whose text matches, or an edit whose path matches) is recorded in `result.json` `first_useful` with its `issued_s` (the event's issue time) and `steps` (events up to and including it). A run with no hit says - why (`never`, `unobserved`, `untimed`, or `timeout`: a `command` pattern, searched with the `regex` module, ran past - 2 s on one event, so the run has no verdict); a task with no declaration is `not_declared`. + why (`never`, `unobserved`, or `timeout`: a `command` pattern, searched with the `regex` module, ran past + 2 s on one event, so the run has no verdict); a task with no declaration is `not_declared`. A hit in a trace + without timestamps is `untimed`: it keeps its `steps` and counts as observed, with no seconds. `repo report` shows a row's observed count, medians and reasons under the row, and `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`, and the share of declared cells that observed a hit as `first_useful_rate`; its header counts diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 38c897e5e..c8fddd465 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -133,6 +133,9 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, elif kind == "assistant": for block in _content(rec, out): if block.get("type") == "tool_use": + if not _well_formed_tool_use(block): # an unreadable call is a dropped line, not an empty one + out.dropped_lines += 1 + continue event = _tool_event(block, t, workdir) if rec.get("parent_tool_use_id"): event["agent"] = str(rec["parent_tool_use_id"]) @@ -169,6 +172,13 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, by_id[denial["tool_use_id"]]["denied"] = True +def _well_formed_tool_use(block: dict[str, Any]) -> bool: + """A tool call names itself, has an id its result can find, and has an object of input.""" + return (isinstance(block.get("id"), str) and bool(block["id"]) + and isinstance(block.get("name"), str) and bool(block["name"]) + and isinstance(block.get("input"), dict)) + + def _tool_event(block: dict[str, Any], t: float | None, workdir: str | None) -> dict[str, Any]: name = str(block.get("name") or "") raw_args = block.get("input") diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index ca2148719..d86e53ebf 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""The first useful result of a run, for ``result.json`` ``useful`` (contract §4.1). +"""The first useful result of a run, for ``result.json`` ``first_useful`` (contract §4.1). The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never @@ -59,7 +59,7 @@ def hits(useful: Useful, event: Mapping[str, Any]) -> bool: def first_useful( useful: Useful | None, events: Sequence[Mapping[str, Any]] | None ) -> dict[str, Any]: - """``result.json`` ``useful`` of one run; ``events`` is None when its trace could not be read.""" + """``result.json`` ``first_useful`` of one run; ``events`` is None when its trace could not be read.""" if useful is None: return _record(None, "not_declared") if events is None: @@ -85,10 +85,16 @@ def _record( def _declared(recs: Iterable[Mapping[str, Any]]) -> list[Mapping[str, Any]]: - """The ``useful`` objects of the cells that record a definition: an older batch has none.""" + """The ``first_useful`` objects of the cells that record a definition: an older batch has none.""" return [u for r in recs if isinstance(u := r.get("first_useful"), dict) and u.get("kind")] +def _observed(have: Sequence[Mapping[str, Any]]) -> list[Mapping[str, Any]]: + """The cells with a measurable ``steps``: the ones that observed a hit. Their ``issued_s`` is the only timing + that goes with a hit; a cell that observed none has no seconds, even when a stored ``issued_s`` is a number.""" + return [u for u in have if as_measure(u.get("steps")) is not None] + + def _nums(values: Iterable[Any]) -> list[float]: """The stored numbers that are measures: finite and within the report bound (a NaN or a 10**400 is no measure).""" return [num for v in values if (num := as_measure(v)) is not None] @@ -105,7 +111,7 @@ def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: have = _declared(recs) if not have: return None - seconds = _nums(u.get("issued_s") for u in have) + seconds = _nums(u.get("issued_s") for u in _observed(have)) steps = _nums(u.get("steps") for u in have) return { "kind": have[0]["kind"], @@ -135,7 +141,7 @@ def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[floa return {measure: [None] for measure in USEFUL_MEASURES} observed = _nums(u.get("steps") for u in have) return { - "first_useful_s": [_median(_nums(u.get("issued_s") for u in have))], + "first_useful_s": [_median(_nums(u.get("issued_s") for u in _observed(have)))], "first_useful_steps": [_median(observed)], "first_useful_rate": [len(observed) / len(have) if have else None], } @@ -146,7 +152,7 @@ def useful_task_counts(recs: Sequence[Mapping[str, Any]]) -> dict[str, int]: steps, and the declared cells for the rate (a median of one value is not one cell).""" have = _declared(recs) return { - "first_useful_s": len(_nums(u.get("issued_s") for u in have)), + "first_useful_s": len(_nums(u.get("issued_s") for u in _observed(have))), "first_useful_steps": len(_nums(u.get("steps") for u in have)), "first_useful_rate": len(have), } diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py index cd3b6a7b9..7bde9c9b3 100644 --- a/dokimi/tests/test_repo_reach.py +++ b/dokimi/tests/test_repo_reach.py @@ -482,7 +482,6 @@ def msg(mid, usage, parent=None): @pytest.mark.parametrize("line", [ {"type": "assistant", "message": "hi"}, {"type": "result", "result": "x", "usage": "n/a"}, - {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": [1], "name": "Read"}]}}, ]) def test_a_wrong_shaped_trace_line_is_an_error_not_a_crash(line): t = convert("claude-stream-json", json.dumps(line) + '\n{"type": "result", "result": "ok"}\n') diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index dd2b05b1c..0d08ccb49 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -531,6 +531,10 @@ def test_a_stored_non_finite_or_huge_number_is_no_measure(steps): assert useful_task_values(recs)["first_useful_steps"] == [None] assert useful_task_values(recs)["first_useful_rate"] == [0.0] assert useful_counts(recs) == {"observed": 0, "declared": 1} + # a cell that observed no hit has no timing either: its issued_s is not a timed hit + assert useful_task_values(recs)["first_useful_s"] == [None] + assert useful_task_counts(recs)["first_useful_s"] == 0 + assert useful_summary(recs)["median_s"] is None def test_a_command_regex_that_overflows_the_parser_is_a_task_error_not_a_crash(): @@ -748,6 +752,38 @@ def test_a_user_record_without_content_is_not_dropped(): assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 0 +def test_a_user_prompt_given_as_a_plain_string_is_not_dropped(): + line = {"type": "user", "message": {"content": "hi"}} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 0 + + +@pytest.mark.parametrize("block", [ + {"type": "tool_use", "id": "e", "name": "Edit", "input": 42}, # the input is not an object + {"type": "tool_use", "id": "e", "name": "Edit"}, # no input at all + {"type": "tool_use", "name": "Edit", "input": {"file_path": "/ws/src/x.py"}}, # no id: no result can find it + {"type": "tool_use", "id": "e", "input": {}}, # no name + {"type": "tool_use", "id": [1], "name": "Read", "input": {}}, # an id that is not a string +]) +def test_a_tool_use_block_that_is_not_a_call_is_a_dropped_line(block): + line = {"type": "assistant", "message": {"content": [block]}} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 1 + + +def test_an_edit_call_with_a_non_object_input_is_unobserved_not_never(): + # the call's input is not an object, so its path is unreadable: the run must not say the edit never happened + lines = [ + {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "e", "name": "Edit", "input": 42}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "e", "is_error": False, + "content": "ok"}]}}, + {"type": "result", "subtype": "success", "is_error": False, "num_turns": 2, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}, + ] + transcript = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws") + assert transcript.dropped_lines == 1 + events = None if transcript.dropped_lines else transcript.events + assert first_useful(parse_useful({"edit": "**"}), events)["reason"] == "unobserved" + + def test_claude_text_and_thinking_blocks_are_not_dropped(): line = {"type": "assistant", "message": {"content": [ {"type": "text", "text": "hi"}, {"type": "thinking", "thinking": "t", "signature": "s"}]}} From 0f296574d0471c363a62c0cd242dc5331fcff8a8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 18:55:26 -0800 Subject: [PATCH 17/26] refactor(repo): split events._apply by record kind, drop its baseline entry _apply was at complexity 18 against a baseline of 17, so the growth ratchet failed on both Tests jobs. Each record kind now has its own helper; the behaviour is unchanged. The function is under the cap now, so its baseline entry is removed (check_growth --update). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/events.py | 91 ++++++++++++++++------------- dokimi/scripts/growth_baseline.json | 1 - 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index c8fddd465..817910911 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -125,51 +125,64 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, by_id: dict[str, dict[str, Any]], workdir: str | None) -> None: kind = rec.get("type") if kind == "system" and rec.get("subtype") == "init": - out.harness_version = rec.get("claude_code_version") - out.init_model = str(rec["model"]) if rec.get("model") else None - # Claude Code loads AGENTS.md through a built-in plugin (cc-plugin-agents-md) on some versions - out.plugins = sorted(str(p.get("name")) for p in rec.get("plugins") or [] - if isinstance(p, dict) and p.get("name")) + _apply_init(out, rec) elif kind == "assistant": - for block in _content(rec, out): - if block.get("type") == "tool_use": - if not _well_formed_tool_use(block): # an unreadable call is a dropped line, not an empty one - out.dropped_lines += 1 - continue - event = _tool_event(block, t, workdir) - if rec.get("parent_tool_use_id"): - event["agent"] = str(rec["parent_tool_use_id"]) - if block.get("name") in _AGENT_TOOLS: - out.subagent_calls = (out.subagent_calls or 0) + 1 - out.events.append(event) - if block.get("id"): - by_id[block["id"]] = event - model = (rec.get("message") or {}).get("model") - # A subagent's model is its definition's, not the run's; "" marks a reply Claude Code wrote itself. - if model and model != "" and rec.get("parent_tool_use_id") is None: - out.answered_by.add(str(model)) + _apply_assistant(out, rec, t, by_id, workdir) elif kind == "user": for block in _content(rec, out): if block.get("type") == "tool_result": _apply_result(block, by_id) elif kind == "result": - out.answer = str(rec.get("result") or "").strip() - out.cost_usd = rec.get("total_cost_usd") - out.result_seen = True - _set_num_turns(out, rec.get("num_turns")) - out.is_error = bool(rec.get("is_error")) - out.result_subtype = rec.get("subtype") - out.subagent_calls = out.subagent_calls or 0 - if not isinstance(rec.get("usage") or {}, dict): - raise TypeError("usage is not an object") - from dokimi.harnesses.claude_json import token_counts - counts = token_counts(rec) - if counts is not None: - out.usage_scope, (out.input_tokens, out.output_tokens, out.cache_read_tokens, - out.cache_write_tokens) = counts - for denial in rec.get("permission_denials") or []: - if isinstance(denial, dict) and denial.get("tool_use_id") in by_id: - by_id[denial["tool_use_id"]]["denied"] = True + _apply_final(out, rec, by_id) + + +def _apply_init(out: Transcript, rec: dict[str, Any]) -> None: + out.harness_version = rec.get("claude_code_version") + out.init_model = str(rec["model"]) if rec.get("model") else None + # Claude Code loads AGENTS.md through a built-in plugin (cc-plugin-agents-md) on some versions + out.plugins = sorted(str(p.get("name")) for p in rec.get("plugins") or [] + if isinstance(p, dict) and p.get("name")) + + +def _apply_assistant(out: Transcript, rec: dict[str, Any], t: float | None, + by_id: dict[str, dict[str, Any]], workdir: str | None) -> None: + for block in _content(rec, out): + if block.get("type") == "tool_use": + if not _well_formed_tool_use(block): # an unreadable call is a dropped line, not an empty one + out.dropped_lines += 1 + continue + event = _tool_event(block, t, workdir) + if rec.get("parent_tool_use_id"): + event["agent"] = str(rec["parent_tool_use_id"]) + if block.get("name") in _AGENT_TOOLS: + out.subagent_calls = (out.subagent_calls or 0) + 1 + out.events.append(event) + if block.get("id"): + by_id[block["id"]] = event + model = (rec.get("message") or {}).get("model") + # A subagent's model is its definition's, not the run's; "" marks a reply Claude Code wrote itself. + if model and model != "" and rec.get("parent_tool_use_id") is None: + out.answered_by.add(str(model)) + + +def _apply_final(out: Transcript, rec: dict[str, Any], by_id: dict[str, dict[str, Any]]) -> None: + out.answer = str(rec.get("result") or "").strip() + out.cost_usd = rec.get("total_cost_usd") + out.result_seen = True + _set_num_turns(out, rec.get("num_turns")) + out.is_error = bool(rec.get("is_error")) + out.result_subtype = rec.get("subtype") + out.subagent_calls = out.subagent_calls or 0 + if not isinstance(rec.get("usage") or {}, dict): + raise TypeError("usage is not an object") + from dokimi.harnesses.claude_json import token_counts + counts = token_counts(rec) + if counts is not None: + out.usage_scope, (out.input_tokens, out.output_tokens, out.cache_read_tokens, + out.cache_write_tokens) = counts + for denial in rec.get("permission_denials") or []: + if isinstance(denial, dict) and denial.get("tool_use_id") in by_id: + by_id[denial["tool_use_id"]]["denied"] = True def _well_formed_tool_use(block: dict[str, Any]) -> bool: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index b0b75b387..2044445e2 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -76,7 +76,6 @@ "dokimi/repos/environment/risks.py::_runs_repository_code": {"complexity": 12, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/environment/risks.py::_segments": {"complexity": 29, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/environment/risks.py::detect_capabilities": {"complexity": 16, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/repos/events.py::_apply": {"complexity": 17, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/harness_profile.py::parse_profile": {"complexity": 11}, "dokimi/repos/l0/adapters.py::_build_names": {"complexity": 15, "rationale": "grew on main in #115 (L0 rules for static_check parity) before the ratchet landed; split it in its own PR"}, "dokimi/repos/l0/adapters.py::_make": {"complexity": 11}, From 48df4a5bcf37b7d931fb5eea064dc2cf72afa53f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 18:55:29 -0800 Subject: [PATCH 18/26] fix(repo): a Codex file_change path must be a non-empty string A truthy non-string path (42, true, a list) was accepted and stringified into an edit with rc 0. Such an entry is now one dropped line with no edits, as the other malformed entries are. Red cases added to test_a_codex_file_change_with_a_malformed_entry_is_one_dropped_line_and_no_edits. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 3 ++- dokimi/tests/test_repo_cli_traces.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index ea5de276b..b5b8f2f4c 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -624,7 +624,8 @@ def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[ rc = {"completed": 0, "failed": 1}.get(status) if isinstance(status, str) else None changes = [] if item.get("changes") is None else item["changes"] # an entry that is not a change makes the record malformed: it is dropped whole (counted), not read in part - if not isinstance(changes, list) or not all(isinstance(c, dict) and c.get("path") for c in changes): + if not isinstance(changes, list) or not all( + isinstance(c, dict) and isinstance(c.get("path"), str) and c["path"] for c in changes): raise ValueError("file_change with a malformed changes entry") for change in changes: event: dict[str, Any] = {"t": None, "kind": "edit", "path": relative_path(str(change["path"]), workdir)} diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 8dc85857e..02272e18b 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -317,7 +317,7 @@ def test_codex_file_change_without_a_usable_path_records_no_edit(): def test_a_codex_file_change_with_a_malformed_entry_is_one_dropped_line_and_no_edits(): for changes in (42, [42], [{"path": None, "kind": "update"}], [{"path": "/ws/src/a.c"}, 42], - {}, "", 0, False): + {}, "", 0, False, [{"path": 42}], [{"path": True}], [{"path": ["/ws/a.c"]}]): line = json.dumps({"type": "item.completed", "item": {"id": "1", "type": "file_change", "changes": changes}}) transcript = convert("codex-json", line, "/ws") assert transcript.dropped_lines == 1, changes From 5481ef01c1d9f98900df3dabb7d1b352d5f91f7f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 20:31:35 -0800 Subject: [PATCH 19/26] fix(repo): malformed Copilot tool arguments and assistant string content count as dropped lines Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 6 ++++-- dokimi/dokimi/repos/events.py | 2 +- dokimi/tests/test_repo_useful.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index b5b8f2f4c..9b2cbd37b 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -745,13 +745,15 @@ def copilot_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: def _copilot_tool(data: dict[str, Any], t: float | None, workdir: str | None, cwd: str = "") -> list[dict[str, Any]]: name = str(data.get("toolName") or "") args = data.get("arguments") + if name in _COPILOT_SHELL | _COPILOT_READ | _COPILOT_EDIT and not isinstance(args, dict): + raise TypeError(f"{name} arguments are not an object") # counted as a dropped line by the caller if name in _COPILOT_SHELL: - command = str((args or {}).get("command", "")) if isinstance(args, dict) else "" + command = str(args.get("command", "")) # only bash is a POSIX script; a PowerShell command is not parsed for the files it reads scripts = shell_reads(command, cwd) if name == "bash" else [] reads = [{"t": t, "kind": "read", "path": relative_path(p, workdir)} for p in scripts] return [*reads, {"t": t, "kind": "command", "text": command, "rc": None}] - if name in _COPILOT_READ | _COPILOT_EDIT and isinstance(args, dict): + if name in _COPILOT_READ | _COPILOT_EDIT: path = str(args.get("path") or args.get("file_path") or "") if not path: # a call without a path names no file return [] diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 817910911..b1900165c 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -221,7 +221,7 @@ def _result_text(content: Any) -> str | None: def _content(rec: dict[str, Any], out: Transcript) -> list[dict[str, Any]]: """The object blocks of a message; a record with a block or a content of another shape counts as dropped.""" content = (rec.get("message") or {}).get("content") - if isinstance(content, str) or (content is None and rec.get("type") == "user"): # a user's prompt is a plain string: no block, nothing lost + if rec.get("type") == "user" and (isinstance(content, str) or content is None): # a user's prompt is a plain string: nothing lost return [] if not isinstance(content, list): out.dropped_lines += 1 diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 0d08ccb49..8f8e9b7a0 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -830,3 +830,33 @@ def slow(useful, event): assert first_useful(parse_useful({"command": "(a|a)+$"}), events) == { "kind": "command", "reason": "timeout", "issued_s": None, "steps": None} assert len(searched) == 1 + + +def test_an_assistant_record_with_a_plain_string_content_is_a_dropped_line(): + line = {"type": "assistant", "message": {"content": "unexpected shape"}} + assert convert("claude-stream-json", json.dumps(line), "/ws").dropped_lines == 1 + + +def _copilot_lines(name, arguments): + return "\n".join(json.dumps(x) for x in [ + {"type": "tool.execution_start", "data": {"toolCallId": "e", "toolName": name, "arguments": arguments}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "e", "success": True}}, + {"type": "result", "exitCode": 0}, + ]) + + +@pytest.mark.parametrize("arguments", [42, None]) +@pytest.mark.parametrize("name", ["edit", "create", "view", "bash", "powershell"]) +def test_a_copilot_call_with_non_object_arguments_is_a_dropped_line(name, arguments): + assert convert("copilot-json", _copilot_lines(name, arguments), "/ws").dropped_lines == 1 + + +def test_a_copilot_apply_patch_given_as_a_string_is_not_dropped(): + patch = "*** Update File: /ws/a.py\n" + assert convert("copilot-json", _copilot_lines("apply_patch", patch), "/ws").dropped_lines == 0 + + +def test_a_copilot_edit_with_non_object_arguments_is_unobserved_not_never(): + transcript = convert("copilot-json", _copilot_lines("edit", 42), "/ws") + events = None if transcript.dropped_lines else transcript.events + assert first_useful(parse_useful({"edit": "**"}), events)["reason"] == "unobserved" From 47c0880965443698c2db4659078bc2a73c670c2c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 21:38:49 -0800 Subject: [PATCH 20/26] fix(repo): narrow Copilot tool arguments for mypy Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 9b2cbd37b..2091cc341 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -747,14 +747,15 @@ def _copilot_tool(data: dict[str, Any], t: float | None, workdir: str | None, cw args = data.get("arguments") if name in _COPILOT_SHELL | _COPILOT_READ | _COPILOT_EDIT and not isinstance(args, dict): raise TypeError(f"{name} arguments are not an object") # counted as a dropped line by the caller + obj: dict[str, Any] = args if isinstance(args, dict) else {} if name in _COPILOT_SHELL: - command = str(args.get("command", "")) + command = str(obj.get("command", "")) # only bash is a POSIX script; a PowerShell command is not parsed for the files it reads scripts = shell_reads(command, cwd) if name == "bash" else [] reads = [{"t": t, "kind": "read", "path": relative_path(p, workdir)} for p in scripts] return [*reads, {"t": t, "kind": "command", "text": command, "rc": None}] if name in _COPILOT_READ | _COPILOT_EDIT: - path = str(args.get("path") or args.get("file_path") or "") + path = str(obj.get("path") or obj.get("file_path") or "") if not path: # a call without a path names no file return [] return [{"t": t, "kind": "read" if name in _COPILOT_READ else "edit", "path": relative_path(path, workdir)}] From c6ff5295842ab2fc49fa8b704b37ff3d67aaf730 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 11:02:16 +0000 Subject: [PATCH 21/26] test(repo): pin empty tool_use id and name and empty Codex path Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_cli_traces.py | 3 ++- dokimi/tests/test_repo_useful.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 02272e18b..8073d54a5 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -317,7 +317,8 @@ def test_codex_file_change_without_a_usable_path_records_no_edit(): def test_a_codex_file_change_with_a_malformed_entry_is_one_dropped_line_and_no_edits(): for changes in (42, [42], [{"path": None, "kind": "update"}], [{"path": "/ws/src/a.c"}, 42], - {}, "", 0, False, [{"path": 42}], [{"path": True}], [{"path": ["/ws/a.c"]}]): + {}, "", 0, False, [{"path": 42}], [{"path": True}], [{"path": ["/ws/a.c"]}], + [{"path": ""}]): line = json.dumps({"type": "item.completed", "item": {"id": "1", "type": "file_change", "changes": changes}}) transcript = convert("codex-json", line, "/ws") assert transcript.dropped_lines == 1, changes diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 8f8e9b7a0..84e4faeb3 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -763,6 +763,8 @@ def test_a_user_prompt_given_as_a_plain_string_is_not_dropped(): {"type": "tool_use", "name": "Edit", "input": {"file_path": "/ws/src/x.py"}}, # no id: no result can find it {"type": "tool_use", "id": "e", "input": {}}, # no name {"type": "tool_use", "id": [1], "name": "Read", "input": {}}, # an id that is not a string + {"type": "tool_use", "id": "", "name": "Edit", "input": {}}, # an empty id + {"type": "tool_use", "id": "e", "name": "", "input": {}}, # an empty name ]) def test_a_tool_use_block_that_is_not_a_call_is_a_dropped_line(block): line = {"type": "assistant", "message": {"content": [block]}} From 4a131a7d9e12fa080777639e06f2f17f339ed7bc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 11:02:34 +0000 Subject: [PATCH 22/26] docs(repo): say what drops a line of the trace Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index df4de36be..3fef7fac8 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -489,7 +489,7 @@ is not timed. `first_useful` is `{"kind": "command"|"edit"|null, "reason": ..., "issued_s": number|null, "steps": int|null}`. `reason` is `null` when the hit was found and timed; otherwise it says why `issued_s` (and `steps`, with no hit) is null: `not_declared` (no definition), `unobserved` (the trace was not read: the -harness never started, it did not parse, or a line of the trace was dropped), `never` (no event hit), `timeout` (a `command` pattern ran past +harness never started, it did not parse, or a line of the trace was dropped: an unreadable record or block, such as a tool call without an id, a name or an object of input, a content block without a string type, a Codex file change without a path, or Copilot shell, read or edit arguments that are not an object), `never` (no event hit), `timeout` (a `command` pattern ran past 2 s on one event: no verdict, and the rest of the trace is not searched) or `untimed` (the trace format has no timestamps, so `steps` is set and `issued_s` is null). A cell of an older batch has no `first_useful`. Each cell that declares a definition From 81484ad4eb40339aaac873f1a5236f9169d061c2 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 11:42:24 +0000 Subject: [PATCH 23/26] test(repo): cover write and str_replace_editor in Copilot arguments test Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_useful.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 84e4faeb3..3ab2de2b1 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -848,7 +848,9 @@ def _copilot_lines(name, arguments): @pytest.mark.parametrize("arguments", [42, None]) -@pytest.mark.parametrize("name", ["edit", "create", "view", "bash", "powershell"]) +@pytest.mark.parametrize( + "name", ["edit", "create", "write", "str_replace_editor", "view", "bash", "powershell"] +) def test_a_copilot_call_with_non_object_arguments_is_a_dropped_line(name, arguments): assert convert("copilot-json", _copilot_lines(name, arguments), "/ws").dropped_lines == 1 From 2b0e1730d33f74849a6af30161436269e2af02fc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 08:09:50 -0800 Subject: [PATCH 24/26] fix(repo): a tool result with no usable id is a dropped line; mixed first-useful kinds have no median A Claude tool_result that names no call (missing, null, numeric or empty tool_use_id) was ignored silently, so the trace looked complete and the first useful result read "never" instead of "unobserved". It now counts as a dropped line, as other malformed blocks do. repo report named a mixed row by its first cell's kind and pooled the command and edit medians. A mixed row now has kind "mixed" and no median, as repo compare already shows. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/events.py | 8 +++++++ dokimi/dokimi/repos/useful.py | 8 ++++--- dokimi/tests/test_repo_useful.py | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index b1900165c..da3dae0c1 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -131,6 +131,9 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, elif kind == "user": for block in _content(rec, out): if block.get("type") == "tool_result": + if not _well_formed_tool_result(block): # a result that names no call is lost: a dropped line + out.dropped_lines += 1 + continue _apply_result(block, by_id) elif kind == "result": _apply_final(out, rec, by_id) @@ -192,6 +195,11 @@ def _well_formed_tool_use(block: dict[str, Any]) -> bool: and isinstance(block.get("input"), dict)) +def _well_formed_tool_result(block: dict[str, Any]) -> bool: + """A tool result names the call it answers by a non-empty string id.""" + return isinstance(block.get("tool_use_id"), str) and bool(block["tool_use_id"]) + + def _tool_event(block: dict[str, Any], t: float | None, workdir: str | None) -> dict[str, Any]: name = str(block.get("name") or "") raw_args = block.get("input") diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index d86e53ebf..8d125110e 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -113,12 +113,14 @@ def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: return None seconds = _nums(u.get("issued_s") for u in _observed(have)) steps = _nums(u.get("steps") for u in have) + kind = useful_kind(recs) # "mixed" when the cells name both kinds: no median of those is one measure + mixed = kind == "mixed" return { - "kind": have[0]["kind"], + "kind": kind, "n": len(have), "observed": len(steps), - "median_s": _median(seconds), - "median_steps": _median(steps), + "median_s": None if mixed else _median(seconds), + "median_steps": None if mixed else _median(steps), "reasons": dict(Counter(u["reason"] for u in have if u.get("reason"))), } diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 3ab2de2b1..7c2be5b97 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -314,6 +314,25 @@ def test_useful_summary_is_the_observed_cells_medians_and_why_the_rest_are_not() assert useful_line(None, 4) == [] +def test_useful_summary_of_mixed_kinds_names_no_kind_and_no_median(): + # a command hit and an edit hit in one row: their seconds and steps are no one measure + recs = [ + {"first_useful": _hit(10.0, 4)}, + {"first_useful": {"kind": "edit", "reason": None, "issued_s": 40.0, "steps": 9}}, + ] + assert useful_summary(recs) == { + "kind": "mixed", + "n": 2, + "observed": 2, + "median_s": None, + "median_steps": None, + "reasons": {}, + } + assert useful_line(useful_summary(recs), 4) == [ + " " * 8 + "first useful (mixed): 2 of 2 observed" + ] + + def test_a_task_value_is_the_median_of_the_cells_that_observed_it(): recs = [ {"first_useful": _hit(10.0, 4)}, @@ -648,6 +667,25 @@ def test_a_claude_trace_with_a_malformed_content_block_is_unobserved_not_never() assert first_useful(parse_useful({"edit": "src/*.py"}), events)["reason"] == "unobserved" +@pytest.mark.parametrize("result_id", [None, 7, "", "missing"]) +def test_a_claude_tool_result_with_no_usable_id_is_a_dropped_line_not_a_silent_miss(result_id): + # the Edit's completion names no call, so it cannot be matched: the trace is unreadable, not "never" + result = {"type": "tool_result", "is_error": False, "content": "ok"} + if result_id != "missing": + result["tool_use_id"] = result_id + lines = [ + {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "e", "name": "Edit", + "input": {"file_path": "/ws/src/x.py"}}]}}, + {"type": "user", "message": {"content": [result]}}, + {"type": "result", "subtype": "success", "is_error": False, "num_turns": 2, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}, + ] + transcript = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws") + assert transcript.dropped_lines == 1 + events = None if transcript.dropped_lines else transcript.events + assert first_useful(parse_useful({"edit": "src/*.py"}), events)["reason"] == "unobserved" + + # -- the hit rules, the summary and the dropped lines, pinned ----------------------------------------------------- From dea1e88d285f67005d556f1d33a37eea2a699d13 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 11:17:24 -0800 Subject: [PATCH 25/26] fix(repo): an unobserved or timeout cell is no miss in first_useful_rate A cell whose trace was not read, or whose pattern timed out, has no verdict, but the rate divided by every declared cell and read it as a miss. The rate and its count now use the cells with a verdict, and the compare header shows the rest as unmeasured. The contract doc says that a shell-made edit is not an edit event. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_repo-first-useful.md | 5 +- dokimi/docs/repo-evaluation-contract.md | 8 +-- dokimi/dokimi/repos/compare.py | 5 +- dokimi/dokimi/repos/useful.py | 43 ++++++++++++---- dokimi/tests/test_repo_useful.py | 49 ++++++++++++++++--- 5 files changed, 86 insertions(+), 24 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md index 9e252e804..734f10046 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-first-useful.md @@ -11,8 +11,9 @@ without timestamps is `untimed`: it keeps its `steps` and counts as observed, with no seconds. `repo report` shows a row's observed count, medians and reasons under the row, and `repo compare` pairs the per-task medians as `first_useful_s` and `first_useful_steps`, and - the share of declared cells that observed a hit as `first_useful_rate`; its header counts - observed of declared cells per arm. + the share of the cells with a verdict that observed a hit as `first_useful_rate` (`unobserved` and `timeout` + cells are unmeasured: neither a hit nor a miss); its header counts observed of measured cells per arm and + how many are unmeasured. A malformed `useful` is refused by `repo validate`, `repo tasks check` and `repo run`. Batches run before this change have no `first_useful` and no such rows. - A Codex `file_change` record whose `changes` has a malformed entry (not an object, or no path), or is a diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 3fef7fac8..69692f175 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -495,8 +495,10 @@ harness never started, it did not parse, or a line of the trace was dropped: an A cell of an older batch has no `first_useful`. Each cell that declares a definition is counted in `repo report`: the row says how many observed a hit, and the medians cover only those. `repo compare` pairs `first_useful_rate` per task (the share of its -declared cells that observed a hit, so a task where most runs never hit it is not read -through its medians alone) and its header counts observed of declared cells per arm. A +declared cells with a verdict that observed a hit, so a task where most runs never hit it is not read +through its medians alone; a cell `unobserved` or `timeout` has no verdict, so it is neither a hit nor +a miss and is left out of the share) and its header counts observed of measured cells per arm, with +the unmeasured ones. An edit made by a shell command (`sed -i`) is not an `edit` event, so it is not seen. A compared task has one kind, but the mean delta pools the tasks of both kinds: a per-task row shows which. @@ -1513,7 +1515,7 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --metric re task is the unit. Per treatment arm, model and measure (gate share, score, cost, turns, `first_prompt_tokens`, `first_useful_s` and `first_useful_steps` (the median over a task's cells that observed the first useful result, §4.1), `first_useful_rate` (the share of a task's - declared cells that observed one), each `--metric` from `grade.json`, and each `process` count a + declared cells with a verdict that observed one), each `--metric` from `grade.json`, and each `process` count a compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for gate and score, a t interval for the descriptive measures, `-` for a single diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index e6c714e5e..ac581458f 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -17,8 +17,9 @@ compared only when both arms report it for some shared task: a batch run before it was recorded has none; - **first_useful_s**, **first_useful_steps**: the median seconds and steps to the first useful result (``first_useful`` in ``result.json``, :mod:`.useful`) over the cells that observed it; compared like ``first_prompt_tokens``; -- **first_useful_rate**: the share of the cells that declare a first useful result and observed one, per task; the - header counts observed of declared cells per arm; +- **first_useful_rate**: the share of the cells that declare a first useful result, and have a verdict (not + `unobserved` or `timeout`), that observed one, per task; the header counts observed of measured cells per arm and + the unmeasured ones; - **metric:**: the mean of ``grade.json`` ``metrics.`` over the cells that report it (``true`` is 1), for ``--metric``; - **pass^k**: for ``--pass-k K``, the chance that K cells drawn from the diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 8d125110e..d2eabcbbc 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -30,6 +30,9 @@ #: The ``repo compare`` measures of the first useful result, one value per task (see :func:`useful_task_values`). USEFUL_MEASURES = ("first_useful_s", "first_useful_steps", "first_useful_rate") +#: The reasons that give a cell no verdict on the first useful result (its trace was not read, or its pattern timed out). +_NO_VERDICT = ("unobserved", "timeout") + #: A drive-qualified (``C:/x``, ``c:x``) or backslash-rooted (``\\server\share``, ``\\x``) path: not a POSIX #: path, so ``_workspace_path`` would take it for relative; it is outside the workspace. _NON_POSIX_ROOT = re.compile(r"^(?:[A-Za-z]:|\\)") @@ -95,6 +98,11 @@ def _observed(have: Sequence[Mapping[str, Any]]) -> list[Mapping[str, Any]]: return [u for u in have if as_measure(u.get("steps")) is not None] +def _measured(have: Sequence[Mapping[str, Any]]) -> list[Mapping[str, Any]]: + """The cells with a verdict: an ``unobserved`` or ``timeout`` cell has none, so it is no miss either.""" + return [u for u in have if u.get("reason") not in _NO_VERDICT] + + def _nums(values: Iterable[Any]) -> list[float]: """The stored numbers that are measures: finite and within the report bound (a NaN or a 10**400 is no measure).""" return [num for v in values if (num := as_measure(v)) is not None] @@ -135,42 +143,57 @@ def useful_kind(recs: Sequence[Mapping[str, Any]]) -> str | None: def useful_task_values(recs: Sequence[Mapping[str, Any]]) -> dict[str, list[float | None]]: - """``repo compare``'s per-task values: the medians over the cells that observed it, and the share of the declared - cells that did (None if no cell declares it). The medians alone see only the observed cells. Cells that name two - kinds have no value: a command's timing and an edit's are not one measure.""" + """``repo compare``'s per-task values: the medians over the cells that observed it, and the share of the cells + with a verdict that observed it (None if none has one). A cell with no verdict (``unobserved``, ``timeout``) is + neither a hit nor a miss. The medians alone see only the observed cells. Cells that name two kinds have no value: + a command's timing and an edit's are not one measure.""" have = _declared(recs) if useful_kind(recs) == "mixed": return {measure: [None] for measure in USEFUL_MEASURES} + measured = _measured(have) observed = _nums(u.get("steps") for u in have) return { "first_useful_s": [_median(_nums(u.get("issued_s") for u in _observed(have)))], "first_useful_steps": [_median(observed)], - "first_useful_rate": [len(observed) / len(have) if have else None], + "first_useful_rate": [ + len(_nums(u.get("steps") for u in measured)) / len(measured) if measured else None + ], } def useful_task_counts(recs: Sequence[Mapping[str, Any]]) -> dict[str, int]: """The cells behind each :func:`useful_task_values` measure: the timed hits for seconds, the observed hits for - steps, and the declared cells for the rate (a median of one value is not one cell).""" + steps, and the cells with a verdict for the rate (a median of one value is not one cell).""" have = _declared(recs) return { "first_useful_s": len(_nums(u.get("issued_s") for u in _observed(have))), "first_useful_steps": len(_nums(u.get("steps") for u in have)), - "first_useful_rate": len(have), + "first_useful_rate": len(_measured(have)), } def useful_counts(recs: Sequence[Mapping[str, Any]]) -> dict[str, int] | None: - """One arm's cells that observed a first useful result, of those that declare one; None if none declare.""" + """One arm's cells that observed a first useful result, of those with a verdict, and the cells without one; None + if none declares a definition.""" have = _declared(recs) if not have: return None - return {"observed": len(_nums(u.get("steps") for u in have)), "declared": len(have)} + measured = _measured(have) + return { + "observed": len(_nums(u.get("steps") for u in measured)), + "measured": len(measured), + "unmeasured": len(have) - len(measured), + } def useful_arms_line(counts: Mapping[str, Mapping[str, int] | None]) -> str | None: - """The compare header's line of observed-of-declared cells per arm; None when no arm declares one.""" - shown = [f"{arm} {c['observed']} of {c['declared']}" for arm, c in counts.items() if c] + """The compare header's line of observed-of-measured cells per arm, with the unmeasured ones; None when no arm + declares one.""" + shown = [ + f"{arm} {c['observed']} of {c['measured']}" + + (f" ({c['unmeasured']} unmeasured)" if c["unmeasured"] else "") + for arm, c in counts.items() if c + ] return f"first useful observed: {', '.join(shown)}" if shown else None diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 7c2be5b97..c0b8ff585 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -350,10 +350,10 @@ def test_a_task_value_is_the_median_of_the_cells_that_observed_it(): "first_useful_steps": [None], "first_useful_rate": [None], } - assert useful_counts(recs) == {"observed": 3, "declared": 4} + assert useful_counts(recs) == {"observed": 3, "measured": 4, "unmeasured": 0} assert useful_counts([{}]) is None assert ( - useful_arms_line({"a": {"observed": 1, "declared": 2}, "b": None}) + useful_arms_line({"a": {"observed": 1, "measured": 2, "unmeasured": 0}, "b": None}) == "first useful observed: a 1 of 2" ) assert useful_arms_line({"a": None}) is None @@ -436,8 +436,8 @@ def test_the_rate_says_how_many_declared_cells_observed_a_hit_where_the_medians_ assert by_task["t1"]["baseline"] < 1.0 # the medians alone would read this arm as no slower assert _row(s, "first_useful_s")["delta"]["mean"] == 1.0 # medians 10 and 11 assert s["first_useful"] == { - "guidance:none": {"observed": 1, "declared": 2}, - "guidance:raw": {"observed": 2, "declared": 2}, + "guidance:none": {"observed": 1, "measured": 2, "unmeasured": 0}, + "guidance:raw": {"observed": 2, "measured": 2, "unmeasured": 0}, } assert "first useful observed: guidance:none 1 of 2, guidance:raw 2 of 2" in format_compare(s) @@ -549,7 +549,7 @@ def test_a_stored_non_finite_or_huge_number_is_no_measure(steps): recs = [{"first_useful": {"kind": "edit", "reason": None, "issued_s": 1.0, "steps": steps}}] assert useful_task_values(recs)["first_useful_steps"] == [None] assert useful_task_values(recs)["first_useful_rate"] == [0.0] - assert useful_counts(recs) == {"observed": 0, "declared": 1} + assert useful_counts(recs) == {"observed": 0, "measured": 1, "unmeasured": 0} # a cell that observed no hit has no timing either: its issued_s is not a timed hit assert useful_task_values(recs)["first_useful_s"] == [None] assert useful_task_counts(recs)["first_useful_s"] == 0 @@ -601,7 +601,7 @@ def test_the_header_counts_only_the_models_compared(tmp_path): _set_useful(batch, "t1__guidance-none__m2__r1", _hit(5.0, 3)) _set_useful(batch, "t1__guidance-raw__m1__r1", _hit(9.0, 4)) s = compare([batch], "guidance:none", models=["m1"]) - assert s["first_useful"]["guidance:none"] == {"observed": 0, "declared": 1} + assert s["first_useful"]["guidance:none"] == {"observed": 0, "measured": 1, "unmeasured": 0} def test_a_task_whose_arms_define_different_kinds_of_first_useful_result_is_unpaired(tmp_path): @@ -736,6 +736,41 @@ def test_the_rate_is_over_the_declared_cells_and_its_count_is_the_declared_cells assert useful_task_counts(recs)["first_useful_rate"] == 3 +def test_a_cell_with_no_verdict_is_left_out_of_the_rate_not_counted_as_a_miss(): + recs = [ + {"first_useful": _hit(1.0, 1)}, + {"first_useful": {"kind": "command", "reason": "unobserved", "issued_s": None, "steps": None}}, + {"first_useful": {"kind": "command", "reason": "timeout", "issued_s": None, "steps": None}}, + ] + assert useful_task_values(recs)["first_useful_rate"] == [1.0] + assert useful_task_counts(recs)["first_useful_rate"] == 1 + assert useful_counts(recs) == {"observed": 1, "measured": 1, "unmeasured": 2} + + +def test_a_task_whose_cells_have_no_verdict_has_no_rate(): + recs = [{"first_useful": {"kind": "edit", "reason": "unobserved", "issued_s": None, "steps": None}}] + assert useful_task_values(recs)["first_useful_rate"] == [None] + assert useful_task_counts(recs)["first_useful_rate"] == 0 + assert useful_counts(recs) == {"observed": 0, "measured": 0, "unmeasured": 1} + + +def test_compare_does_not_read_unobserved_and_timeout_cells_as_a_lower_rate(tmp_path): + batch = tmp_path / "b" + unobserved = {"kind": "command", "reason": "unobserved", "issued_s": None, "steps": None} + timeout = {"kind": "command", "reason": "timeout", "issued_s": None, "steps": None} + _timed(batch, "t1", "guidance:none", 1, _hit(10.0, 4)) + _timed(batch, "t1", "guidance:none", 2, unobserved) + _timed(batch, "t1", "guidance:none", 3, timeout) + for rep, (s, n) in enumerate([(12.0, 5), (11.0, 4), (13.0, 6)], 1): + _timed(batch, "t1", "guidance:raw", rep, _hit(s, n)) + s = compare([batch], "guidance:none") + rate = _row(s, "first_useful_rate")["tasks"][0] + assert (rate["baseline"], rate["treatment"]) == (1.0, 1.0) + assert _row(s, "first_useful_rate")["delta"]["mean"] == 0.0 + assert s["first_useful"]["guidance:none"] == {"observed": 1, "measured": 1, "unmeasured": 2} + assert "first useful observed: guidance:none 1 of 1 (2 unmeasured), guidance:raw 3 of 3" in format_compare(s) + + @pytest.mark.parametrize("dropped, reason", [(0, None), (1, "unobserved")]) def test_result_json_says_a_trace_with_a_dropped_line_is_unobserved_not_never(setup, monkeypatch, dropped, reason): # noqa: F811 """The runner's own guard: a dropped line may hold the hit, so the kept events are not used for it.""" @@ -769,7 +804,7 @@ def test_the_treatment_header_counts_only_the_models_compared(tmp_path): {"kind": "command", "reason": "never", "issued_s": None, "steps": None}) _set_useful(batch, "t1__guidance-raw__m2__r1", _hit(9.0, 4)) s = compare([batch], "guidance:none", models=["m1"]) - assert s["first_useful"]["guidance:raw"] == {"observed": 0, "declared": 1} + assert s["first_useful"]["guidance:raw"] == {"observed": 0, "measured": 1, "unmeasured": 0} @pytest.mark.parametrize("content", [[{}], [{"type": 7}], [{"type": None}], [{"text": "no type"}]]) From 49add24e39eed896c2285e9d2fdcfb9c61b7a7ae Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 16:21:53 -0800 Subject: [PATCH 26/26] fix(repo): say why a trace part is dropped; report mixed first-useful kinds per kind Every dropped part of a trace now adds a reason to the cell's trace errors, so an unobserved first useful result says why. A tool_result that names no recorded call is a dropped line. A compare pair with differing first-useful kinds is unpaired under first_useful. A report row whose cells name both kinds gets one summary per kind. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 2 ++ dokimi/dokimi/repos/compare.py | 5 +-- dokimi/dokimi/repos/events.py | 24 +++++++++---- dokimi/dokimi/repos/report.py | 3 +- dokimi/dokimi/repos/useful.py | 36 ++++++++++++------- dokimi/tests/test_repo_useful.py | 58 +++++++++++++++++++++++++------ 6 files changed, 96 insertions(+), 32 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 2091cc341..0a44bf5cc 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -536,6 +536,7 @@ def codex_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: continue if not isinstance(rec, dict): out.dropped_lines += 1 + out.errors.append(f"line {n}: not a JSON object") continue try: _codex_apply(out, rec, messages, workdir, by_id, cwd, event_reads) @@ -729,6 +730,7 @@ def copilot_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: continue if not isinstance(rec, dict): out.dropped_lines += 1 + out.errors.append(f"line {n}: not a JSON object") continue t = run.offset(rec) try: diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index ac581458f..0e8c89af1 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -392,7 +392,7 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment")) bk, tk = base_vals[(model, task)]["useful_kind"], treat_vals[(model, task)]["useful_kind"] if (bk and tk and bk != tk) or "mixed" in (bk, tk): # a command's timing and an edit's are not one measure - unpaired.append(_unpaired(treatment, model, task, None, + unpaired.append(_unpaired(treatment, model, task, "first_useful", f"first useful kinds differ: {bk} in baseline, {tk} in treatment")) else: useful_ok.append(task) @@ -491,7 +491,8 @@ def at_top(value: float, top: float | None) -> bool: def _unpaired(treatment: str, model: str, task: str, measure: str | None, reason: str) -> dict[str, Any]: - """A task left out of the comparison: of every measure (``measure`` None) or of one.""" + """A task left out of the comparison: of every measure (``measure`` None), of the first useful ones + (``first_useful``: the three ``first_useful_*`` measures) or of one.""" return {"treatment": treatment, "model": model, "task": task, "measure": measure, "reason": reason} diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index da3dae0c1..97404317f 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -74,6 +74,7 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: continue if not isinstance(rec, dict): out.dropped_lines += 1 + out.errors.append(f"line {n}: not a JSON object") continue t = None stamp = _parse_time(rec.get("timestamp")) @@ -131,10 +132,13 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, elif kind == "user": for block in _content(rec, out): if block.get("type") == "tool_result": - if not _well_formed_tool_result(block): # a result that names no call is lost: a dropped line - out.dropped_lines += 1 - continue - _apply_result(block, by_id) + # a result that names no recorded call is lost: a dropped line + if not _well_formed_tool_result(block): + _drop(out, "a tool_result with no usable tool_use_id") + elif block["tool_use_id"] not in by_id: + _drop(out, "a tool_result for no recorded tool call") + else: + _apply_result(block, by_id) elif kind == "result": _apply_final(out, rec, by_id) @@ -152,7 +156,7 @@ def _apply_assistant(out: Transcript, rec: dict[str, Any], t: float | None, for block in _content(rec, out): if block.get("type") == "tool_use": if not _well_formed_tool_use(block): # an unreadable call is a dropped line, not an empty one - out.dropped_lines += 1 + _drop(out, "a tool_use with no usable id, name or input") continue event = _tool_event(block, t, workdir) if rec.get("parent_tool_use_id"): @@ -232,9 +236,15 @@ def _content(rec: dict[str, Any], out: Transcript) -> list[dict[str, Any]]: if rec.get("type") == "user" and (isinstance(content, str) or content is None): # a user's prompt is a plain string: nothing lost return [] if not isinstance(content, list): - out.dropped_lines += 1 + _drop(out, "a message whose content is not a list") return [] blocks = [b for b in content if isinstance(b, dict) and isinstance(b.get("type"), str)] # a block says its type if len(blocks) != len(content): - out.dropped_lines += 1 + _drop(out, "a content block with no string type") return blocks + + +def _drop(out: Transcript, why: str) -> None: + """Count one unreadable part of the trace as dropped, and say why in the cell's trace errors.""" + out.dropped_lines += 1 + out.errors.append(why) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 089a1a99a..654717be7 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -21,7 +21,8 @@ cells with a trace to count from. - **first useful**: for a task that declares a ``useful`` definition, the cells that observed the first useful result (``result.json`` ``first_useful``, see :mod:`.useful`): how many, their median seconds and steps, and why the - others did not (``never``, ``unobserved``, ``untimed``, ``timeout``). + others did not (``never``, ``unobserved``, ``untimed``, ``timeout``). Cells that name both kinds get one such line + per kind. - skill arms add ``skill_invoked`` (cells whose agent used the expected skill, or any installed one when the task names none) and, when the task has an ``expected_skill``, ``skill_selected_correct`` (cells that used it first; ``skill_expected`` diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index d2eabcbbc..b04980dc0 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -112,27 +112,37 @@ def _median(values: list[float]) -> float | None: return statistics.median(values) if values else None -def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: - """``repo report``'s ``first_useful`` for one row: the cells observed, their medians and why the rest are not. - - None when no cell records a definition.""" - have = _declared(recs) - if not have: - return None +def _kind_summary(have: Sequence[Mapping[str, Any]], kind: str) -> dict[str, Any]: + """The cells of one kind: how many observed the hit, their medians and why the rest did not.""" seconds = _nums(u.get("issued_s") for u in _observed(have)) steps = _nums(u.get("steps") for u in have) - kind = useful_kind(recs) # "mixed" when the cells name both kinds: no median of those is one measure - mixed = kind == "mixed" return { "kind": kind, "n": len(have), "observed": len(steps), - "median_s": None if mixed else _median(seconds), - "median_steps": None if mixed else _median(steps), + "median_s": _median(seconds), + "median_steps": _median(steps), "reasons": dict(Counter(u["reason"] for u in have if u.get("reason"))), } +def useful_summary(recs: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: + """``repo report``'s ``first_useful`` for one row: the cells observed, their medians and why the rest are not. + + Cells that name both kinds give ``kind`` ``mixed`` and one summary per kind in ``by_kind``: a command's seconds + and an edit's are no one measure, so no count or median pools them. None when no cell records a definition.""" + have = _declared(recs) + if not have: + return None + kinds = sorted({u["kind"] for u in have}) + if len(kinds) == 1: + return _kind_summary(have, kinds[0]) + return { + "kind": "mixed", + "by_kind": {k: _kind_summary([u for u in have if u["kind"] == k], k) for k in kinds}, + } + + def useful_kind(recs: Sequence[Mapping[str, Any]]) -> str | None: """The kind the declared cells of ``recs`` define: ``command`` or ``edit``; ``mixed`` when they name both; None when none declares one.""" @@ -198,9 +208,11 @@ def useful_arms_line(counts: Mapping[str, Mapping[str, int] | None]) -> str | No def useful_line(summary: Mapping[str, Any] | None, width: int) -> list[str]: - """The row's first-useful summary on one indented line; none for a row without a definition.""" + """The row's first-useful summary on one indented line per kind; none for a row without a definition.""" if not summary: return [] + if summary["kind"] == "mixed": + return [line for part in summary["by_kind"].values() for line in useful_line(part, width)] parts = [f"{summary['observed']} of {summary['n']} observed"] if summary["median_s"] is not None: parts.append(f"median {summary['median_s']:.1f} s") diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index c0b8ff585..765884998 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -314,22 +314,23 @@ def test_useful_summary_is_the_observed_cells_medians_and_why_the_rest_are_not() assert useful_line(None, 4) == [] -def test_useful_summary_of_mixed_kinds_names_no_kind_and_no_median(): - # a command hit and an edit hit in one row: their seconds and steps are no one measure +def test_useful_summary_of_mixed_kinds_gives_one_summary_per_kind(): + # a command hit and an edit hit in one row: their seconds and steps are no one measure, so no kind pools the other recs = [ {"first_useful": _hit(10.0, 4)}, {"first_useful": {"kind": "edit", "reason": None, "issued_s": 40.0, "steps": 9}}, ] assert useful_summary(recs) == { "kind": "mixed", - "n": 2, - "observed": 2, - "median_s": None, - "median_steps": None, - "reasons": {}, + "by_kind": { + "command": {"kind": "command", "n": 1, "observed": 1, "median_s": 10.0, "median_steps": 4, + "reasons": {}}, + "edit": {"kind": "edit", "n": 1, "observed": 1, "median_s": 40.0, "median_steps": 9, "reasons": {}}, + }, } assert useful_line(useful_summary(recs), 4) == [ - " " * 8 + "first useful (mixed): 2 of 2 observed" + " " * 8 + "first useful (command): 1 of 1 observed, median 10.0 s, 4 steps", + " " * 8 + "first useful (edit): 1 of 1 observed, median 40.0 s, 9 steps", ] @@ -567,6 +568,42 @@ def test_a_trace_line_that_does_not_parse_marks_the_trace_incomplete(): assert convert("claude-stream-json", "", None).dropped_lines == 0 +@pytest.mark.parametrize("fmt", ["claude-stream-json", "codex-json", "copilot-json"]) +def test_a_trace_line_that_is_not_an_object_says_why_in_the_trace_errors(fmt): + transcript = convert(fmt, "[1]\n", None) + assert transcript.dropped_lines == 1 + assert any("not a JSON object" in e for e in transcript.errors) + + +@pytest.mark.parametrize("line", [ + {"type": "user", "message": {"content": [{"type": "tool_result", "is_error": False, "content": "ok"}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "nope", "content": "ok"}]}}, + {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "Edit", "input": {}}]}}, + {"type": "assistant", "message": {"content": [{"type": "text", "text": "a"}, 42]}}, + {"type": "assistant", "message": {"content": "not a list"}}, +]) +def test_each_dropped_claude_line_says_why_in_the_trace_errors(line): + transcript = convert("claude-stream-json", json.dumps(line), "/ws") + assert transcript.dropped_lines == 1 + assert transcript.errors, "a dropped block must leave a reason the cell can show" + + +def test_a_tool_result_that_names_no_recorded_call_is_unobserved_not_never(): + # the result's id is a non-empty string, but no call of the trace has it: the edit's result is lost, not a miss + lines = [ + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": "e", "name": "Edit", "input": {"file_path": "/ws/src/x.py"}}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "other", + "is_error": False, "content": "ok"}]}}, + {"type": "result", "subtype": "success", "is_error": False, "num_turns": 2, "result": "done", + "usage": {"input_tokens": 1, "output_tokens": 1}}, + ] + transcript = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws") + assert transcript.dropped_lines == 1 + events = None if transcript.dropped_lines else transcript.events + assert first_useful(parse_useful({"edit": "src/*.py"}), events)["reason"] == "unobserved" + + def test_the_compare_counts_are_the_cells_behind_each_value(tmp_path): batch = tmp_path / "b" _timed(batch, "t1", "guidance:none", 1, _hit(10.0, 4)) @@ -617,7 +654,8 @@ def test_a_task_whose_arms_define_different_kinds_of_first_useful_result_is_unpa s = compare([batch], "guidance:none") assert not [r for r in s["comparisons"] if r["measure"].startswith("first_useful")] assert any( - u["task"] == "t1" and "first useful kinds differ" in u["reason"] for u in s["unpaired"] + u["task"] == "t1" and u["measure"] == "first_useful" and "first useful kinds differ" in u["reason"] + for u in s["unpaired"] ) @@ -854,7 +892,7 @@ def test_an_edit_call_with_a_non_object_input_is_unobserved_not_never(): "usage": {"input_tokens": 1, "output_tokens": 1}}, ] transcript = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws") - assert transcript.dropped_lines == 1 + assert transcript.dropped_lines == 2 # the call's line, and its result's line, which names no recorded call events = None if transcript.dropped_lines else transcript.events assert first_useful(parse_useful({"edit": "**"}), events)["reason"] == "unobserved"