From f7ee245d824cc2037b5072215564f3883e62f381 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:42:58 -0700 Subject: [PATCH 01/17] feat(repo): rule ledger; repo compare --ledger A guidance rule is a bet that an agent reading it errs less on the cases it was written to prevent. repo_ledger.v1 records, per rule, its hypothesis, the arm that carries it, the linked replay or executable cases, and the date a person last reviewed it. repo compare --ledger runs compare() per rule over its linked cases and gives keep, review or insufficient-evidence per rule and model, with Holm correction across the rules. A review older than the policy is a warning. Verdicts are computed; cutting a rule stays with a person. Adds dokimi/repos/ledger.py, the schema and its fixture, the --ledger flag (inputs concept), the design note docs/rule-ledger.md, and tests on synthetic batches. The benchmark golden manifest allows one more doc. Signed-off-by: Nikolay Petrov --- dokimi/benchmarks/spec.v1.yaml | 2 +- ...261031_120000_napetrov_repo-rule-ledger.md | 3 + dokimi/docs/README.md | 4 + dokimi/docs/architecture.md | 4 + dokimi/docs/reference/cli-interface.json | 17 ++ dokimi/docs/rule-ledger.md | 145 +++++++++ dokimi/dokimi/artifacts.py | 1 + dokimi/dokimi/cli_flags.py | 1 + dokimi/dokimi/commands/repo_run.py | 17 +- dokimi/dokimi/repos/ledger.py | 215 +++++++++++++ dokimi/dokimi/runs/kinds.py | 1 + dokimi/dokimi/schemas/repo_ledger.v1.json | 40 +++ dokimi/mkdocs.yml | 1 + .../fixtures/artifacts/repo_ledger.v1.json | 23 ++ dokimi/tests/test_repo_ledger.py | 284 ++++++++++++++++++ dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + 17 files changed, 756 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md create mode 100644 dokimi/docs/rule-ledger.md create mode 100644 dokimi/dokimi/repos/ledger.py create mode 100644 dokimi/dokimi/schemas/repo_ledger.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/repo_ledger.v1.json create mode 100644 dokimi/tests/test_repo_ledger.py diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index b36efa758..753c06a0d 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 37 + max_docs: 38 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md new file mode 100644 index 000000000..0c1063a73 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md @@ -0,0 +1,3 @@ +### Added + +- **`repo compare --ledger FILE`** — gives each guidance rule of a `repo_ledger.v1` file a verdict: its arm against its baseline on the rule's linked cases (task ids), with the paired statistics of `repo compare`, Holm correction across the rules, and `keep`, `review` or `insufficient-evidence` with the reason. A rule whose `reviewed` date is older than `review_within_days` (default 180) gets a warning, not a verdict. The ledger only computes verdicts; cutting a rule is left to a person. The design, the ledger format and the incident-to-case path are in `docs/rule-ledger.md`. `--json` adds a `ledger` key to the comparison. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 4133acaa6..5903ad28a 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -368,6 +368,10 @@ code. - [preparing-a-repository-evidence.md](preparing-a-repository-evidence.md) — the runs behind each figure on that page: task set, arms, model, cell count and raw count. +- [rule-ledger.md](rule-ledger.md) — one row per guidance rule: its + hypothesis, the replay or executable cases of the failures it prevents, and + the review date; `repo compare --ledger` gives each rule a keep, review or + insufficient-evidence verdict. The cut decision stays with a person. ## Extending Dokimi diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 5aafd5a88..f145f7cb4 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -347,6 +347,10 @@ next release. what its `--task` selection keeps; `dokimi/repos/compare_reach.py` how small an effect a row can find (`mde`, an uncorrected t-based estimate, and the lowest `p_holm` Holm's family leaves it). +- `dokimi/repos/ledger.py` — `repo compare --ledger`: loads a `repo_ledger.v1` + file (rules with their hypothesis, arm, linked cases and review date) and gives + each rule a verdict from `repo compare` over its own cases, Holm-corrected + across the rules. [rule-ledger.md](rule-ledger.md). - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..798d5e18e 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10391,6 +10391,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--ledger", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_AppendAction", "choices": null, @@ -12894,6 +12910,7 @@ "'questions'", "'repo_budget'", "'repo_environment'", + "'repo_ledger'", "'repo_readiness'", "'result'", "'subject_scorecard'", diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md new file mode 100644 index 000000000..d33e2288a --- /dev/null +++ b/dokimi/docs/rule-ledger.md @@ -0,0 +1,145 @@ + + +# Rule ledger: past failures as replay cases + +A guidance rule is a bet: the author expects an agent that reads it to make a +specific mistake less often. The ledger keeps one row per rule with that bet +written down, the cases that stand for the failures the rule was written to +prevent, and the date a person last confirmed it still earns its place. +`repo compare --ledger` then checks the bet on those cases only and gives a +verdict. It does not cut anything. This follows the held-out ablation proposal +in [agentsmd/agents.md#213](https://github.com/agentsmd/agents.md/issues/213). + +## What exists + +| Piece | Where | What it does | +|---|---|---| +| Replay tasks from review threads | `dokimi/repos/replay/`, `repo tasks replay` ([contract §4.6](repo-evaluation-contract.md#46-replay-tasks-from-review-threads)) | One `dokimi:review_replay` task per merged PR, with the change as reviewed and the reviewers' threads as gold. No GitHub call. | +| Deterministic review grader | `dokimi/repos/graders/review.py` | Matches the agent's findings JSON to gold threads by file and a line window (`window`, default 5). No LLM. | +| Executable tasks and self-check | `repo tasks check` (`dokimi/repos/selfcheck.py`, [contract §4.3](repo-evaluation-contract.md#43-self-check-tasks-check-no-llm)) | A task must fail on the untouched workspace and pass after its oracle, so a case with a broken check cannot reach a paid run. | +| Arms | `dokimi/repos/arms.py` | `guidance:none`, `guidance:raw`, `guidance:rev:`, `patch:` and the rest. | +| Repetitions | `repo run --reps` | Several reps per (task, arm, model). | +| Paired statistics | `repo compare` (`dokimi/repos/compare.py`) | Per-task deltas, sign-flip p, Holm over the gate and score rows of one invocation. | +| Rule vocabulary | [repo-evaluation-agent-modes.md §4](repo-evaluation-agent-modes.md#4-team-review-rules) | Mined `review_rules.json` entries (`R13`, `cut_date`). Method only: no code reads that file yet. | + +Before this change nothing tied a rule to the cases it was written from, and +`repo compare` had no notion of a rule. + +## From an incident to a held-out case + +An incident is a failure a rule should prevent. Three sources are in use or +planned: + +1. **A review comment on a merged PR.** Run `repo tasks replay` on the PR's + threads. The case is `review_replay_pr` and already exists. Its grader is + recall of the gold threads, so the check is deterministic. +2. **A failed agent run, or a bug a fix PR corrected.** Write an executable task + in the content repository: a test that fails on the base commit and passes + with the oracle. `repo tasks check` enforces both halves. No scaffold command + exists yet; the task is written by hand. +3. **A transcript slice.** A transcript shows what the agent did, not what + should have happened. It is evidence for case 1 or 2, not a case: a case + needs a check that does not read the transcript. + +Then, per rule: + +1. Record the incident as the rule's `source` (a file and anchor, or the + comment URL) and state the `hypothesis`: what the rule should change in + an answer on its cases. +2. Put the case ids in `cases`. Use cases the rule was not written from: a + rule checked on the failure it was mined from only shows it can repeat + itself. +3. Run `repo tasks check` on the cases, then the rule's arm and its baseline + with `--reps 3` to `5`. +4. Compare with `repo compare --ledger`. + +## Ledger format (`repo_ledger.v1`) + +`dokimi/schemas/repo_ledger.v1.json` is the schema. `rules` must not be +empty, and ids must be unique in the file (the loader checks this, since JSON +Schema cannot). + +```json +{ + "schema_version": "repo_ledger.v1", + "policy": {"alpha": 0.05, "review_within_days": 180}, + "rules": [ + { + "id": "R13", + "rule": "Revert edits to files the change does not need.", + "source": "AGENTS.md#commits-and-pull-requests", + "hypothesis": "With the rule, agents stop touching unrelated files on the replayed PRs.", + "arm": "guidance:rules", + "baseline": "guidance:none", + "cases": ["review_replay_pr3580", "review_replay_pr3659", "fix_df_extratrees_reg"], + "reviewed": "2026-10-01" + } + ] +} +``` + +| Field | Meaning | +|---|---| +| `id` | Stable across edits, `[A-Za-z0-9][A-Za-z0-9._-]*`. | +| `rule`, `source` | Optional text and anchor, printed for the reader. | +| `hypothesis` | Required. The claim the verdict tests, in words. | +| `arm` | The treatment arm that carries the rule, `ARM` or `BATCH/ARM`. | +| `baseline` | Optional. Overrides the `--baseline` of the compare for this rule. | +| `cases` | Task ids, at least one, unique. Ids with no finished cell are listed as missing. | +| `reviewed` | `YYYY-MM-DD`, the last date a person confirmed the rule. | +| `policy.alpha` | Default 0.05. | +| `policy.review_within_days` | Default 180. An older `reviewed` gets a warning. | + +## Verdicts + +For each rule and model, `repo compare --ledger` runs the ordinary paired +comparison of the rule's arm against its baseline over the rule's linked cases +(`repo compare` itself, so the pairing, headroom and provenance warnings are +the same). The p values of all rule results are Holm-corrected together. The +rules' family is separate from the family of the comparison's own rows. + +| Verdict | When | Meaning | +|---|---|---| +| `keep` | Mean delta above zero and Holm p below alpha | The arm beats its baseline on the linked cases. | +| `review` | Mean delta below zero and Holm p below alpha | The baseline beats the arm there. A person decides whether to fix or cut the rule. | +| `insufficient-evidence` | Anything else | Not measured, mixed kinds, every case tied, no significant difference, or too few changed cases for any outcome to reach alpha. The reason says which. | + +A review older than `review_within_days` adds a warning and leaves the verdict +alone. "No significant difference" is not a verdict to cut: the rule is not +shown to help, and whether that is enough to cut it is a decision for a person. + +## CLI + +```bash +python cli.py repo compare repo-runs/replay --baseline guidance:none --ledger repo-ledger.json +python cli.py repo compare repo-runs/replay --baseline guidance:none --ledger repo-ledger.json --json +``` + +The text output keeps the ordinary comparison and adds a `rule ledger` section. +A ledger that does not validate, or a baseline the batches do not have, exits 2 +with a `✗` line naming the file (and the rule, where the error is the rule's). The +ordinary comparison's flags (`--arm`, `--task`, `--metric`, ...) do not change the +ledger, which uses the rules' own cases and arms. + +## Limits + +- A rule's arm carries every rule that is in it. A verdict on one rule compares + the whole arm with its baseline, so it confounds the rule with its neighbours. + A per-rule claim needs an arm that differs by that rule alone (a `patch:` + arm that removes one rule, say). The ledger does not build those arms. +- Few cases give few verdicts. With fewer than six cases whose delta is not zero, + no outcome reaches p below 0.05, and the reason says so. +- Gate and score cases do not mix in one rule. + +## Not built + +- A scaffold that turns a failed run directory, or a description and a failing + check, into a task skeleton. The executable-task format needs an oracle and a + verifier, so a skeleton would still need hand work. +- A check that a rule's cases are held out from the PRs it was mined from. The + [mining leak guard](repo-evaluation-agent-modes.md#4-team-review-rules) + (`cut_date`) is the method; nothing enforces it for a ledger yet. +- Reading the mined `review_rules.json` into the ledger. diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..b178ea7d4 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -58,6 +58,7 @@ "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", + "repo_ledger": "repo_ledger.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index 9e2655825..a592b9b62 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -242,6 +242,7 @@ class Concept: Flag("--answers", "FILE", "path"), Flag("--registry", "FILE", "path"), Flag("--from", "RUN", "text"), + Flag("--ledger", "FILE", "path"), ), banned=("--product", "--library", "--libraries", "--target", "--root", "--repo-root", "--tasks-root", "--subjects-root", "--questions-root", "--questions-from", diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 8c81f0859..de3d28dc3 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -352,8 +352,10 @@ def cmd_repo_report(args: argparse.Namespace) -> None: def cmd_repo_compare(args: argparse.Namespace) -> None: - """Per (treatment, model, measure): per-task deltas against the baseline, 95% CI, sign-flip p.""" + """Per (treatment, model, measure): per-task deltas against the baseline, 95% CI, sign-flip p; ``--ledger`` + adds one verdict per guidance rule.""" from dokimi.repos.compare import CompareError, compare, format_compare + from dokimi.repos.ledger import LedgerError, evaluate, format_ledger, load_ledger from dokimi.repos.report import CellFileError dirs = [Path(d) for d in args.batch_dirs] @@ -365,10 +367,16 @@ def cmd_repo_compare(args: argparse.Namespace) -> None: try: summary = compare(dirs, args.baseline, arms, args.model, tuple(args.metric or ()), pass_k=args.pass_k, path_stability=args.path_stability, tasks=args.task) - except (CompareError, CellFileError) as exc: # an unusable baseline, arm, metric or cell file: exit 2 + if args.ledger: + summary["ledger"] = evaluate(load_ledger(Path(args.ledger)), dirs, args.baseline, args.model, + ledger_name=args.ledger) + except (CompareError, CellFileError, LedgerError) as exc: # an unusable baseline, arm, metric, cell or ledger print(f"✗ {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) - print(json.dumps(summary, indent=2) if args.json else format_compare(summary)) + text = format_compare(summary) + if "ledger" in summary: + text += "\n\n" + format_ledger(summary["ledger"]) + print(json.dumps(summary, indent=2) if args.json else text) def _usd(text: str) -> float: @@ -520,5 +528,8 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: help="Also compare pass^K: the chance K cells of a task all pass its gate") compare_p.add_argument("--path-stability", action="store_true", dest="path_stability", help="Also compare path_distance: how much the tool paths of a task's cells differ") + compare_p.add_argument("--ledger", default=None, metavar="FILE", + help="Also give each guidance rule of this repo_ledger.v1 file a verdict (keep, review or " + "insufficient-evidence) from its linked cases, against its baseline (dokimi/docs/rule-ledger.md)") compare_p.add_argument("--json", action="store_true", help="Print the comparison as JSON") compare_p.set_defaults(func=cmd_repo_compare) diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py new file mode 100644 index 000000000..2f54c70eb --- /dev/null +++ b/dokimi/dokimi/repos/ledger.py @@ -0,0 +1,215 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``repo compare --ledger``: one verdict per guidance rule, from the cases it was written to prevent. + +A ledger (``repo_ledger.v1``, dokimi/docs/rule-ledger.md) gives each rule its hypothesis, the arm that +carries it, the task ids of the cases standing for the failures it prevents, and the date it was last +reviewed. Each rule is compared with its baseline by :func:`.compare.compare` over its own cases, so the +pairing, headroom and provenance warnings are those of ``repo compare``. The rules' p values are +Holm-corrected together here, not in the comparison's own family. + +Verdicts are computed, cutting is not: ``keep`` (the arm beats its baseline on the linked cases at +Holm-adjusted p below alpha), ``review`` (the baseline beats it at that level), ``insufficient-evidence`` +(anything else, with the reason). A review date older than the policy allows is a warning, not a verdict. +""" + +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Any + +from dokimi.artifacts import schema_errors +from dokimi.eval.stats import adjust_pvalues + +from .compare import PRIMARY, compare, load_batches, min_sign_flip_p, parse_selector +from .compare_reach import ALPHA +from .compare_tasks import CompareError + +KEEP, REVIEW, INSUFFICIENT = "keep", "review", "insufficient-evidence" +DEFAULT_REVIEW_DAYS = 180 +_EPS = 1e-9 # a mean delta this close to zero is a tie, as in compare + + +class LedgerError(ValueError): + """A ledger that cannot be read or does not validate; the message says where.""" + + +@dataclass(frozen=True) +class Rule: + id: str + hypothesis: str + arm: str + cases: tuple[str, ...] + reviewed: date + baseline: str | None + text: str | None + source: str | None + + +@dataclass(frozen=True) +class Ledger: + rules: tuple[Rule, ...] + alpha: float + review_within_days: int + + +def load_ledger(path: Path) -> Ledger: + """The ledger in ``path``, validated against ``repo_ledger.v1`` and checked for duplicate ids.""" + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise LedgerError(f"{path}: cannot read the ledger: {exc}") from exc + errors = schema_errors("repo_ledger.v1", raw) + if errors: + raise LedgerError(f"{path}: " + "; ".join(errors)) + rules, seen = [], set() + for entry in raw["rules"]: + rule = _rule(entry, path) + if rule.id in seen: + raise LedgerError(f"{path}: rule id {rule.id!r} is listed twice") + seen.add(rule.id) + rules.append(rule) + policy = raw.get("policy", {}) + return Ledger(tuple(rules), float(policy.get("alpha", ALPHA)), + int(policy.get("review_within_days", DEFAULT_REVIEW_DAYS))) + + +def _rule(entry: dict[str, Any], path: Path) -> Rule: + try: + parse_selector(entry["arm"]) + if entry.get("baseline") is not None: + parse_selector(entry["baseline"]) + reviewed = date.fromisoformat(entry["reviewed"]) + except ValueError as exc: # a bad selector (CompareError is one) or an impossible date such as 2026-02-30 + raise LedgerError(f"{path}: rule {entry['id']!r}: {exc}") from exc + return Rule(id=entry["id"], hypothesis=entry["hypothesis"], arm=entry["arm"], cases=tuple(entry["cases"]), + reviewed=reviewed, baseline=entry.get("baseline"), text=entry.get("rule"), source=entry.get("source")) + + +def evaluate(ledger: Ledger, batch_dirs: list[Path], baseline: str, models: list[str] | None = None, + seed: int = 0, today: date | None = None, ledger_name: str = "") -> dict[str, Any]: + """One result per rule and model: paired statistics, Holm-adjusted p, verdict and reason.""" + today = today or datetime.now(UTC).date() + known = {c["task"] for c in load_batches(batch_dirs)[0]} + entries: list[dict[str, Any]] = [] + warnings: list[str] = [] + for rule in ledger.rules: + rule_entries = _rule_entries(rule, batch_dirs, baseline, models, known, seed, warnings) + age = (today - rule.reviewed).days + stale = age > ledger.review_within_days + for entry in rule_entries: + entry.update(reviewed=rule.reviewed.isoformat(), days_since_review=age, stale=stale) + if stale: + entry["warnings"].append(f"review date {rule.reviewed.isoformat()} is {age} days old (policy: " + f"{ledger.review_within_days}): re-review the rule") + entries += rule_entries + _holm(entries) + for entry in entries: + entry["verdict"], entry["reason"] = _verdict(entry, ledger.alpha) + return {"ledger": ledger_name, "alpha": ledger.alpha, "review_within_days": ledger.review_within_days, + "today": today.isoformat(), "family": sum(e["p"] is not None for e in entries), + "rules": entries, "warnings": list(dict.fromkeys(warnings))} + + +def _head(rule: Rule, baseline: str, linked: int, missing: list[str]) -> dict[str, Any]: + return {"rule": rule.id, "rule_text": rule.text, "source": rule.source, "hypothesis": rule.hypothesis, + "arm": rule.arm, "baseline": baseline, "linked": linked, "missing": missing, "model": None, + "measure": None, "paired": 0, "error": None, "mixed": False, "changed": 0, "headroom": None, + "baseline_mean": None, "treatment_mean": None, "delta": {"mean": None, "lo": None, "hi": None}, + "up": 0, "down": 0, "tied": 0, "p": None, "p_holm": None, "warnings": []} + + +def _rule_entries(rule: Rule, batch_dirs: list[Path], default_baseline: str, models: list[str] | None, + known: set[str], seed: int, warnings: list[str]) -> list[dict[str, Any]]: + """A rule's results: one per model with a paired gate or score measure, else one that says why not.""" + missing = [c for c in rule.cases if c not in known] + present = [c for c in rule.cases if c in known] + base = rule.baseline or default_baseline + head = _head(rule, base, len(rule.cases), missing) + if not present: + return [{**head, "error": "no linked case has a finished cell"}] + try: + summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present) + except CompareError as exc: # the arm or the baseline has no cell among this rule's cases + return [{**head, "error": str(exc)}] + warnings.extend(summary["warnings"]) + rows = [r for r in summary["comparisons"] if r["measure"] in PRIMARY] + if not rows: + reasons = sorted({u["reason"] for u in summary["unpaired"] if u["task"] in present}) + return [{**head, "error": "; ".join(reasons) or "no linked case has cells in both arms"}] + per_model = Counter(r["model"] for r in rows) + return [_entry(head, r, per_model[r["model"]] > 1) for r in rows] + + +def _entry(head: dict[str, Any], row: dict[str, Any], mixed: bool) -> dict[str, Any]: + d = row["delta"] + out = {**head, "model": row["model"], "measure": row["measure"], "paired": row["n_tasks"], + "baseline_mean": row["baseline_mean"], "treatment_mean": row["treatment_mean"], + "delta": {"mean": d["mean"], "lo": d["lo"], "hi": d["hi"]}, "up": row["up"], "down": row["down"], + "tied": row["tied"], "p": row["p"], "headroom": row["headroom"], "mixed": mixed, + "changed": row["up"] + row["down"], "warnings": []} + if row["headroom"]: + out["warnings"].append(f"{row['measure']} headroom: {row['headroom']} (the cases cannot show an effect)") + return out + + +def _holm(entries: list[dict[str, Any]]) -> None: + """``p_holm`` over every result that has a p, as one family (the exact adjusted value).""" + scored = [e for e in entries if e["p"] is not None] + for entry, adjusted in zip(scored, adjust_pvalues([e["p"] for e in scored], rounded=False), strict=True): + entry["p_holm"] = adjusted + + +def _verdict(entry: dict[str, Any], alpha: float) -> tuple[str, str]: + """The verdict of one result and why. Cutting a rule is left to a person.""" + if entry["error"] or not entry["paired"]: + return INSUFFICIENT, f"not measured: {entry['error'] or 'no linked case has cells in both arms'}" + if entry["mixed"]: + return INSUFFICIENT, "the linked cases mix gate and score tasks: link one kind of task per rule" + if entry["p_holm"] is None: + return INSUFFICIENT, "every linked case ties: there is no difference to test" + p, mean = entry["p_holm"], entry["delta"]["mean"] + if p < alpha and abs(mean) > _EPS: + if mean > 0: + return KEEP, f"beats {entry['baseline']} on its linked cases (p_holm {p:.4f} < {alpha})" + return REVIEW, (f"{entry['baseline']} beats it on its linked cases (p_holm {p:.4f} < {alpha}): " + "a person decides whether to fix or cut the rule") + reason = f"no significant difference from {entry['baseline']} (p_holm {p:.4f}, alpha {alpha})" + if min_sign_flip_p(entry["changed"]) >= alpha: + reason += f"; with {entry['changed']} changed case(s) no outcome reaches p < {alpha}: link more cases" + return INSUFFICIENT, reason + + +def format_ledger(result: dict[str, Any]) -> str: + """The rule results as text: one block per rule and model, the verdict first.""" + lines = [f"rule ledger {result['ledger']}: alpha {result['alpha']}, Holm over {result['family']} " + f"result(s), reviews checked as of {result['today']}"] + for e in result["rules"]: + where = f"{e['rule']}: {e['arm']} vs {e['baseline']}" + (f", model {e['model']}" if e["model"] else "") + lines.append(f"{where}, {e['measure'] or 'not measured'}: {e['verdict']}") + if e["paired"]: + d = e["delta"] + ci = "-" if d["lo"] is None else f"[{d['lo']:+.3f}, {d['hi']:+.3f}]" + lines.append(f" paired {e['paired']} of {e['linked']} linked: pass rate {_num(e['baseline_mean'])} -> " + f"{_num(e['treatment_mean'])}, delta {d['mean']:+.3f} {ci}, p {_p(e['p'])}, " + f"p_holm {_p(e['p_holm'])}, up/down/tied {e['up']}/{e['down']}/{e['tied']}") + lines.append(f" {e['reason']}") + if e["missing"]: + lines.append(f" no finished cell for: {', '.join(e['missing'])}") + lines.append(f" hypothesis: {e['hypothesis']}") + lines += [f" warning: {w}" for w in e["warnings"]] + lines += [f"warning: {w}" for w in result["warnings"]] + return "\n".join(lines) + + +def _num(value: float | None) -> str: + return "-" if value is None else f"{value:.3f}" + + +def _p(value: float | None) -> str: + return "-" if value is None else f"{value:.4f}" diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..f87521b80 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -65,6 +65,7 @@ class KindFields: "repo_budget": KindFields(subject=("tree",), counts=("entries",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "repo_ledger": KindFields(counts=("rules",)), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/repo_ledger.v1.json b/dokimi/dokimi/schemas/repo_ledger.v1.json new file mode 100644 index 000000000..7968cc41f --- /dev/null +++ b/dokimi/dokimi/schemas/repo_ledger.v1.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/repo_ledger.v1.json", + "title": "Guidance rule ledger v1", + "description": "One row per guidance rule: the hypothesis it was written for, the replay or executable cases that stand for the failures it prevents, and the date it was last reviewed. `repo compare --ledger` pairs each rule's cases against its arm and a baseline and reports a verdict (keep, review or insufficient-evidence). Only the verdict is computed; cutting a rule is a human decision. Ids are unique within the file (checked by the loader, which JSON Schema cannot express).", + "type": "object", + "required": ["schema_version", "rules"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "repo_ledger.v1"}, + "policy": {"$ref": "#/$defs/policy"}, + "rules": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/rule"}} + }, + "$defs": { + "policy": { + "type": "object", + "description": "Optional. alpha: the Holm-adjusted p below which a delta counts. review_within_days: a review older than this is flagged as stale.", + "additionalProperties": false, + "properties": { + "alpha": {"type": "number", "exclusiveMinimum": 0, "maximum": 1}, + "review_within_days": {"type": "integer", "minimum": 1} + } + }, + "rule": { + "type": "object", + "required": ["id", "hypothesis", "arm", "cases", "reviewed"], + "additionalProperties": false, + "properties": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$", "description": "Stable across edits of the guidance, unique in the file."}, + "rule": {"type": "string", "minLength": 1, "description": "The guidance text, as written."}, + "source": {"type": "string", "minLength": 1, "description": "Where the rule lives or was mined from: a file and anchor, or a review comment URL."}, + "hypothesis": {"type": "string", "minLength": 1, "description": "What the rule should change in an agent's answer on its cases."}, + "arm": {"type": "string", "minLength": 1, "description": "The treatment arm that carries the rule, ARM or BATCH/ARM."}, + "baseline": {"type": "string", "minLength": 1, "description": "Optional baseline arm for this rule; default: the --baseline of the compare."}, + "cases": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}, "description": "Task ids of the cases that stand for the failures the rule prevents."}, + "reviewed": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "description": "The date (YYYY-MM-DD) a maintainer last confirmed the rule still earns its place."} + } + } + } +} diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 0f2d53b27..083843217 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -131,6 +131,7 @@ nav: - repo-evaluation-agent-modes.md - preparing-a-repository.md - preparing-a-repository-evidence.md + - rule-ledger.md - Extending Dokimi: - architecture.md - contributing-metric.md diff --git a/dokimi/tests/fixtures/artifacts/repo_ledger.v1.json b/dokimi/tests/fixtures/artifacts/repo_ledger.v1.json new file mode 100644 index 000000000..28ebff455 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/repo_ledger.v1.json @@ -0,0 +1,23 @@ +{ + "schema_version": "repo_ledger.v1", + "policy": {"alpha": 0.05, "review_within_days": 180}, + "rules": [ + { + "id": "R13", + "rule": "Revert edits to files the change does not need.", + "source": "AGENTS.md#commits-and-pull-requests", + "hypothesis": "With the rule, agents stop touching unrelated files on the replayed PRs.", + "arm": "guidance:rules", + "baseline": "guidance:none", + "cases": ["review_replay_pr3580", "review_replay_pr3659", "fix_df_extratrees_reg"], + "reviewed": "2026-10-01" + }, + { + "id": "R14", + "hypothesis": "With the rule, agents run the touched tests before they report done.", + "arm": "guidance:rules", + "cases": ["fix_df_extratrees_reg"], + "reviewed": "2026-09-15" + } + ] +} diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py new file mode 100644 index 000000000..c0d480149 --- /dev/null +++ b/dokimi/tests/test_repo_ledger.py @@ -0,0 +1,284 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``repo compare --ledger``: rule verdicts over hand-written batch directories. No docker, no LLM.""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.ledger import KEEP, REVIEW, INSUFFICIENT, LedgerError, evaluate, format_ledger, load_ledger + +TODAY = date(2026, 10, 31) +BASE, ARM = "guidance:none", "guidance:rules" + + +def _cell(batch: Path, task: str, arm: str, *, success: bool | None = None, score: float | None = None, + model: str = "m-1") -> None: + rd = batch / f"{task}__{arm.replace(':', '-')}__{model}__r1" + rd.mkdir(parents=True) + score_task = score is not None + grade = {"status": "graded", "pass": None if score_task else success, "success": None if score_task else success, + "score": score, "score_only": score_task, "reasons": []} + rec = {"task": task, "arm": arm, "model": model, "rep": 1, "score_task": score_task, "grade": grade, + "harness_version": "1.0", "score_max": 1.0 if score_task else None, "model_resolved": None, + "agent": {"cost_usd": 0.1, "num_turns": 10, "timed_out": False, "is_error": False, "rc": None, + "over_budget": False, "network": "none"}} + (rd / "meta.json").write_text("{}") + (rd / "result.json").write_text(json.dumps(rec)) + (rd / "grade.json").write_text(json.dumps({"metrics": {}})) + + +def _gate_batch(root: Path, outcomes: dict[str, tuple[bool, bool]], model: str = "m-1") -> Path: + """One batch; ``outcomes`` maps task -> (baseline passes, rules arm passes).""" + batch = root / "run" + for task, (base, treat) in outcomes.items(): + _cell(batch, task, BASE, success=base, model=model) + _cell(batch, task, ARM, success=treat, model=model) + return batch + + +def _ledger(path: Path, rules: list[dict], **policy) -> Path: + data: dict = {"schema_version": "repo_ledger.v1", "rules": rules} + if policy: + data["policy"] = policy + path.write_text(json.dumps(data), encoding="utf-8") + return path + + +def _rule(rid: str, cases: list[str], reviewed: str = "2026-10-30", **extra) -> dict: + return {"id": rid, "hypothesis": f"{rid} changes the answer", "arm": ARM, "cases": cases, + "reviewed": reviewed, **extra} + + +def _ids(tasks: str, n: int) -> list[str]: + return [f"{tasks}_{i}" for i in range(n)] + + +def _verdicts(result: dict) -> dict[str, str]: + return {(e["rule"], e["model"]): e["verdict"] for e in result["rules"]} + + +def test_a_valid_ledger_loads_with_the_policy_defaults(tmp_path): + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"], source="AGENTS.md#commits", rule="Say why.")]) + ledger = load_ledger(path) + assert [r.id for r in ledger.rules] == ["R1"] + assert (ledger.alpha, ledger.review_within_days) == (0.05, 180) + assert ledger.rules[0].reviewed == date(2026, 10, 30) + assert ledger.rules[0].source == "AGENTS.md#commits" + + +@pytest.mark.parametrize("bad", [ + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": ["t1"], "reviewed": "2026-02-30"}, + {"id": "R1", "hypothesis": "h", "arm": "guidance:rules/", "cases": ["t1"], "reviewed": "2026-01-01"}, + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": [], "reviewed": "2026-01-01"}, + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": ["t1", "t1"], "reviewed": "2026-01-01"}, + {"id": "R 1", "hypothesis": "h", "arm": ARM, "cases": ["t1"], "reviewed": "2026-01-01"}, + {"id": "R1", "arm": ARM, "cases": ["t1"], "reviewed": "2026-01-01"}, + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": ["t1"], "reviewed": "2026-01-01", "verdict": "cut"}, +], ids=["impossible-date", "bad-selector", "no-cases", "repeated-case", "id-with-space", "no-hypothesis", + "unknown-key"]) +def test_a_rule_that_does_not_validate_is_refused(tmp_path, bad): + path = _ledger(tmp_path / "ledger.json", [bad]) + with pytest.raises(LedgerError): + load_ledger(path) + + +def test_a_repeated_rule_id_is_refused(tmp_path): + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"]), _rule("R1", ["t2"])]) + with pytest.raises(LedgerError, match="R1"): + load_ledger(path) + + +def test_an_unreadable_ledger_is_refused(tmp_path): + path = tmp_path / "ledger.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(LedgerError, match="cannot read"): + load_ledger(path) + + +def test_all_cases_improved_is_keep_and_all_worse_is_review(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks} | {u: (True, False) for u in _ids("u", 8)}) + ledger = load_ledger(_ledger(tmp_path / "ledger.json", [_rule("up", tasks), _rule("down", _ids("u", 8))])) + result = evaluate(ledger, [batch], BASE, today=TODAY) + assert _verdicts(result) == {("up", "m-1"): KEEP, ("down", "m-1"): REVIEW} + up = result["rules"][0] + assert (up["up"], up["down"], up["paired"]) == (8, 0, 8) + assert (up["baseline_mean"], up["treatment_mean"]) == (0.0, 1.0) + assert up["p_holm"] == pytest.approx(2 * 2 / 2**8, abs=1e-5) # Holm over the two rules; p is rounded to 6 places + + +def test_few_changed_cases_are_insufficient_evidence_and_the_reason_says_why(tmp_path): + tasks = _ids("t", 3) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY) + entry = result["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert "no outcome reaches p < 0.05" in entry["reason"] + + +def test_holm_corrects_the_rules_together(tmp_path): + # each rule alone reaches p = 2/64 = 0.031 (six cases all improved); two rules together give Holm 0.0625 + first, second = _ids("a", 6), _ids("b", 6) + batch = _gate_batch(tmp_path, {t: (False, True) for t in first + second}) + alone = evaluate(load_ledger(_ledger(tmp_path / "one.json", [_rule("A", first)])), [batch], BASE, today=TODAY) + both = evaluate(load_ledger(_ledger(tmp_path / "two.json", [_rule("A", first), _rule("B", second)])), + [batch], BASE, today=TODAY) + assert alone["rules"][0]["verdict"] == KEEP + assert [e["verdict"] for e in both["rules"]] == [INSUFFICIENT, INSUFFICIENT] + assert both["rules"][0]["p_holm"] == pytest.approx(2 * alone["rules"][0]["p"]) + assert both["family"] == 2 + + +def test_a_missing_case_is_listed_and_the_rest_are_measured(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + rule = _rule("R1", tasks + ["never_ran"]) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, today=TODAY)["rules"][0] + assert entry["missing"] == ["never_ran"] + assert (entry["linked"], entry["paired"], entry["verdict"]) == (9, 8, KEEP) + + +def test_a_rule_with_no_case_that_ran_is_not_measured(tmp_path): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["ghost"])])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert entry["reason"].startswith("not measured") + assert entry["missing"] == ["ghost"] + + +def test_an_arm_with_no_cell_on_the_linked_cases_is_not_measured(tmp_path): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + _cell(batch, "t2", BASE, success=False) # the baseline ran t2, the rules arm did not + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["t2"])])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert "not measured" in entry["reason"] + + +def test_a_tie_on_every_linked_case_has_nothing_to_test(tmp_path): + tasks = _ids("t", 4) + batch = _gate_batch(tmp_path, {t: (True, True) for t in tasks}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert "every linked case ties" in entry["reason"] + + +def test_gate_and_score_cases_in_one_rule_are_refused_as_mixed(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("g", 8)}) + for task in _ids("s", 8): + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=0.9) + rule = _rule("R1", _ids("g", 8) + _ids("s", 8)) + results = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, today=TODAY)["rules"] + assert {e["measure"] for e in results} == {"gate", "score"} + assert all(e["verdict"] == INSUFFICIENT and "mix gate and score" in e["reason"] for e in results) + + +def test_a_score_rule_is_measured_on_score(tmp_path): + batch = _score_batch(tmp_path) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("s", 8))])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["measure"] == "score" + assert entry["verdict"] == KEEP + assert entry["delta"]["mean"] == pytest.approx(0.7) + + +def _score_batch(root: Path) -> Path: + batch = root / "run" + for task in _ids("s", 8): + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=0.9) + return batch + + +def test_results_are_per_model(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}, model="m-1") + for task in _ids("t", 8): # a second model where the rule does nothing + _cell(batch, task, BASE, success=True, model="m-2") + _cell(batch, task, ARM, success=True, model="m-2") + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("t", 8))])), [batch], BASE, + today=TODAY) + assert _verdicts(result) == {("R1", "m-1"): KEEP, ("R1", "m-2"): INSUFFICIENT} + + +def test_a_review_older_than_the_policy_is_stale_but_keeps_its_verdict(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks, reviewed="2026-01-01")], review_within_days=180) + entry = evaluate(load_ledger(path), [batch], BASE, today=TODAY)["rules"][0] + assert entry["stale"] is True + assert entry["days_since_review"] == (TODAY - date(2026, 1, 1)).days + assert entry["verdict"] == KEEP + assert any("re-review" in w for w in entry["warnings"]) + + +def test_a_recent_review_is_not_stale(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["stale"] is False + assert not any("re-review" in w for w in entry["warnings"]) + + +def test_a_rule_can_name_its_own_baseline(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "t_0", "guidance:raw", success=True) # the rule's baseline has a cell for one of its cases only + rule = _rule("R1", tasks, baseline="guidance:raw") + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, today=TODAY)["rules"][0] + assert entry["baseline"] == "guidance:raw" + assert entry["paired"] == 1 # only t_0 has a cell in both arms + + +def test_format_names_the_verdict_the_hypothesis_and_the_stale_review(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks, reviewed="2026-01-01")]) + text = format_ledger(evaluate(load_ledger(path), [batch], BASE, today=TODAY, ledger_name="ledger.json")) + assert "R1: guidance:rules vs guidance:none, model m-1, gate: keep" in text + assert "hypothesis: R1 changes the answer" in text + assert "re-review" in text + + +def test_compare_with_ledger_adds_the_rule_results_to_the_json(tmp_path, capsys): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks)]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path), + "--json"]) + args.func(args) + data = json.loads(capsys.readouterr().out) + assert data["ledger"]["ledger"] == str(path) + assert [(e["rule"], e["verdict"]) for e in data["ledger"]["rules"]] == [("R1", KEEP)] + + +def test_compare_text_prints_the_rule_section(tmp_path, capsys): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks)]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path)]) + args.func(args) + out = capsys.readouterr().out + assert "rule ledger" in out + assert "R1: guidance:rules vs guidance:none, model m-1, gate: keep" in out + + +def test_a_bad_ledger_exits_with_usage_error(tmp_path, capsys): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"], reviewed="2026-02-30")]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path)]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "R1" in capsys.readouterr().err diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..7d748b71c 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -48,6 +48,7 @@ "repo_budget.v1": ({}, "toy", None, None), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "repo_ledger.v1": ({}, None, None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..f00a281f4 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -37,6 +37,7 @@ "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "repo_ledger.v1": {"rules": 2}, "task_inspect.v1": {"items": 4}, "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, "tool_preference.v1": {"items": 16, "results": 16}, From a9ab5579af3ae523399df713ecc31355458d09d0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 14:08:28 -0700 Subject: [PATCH 02/17] fix(repo): ledger keeps a model's result when another model lacks cells A --model with no cell on a rule's linked cases no longer fails the whole rule: each model is compared on its own. Linked cases that do not pair are named on the result. An empty --ledger path is refused, not ignored. The docs say which baseline errors exit 2 and which are not measured; the changelog no longer calls a stale review a non-verdict. Signed-off-by: Nikolay Petrov --- ...261031_120000_napetrov_repo-rule-ledger.md | 2 +- dokimi/docs/rule-ledger.md | 6 ++-- dokimi/dokimi/commands/repo_run.py | 2 +- dokimi/dokimi/repos/ledger.py | 31 +++++++++++++++---- dokimi/tests/test_repo_ledger.py | 30 ++++++++++++++++++ 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md index 0c1063a73..d5414e748 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md @@ -1,3 +1,3 @@ ### Added -- **`repo compare --ledger FILE`** — gives each guidance rule of a `repo_ledger.v1` file a verdict: its arm against its baseline on the rule's linked cases (task ids), with the paired statistics of `repo compare`, Holm correction across the rules, and `keep`, `review` or `insufficient-evidence` with the reason. A rule whose `reviewed` date is older than `review_within_days` (default 180) gets a warning, not a verdict. The ledger only computes verdicts; cutting a rule is left to a person. The design, the ledger format and the incident-to-case path are in `docs/rule-ledger.md`. `--json` adds a `ledger` key to the comparison. +- **`repo compare --ledger FILE`** — gives each guidance rule of a `repo_ledger.v1` file a verdict: its arm against its baseline on the rule's linked cases (task ids), with the paired statistics of `repo compare`, Holm correction across the rules, and `keep`, `review` or `insufficient-evidence` with the reason. A rule whose `reviewed` date is older than `review_within_days` (default 180) gets a warning; its verdict is computed as usual. The ledger only computes verdicts; cutting a rule is left to a person. The design, the ledger format and the incident-to-case path are in `docs/rule-ledger.md`. `--json` adds a `ledger` key to the comparison. diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index d33e2288a..0ac8198de 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -119,8 +119,10 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --ledger re ``` The text output keeps the ordinary comparison and adds a `rule ledger` section. -A ledger that does not validate, or a baseline the batches do not have, exits 2 -with a `✗` line naming the file (and the rule, where the error is the rule's). The +A ledger that does not validate, or a default `--baseline` the batches do not have, +exits 2 with a `✗` line naming the file (and the rule, where the error is the rule's). +A rule whose own `baseline` or `arm` has no cell among its cases is `not measured` for +that rule, and the command still exits 0. The ordinary comparison's flags (`--arm`, `--task`, `--metric`, ...) do not change the ledger, which uses the rules' own cases and arms. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index de3d28dc3..187bd4f4b 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -367,7 +367,7 @@ def cmd_repo_compare(args: argparse.Namespace) -> None: try: summary = compare(dirs, args.baseline, arms, args.model, tuple(args.metric or ()), pass_k=args.pass_k, path_stability=args.path_stability, tasks=args.task) - if args.ledger: + if args.ledger is not None: summary["ledger"] = evaluate(load_ledger(Path(args.ledger)), dirs, args.baseline, args.model, ledger_name=args.ledger) except (CompareError, CellFileError, LedgerError) as exc: # an unusable baseline, arm, metric, cell or ledger diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index 2f54c70eb..b58659d8f 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -133,25 +133,44 @@ def _rule_entries(rule: Rule, batch_dirs: list[Path], default_baseline: str, mod head = _head(rule, base, len(rule.cases), missing) if not present: return [{**head, "error": "no linked case has a finished cell"}] + entries: list[dict[str, Any]] = [] + # One comparison per --model: a model with no cell among this rule's cases fails only its own comparison. + for model in models or [None]: + entries += _compared(rule, head, batch_dirs, base, [model] if model else None, present, seed, warnings) + per_model = Counter(e["model"] for e in entries if not e["error"] and e["measure"] in PRIMARY) + for entry in entries: + entry["mixed"] = not entry["error"] and per_model[entry["model"]] > 1 + return entries + + +def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: str, models: list[str] | None, + present: list[str], seed: int, warnings: list[str]) -> list[dict[str, Any]]: + """The rule's results of one compare() call, with its linked cases that did not pair, as warnings.""" try: summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present) except CompareError as exc: # the arm or the baseline has no cell among this rule's cases - return [{**head, "error": str(exc)}] + return [{**head, "model": models[0] if models else None, "error": str(exc)}] warnings.extend(summary["warnings"]) + linked = [u for u in summary["unpaired"] if u["task"] in present] rows = [r for r in summary["comparisons"] if r["measure"] in PRIMARY] if not rows: - reasons = sorted({u["reason"] for u in summary["unpaired"] if u["task"] in present}) + reasons = sorted({u["reason"] for u in linked}) return [{**head, "error": "; ".join(reasons) or "no linked case has cells in both arms"}] - per_model = Counter(r["model"] for r in rows) - return [_entry(head, r, per_model[r["model"]] > 1) for r in rows] + out = [_entry(head, r) for r in rows] + for entry in out: + own = [u for u in linked if u["model"] == entry["model"]] + if own: + entry["warnings"].append(f"{len({u['task'] for u in own})} linked case(s) not paired: " + + "; ".join(sorted({u["reason"] for u in own}))) + return out -def _entry(head: dict[str, Any], row: dict[str, Any], mixed: bool) -> dict[str, Any]: +def _entry(head: dict[str, Any], row: dict[str, Any]) -> dict[str, Any]: d = row["delta"] out = {**head, "model": row["model"], "measure": row["measure"], "paired": row["n_tasks"], "baseline_mean": row["baseline_mean"], "treatment_mean": row["treatment_mean"], "delta": {"mean": d["mean"], "lo": d["lo"], "hi": d["hi"]}, "up": row["up"], "down": row["down"], - "tied": row["tied"], "p": row["p"], "headroom": row["headroom"], "mixed": mixed, + "tied": row["tied"], "p": row["p"], "headroom": row["headroom"], "mixed": False, "changed": row["up"] + row["down"], "warnings": []} if row["headroom"]: out["warnings"].append(f"{row['measure']} headroom: {row['headroom']} (the cases cannot show an effect)") diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index c0d480149..7e0570c14 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -282,3 +282,33 @@ def test_a_bad_ledger_exits_with_usage_error(tmp_path, capsys): args.func(args) assert exc.value.code == 2 assert "R1" in capsys.readouterr().err + + +def test_a_model_with_no_cell_on_the_rule_does_not_hide_the_others(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "elsewhere", BASE, success=True, model="m-2") # m-2 ran a task this rule does not link + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + models=["m-1", "m-2"], today=TODAY) + assert _verdicts(result) == {("R1", "m-1"): KEEP, ("R1", "m-2"): INSUFFICIENT} + (other,) = [e for e in result["rules"] if e["model"] == "m-2"] + assert "no compared cells for --model m-2" in other["reason"] + + +def test_a_linked_case_that_does_not_pair_is_named_on_the_rule(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "t_8", BASE, success=False) # the baseline's kind is gate, the rule's arm's is score + _cell(batch, "t_8", ARM, score=0.9) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks + ["t_8"])])), [batch], BASE, + today=TODAY)["rules"][0] + assert (entry["paired"], entry["linked"]) == (8, 9) + assert any("1 linked case(s) not paired" in w for w in entry["warnings"]) + + +def test_an_empty_ledger_path_is_refused_not_ignored(tmp_path, capsys): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", ""]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 From bc7b6437ec54f5ddeee0efd0cb394c0d573001ff Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 14:16:05 -0700 Subject: [PATCH 03/17] fix(repo): ledger names an unpaired model, dates a future review, warns once A model whose linked cases do not pair now gets its own not-measured result when other models are measured. The result of a --model with no cell names that model. A review dated after today is stale with a date warning, not silently fresh. Each result keeps its own warnings list, so a stale review is warned once per result. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/ledger.py | 18 ++++++++++----- dokimi/tests/test_repo_ledger.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index b58659d8f..3b3e7dba9 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -101,12 +101,16 @@ def evaluate(ledger: Ledger, batch_dirs: list[Path], baseline: str, models: list for rule in ledger.rules: rule_entries = _rule_entries(rule, batch_dirs, baseline, models, known, seed, warnings) age = (today - rule.reviewed).days - stale = age > ledger.review_within_days + stale = age < 0 or age > ledger.review_within_days + if age < 0: + note = f"review date {rule.reviewed.isoformat()} is after {today.isoformat()}: check the date" + else: + note = (f"review date {rule.reviewed.isoformat()} is {age} days old (policy: " + f"{ledger.review_within_days}): re-review the rule") for entry in rule_entries: entry.update(reviewed=rule.reviewed.isoformat(), days_since_review=age, stale=stale) - if stale: - entry["warnings"].append(f"review date {rule.reviewed.isoformat()} is {age} days old (policy: " - f"{ledger.review_within_days}): re-review the rule") + if stale: # a new list: error entries of one rule share their head's warnings list + entry["warnings"] = [*entry["warnings"], note] entries += rule_entries _holm(entries) for entry in entries: @@ -150,13 +154,17 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present) except CompareError as exc: # the arm or the baseline has no cell among this rule's cases return [{**head, "model": models[0] if models else None, "error": str(exc)}] + model = models[0] if models else None warnings.extend(summary["warnings"]) linked = [u for u in summary["unpaired"] if u["task"] in present] rows = [r for r in summary["comparisons"] if r["measure"] in PRIMARY] if not rows: reasons = sorted({u["reason"] for u in linked}) - return [{**head, "error": "; ".join(reasons) or "no linked case has cells in both arms"}] + return [{**head, "model": model, "error": "; ".join(reasons) or "no linked case has cells in both arms"}] out = [_entry(head, r) for r in rows] + for unpaired in sorted({u["model"] for u in linked} - {r["model"] for r in rows}): # not measured, and named + reasons = sorted({u["reason"] for u in linked if u["model"] == unpaired}) + out.append({**head, "model": unpaired, "error": "; ".join(reasons)}) for entry in out: own = [u for u in linked if u["model"] == entry["model"]] if own: diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index 7e0570c14..442f80f0b 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -312,3 +312,41 @@ def test_an_empty_ledger_path_is_refused_not_ignored(tmp_path, capsys): with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 2 + + +def test_a_model_whose_linked_cases_do_not_pair_is_named_when_others_pair(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + for task in tasks: # m-2 has a baseline cell on each linked case, and no arm cell + _cell(batch, task, BASE, success=True, model="m-2") + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY) + assert _verdicts(result) == {("R1", "m-1"): KEEP, ("R1", "m-2"): INSUFFICIENT} + + +def test_a_model_error_names_the_model(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}) + _cell(batch, "elsewhere", BASE, success=True, model="m-2") + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("t", 8))])), [batch], BASE, + models=["m-2"], today=TODAY)["rules"][0] + assert entry["model"] == "m-2" + assert entry["verdict"] == INSUFFICIENT + + +def test_a_stale_review_is_warned_once_per_result(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}) + rule = _rule("R1", _ids("t", 8), reviewed="2026-01-01") + results = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, models=["m-2", "m-3"], + today=TODAY)["rules"] + assert len(results) == 2 + for entry in results: + assert sum("re-review" in w for w in entry["warnings"]) == 1 + + +def test_a_review_dated_after_today_is_stale_with_a_date_warning(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks, reviewed="2036-10-01")])), + [batch], BASE, today=TODAY)["rules"][0] + assert entry["stale"] is True + assert any("after 2026-10-31" in w for w in entry["warnings"]) From 6975ab22e01f46ab63616f131db751266993eee0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 14:23:05 -0700 Subject: [PATCH 04/17] fix(repo): ledger keeps warnings per model and compares a repeated --model once A not-measured model no longer carries the not-paired warning of another model, and a model given twice is compared once, not counted as mixed. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/ledger.py | 8 ++++---- dokimi/tests/test_repo_ledger.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index 3b3e7dba9..25c33657c 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -139,7 +139,7 @@ def _rule_entries(rule: Rule, batch_dirs: list[Path], default_baseline: str, mod return [{**head, "error": "no linked case has a finished cell"}] entries: list[dict[str, Any]] = [] # One comparison per --model: a model with no cell among this rule's cases fails only its own comparison. - for model in models or [None]: + for model in dict.fromkeys(models) if models else [None]: # a repeated --model is one comparison entries += _compared(rule, head, batch_dirs, base, [model] if model else None, present, seed, warnings) per_model = Counter(e["model"] for e in entries if not e["error"] and e["measure"] in PRIMARY) for entry in entries: @@ -162,14 +162,14 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st reasons = sorted({u["reason"] for u in linked}) return [{**head, "model": model, "error": "; ".join(reasons) or "no linked case has cells in both arms"}] out = [_entry(head, r) for r in rows] - for unpaired in sorted({u["model"] for u in linked} - {r["model"] for r in rows}): # not measured, and named - reasons = sorted({u["reason"] for u in linked if u["model"] == unpaired}) - out.append({**head, "model": unpaired, "error": "; ".join(reasons)}) for entry in out: own = [u for u in linked if u["model"] == entry["model"]] if own: entry["warnings"].append(f"{len({u['task'] for u in own})} linked case(s) not paired: " + "; ".join(sorted({u["reason"] for u in own}))) + for unpaired in sorted({u["model"] for u in linked} - {r["model"] for r in rows}): # not measured, and named + reasons = sorted({u["reason"] for u in linked if u["model"] == unpaired}) + out.append({**head, "model": unpaired, "error": "; ".join(reasons)}) return out diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index 442f80f0b..cf9729ba7 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -350,3 +350,22 @@ def test_a_review_dated_after_today_is_stale_with_a_date_warning(tmp_path): [batch], BASE, today=TODAY)["rules"][0] assert entry["stale"] is True assert any("after 2026-10-31" in w for w in entry["warnings"]) + + +def test_a_not_measured_model_does_not_carry_another_models_warning(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "t_0", BASE, success=True, model="m-2") # m-2 pairs nothing on this rule: its baseline only + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY) + (other,) = [e for e in result["rules"] if e["model"] == "m-2"] + assert other["verdict"] == INSUFFICIENT + assert not any("not paired" in w for w in other["warnings"]) + + +def test_a_repeated_model_is_compared_once(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + models=["m-1", "m-1"], today=TODAY) + assert [(e["model"], e["verdict"]) for e in result["rules"]] == [("m-1", KEEP)] From 54b9992acbdc4df2d762507b3d2ae57f5f982f60 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 17:57:35 -0700 Subject: [PATCH 05/17] fix(repo): ledger keeps mixed-kind rules out of the Holm family A rule whose linked cases mix gate and score tasks has no single test, so its p value no longer takes a slot in the Holm family of the other rules. Such a result is insufficient-evidence with that reason. Signed-off-by: Nikolay Petrov --- dokimi/docs/rule-ledger.md | 6 ++++-- dokimi/dokimi/repos/ledger.py | 14 ++++++++++---- dokimi/tests/test_repo_ledger.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index 0ac8198de..2617b130e 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -98,8 +98,10 @@ Schema cannot). For each rule and model, `repo compare --ledger` runs the ordinary paired comparison of the rule's arm against its baseline over the rule's linked cases (`repo compare` itself, so the pairing, headroom and provenance warnings are -the same). The p values of all rule results are Holm-corrected together. The -rules' family is separate from the family of the comparison's own rows. +the same). The p values of the rule results are Holm-corrected together. A result whose +linked cases mix gate and score tasks has no single test, so it takes no slot in +that family and is `insufficient-evidence`. The rules' family is separate from the +family of the comparison's own rows. | Verdict | When | Meaning | |---|---|---| diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index 25c33657c..e64424d8c 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -116,7 +116,7 @@ def evaluate(ledger: Ledger, batch_dirs: list[Path], baseline: str, models: list for entry in entries: entry["verdict"], entry["reason"] = _verdict(entry, ledger.alpha) return {"ledger": ledger_name, "alpha": ledger.alpha, "review_within_days": ledger.review_within_days, - "today": today.isoformat(), "family": sum(e["p"] is not None for e in entries), + "today": today.isoformat(), "family": sum(_in_family(e) for e in entries), "rules": entries, "warnings": list(dict.fromkeys(warnings))} @@ -185,9 +185,14 @@ def _entry(head: dict[str, Any], row: dict[str, Any]) -> dict[str, Any]: return out +def _in_family(entry: dict[str, Any]) -> bool: + """A result has a p and is not mixed: a gate-and-score rule has no single test, so it takes no slot.""" + return entry["p"] is not None and not entry["mixed"] + + def _holm(entries: list[dict[str, Any]]) -> None: - """``p_holm`` over every result that has a p, as one family (the exact adjusted value).""" - scored = [e for e in entries if e["p"] is not None] + """``p_holm`` over every result in the family (the exact adjusted value); the others get none.""" + scored = [e for e in entries if _in_family(e)] for entry, adjusted in zip(scored, adjust_pvalues([e["p"] for e in scored], rounded=False), strict=True): entry["p_holm"] = adjusted @@ -197,7 +202,8 @@ def _verdict(entry: dict[str, Any], alpha: float) -> tuple[str, str]: if entry["error"] or not entry["paired"]: return INSUFFICIENT, f"not measured: {entry['error'] or 'no linked case has cells in both arms'}" if entry["mixed"]: - return INSUFFICIENT, "the linked cases mix gate and score tasks: link one kind of task per rule" + return INSUFFICIENT, ("the linked cases mix gate and score tasks: link one kind of task per rule " + "(not in the Holm family)") if entry["p_holm"] is None: return INSUFFICIENT, "every linked case ties: there is no difference to test" p, mean = entry["p_holm"], entry["delta"]["mean"] diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index cf9729ba7..8c23a4ee6 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -369,3 +369,21 @@ def test_a_repeated_model_is_compared_once(tmp_path): result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, models=["m-1", "m-1"], today=TODAY) assert [(e["model"], e["verdict"]) for e in result["rules"]] == [("m-1", KEEP)] + + +def test_a_mixed_rule_takes_no_slot_in_the_holm_family(tmp_path): + # A mixes gate and score (both improve); B is six gate cases, all improved. Alone B reaches p = 2/64. + gate_a, score_a, gate_b = _ids("ga", 6), _ids("sa", 6), _ids("gb", 6) + batch = _gate_batch(tmp_path, {t: (False, True) for t in gate_a + gate_b}) + for task in score_a: + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=0.9) + rules = [_rule("A", gate_a + score_a), _rule("B", gate_b)] + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", rules)), [batch], BASE, today=TODAY) + verdicts = {e["rule"]: e for e in result["rules"] if e["rule"] == "B"} + assert result["family"] == 1 + assert verdicts["B"]["verdict"] == KEEP + assert verdicts["B"]["p_holm"] == pytest.approx(2 / 2**6, abs=1e-5) + mixed = [e for e in result["rules"] if e["rule"] == "A"] + assert mixed and all(e["verdict"] == INSUFFICIENT and e["p_holm"] is None and "Holm family" in e["reason"] + for e in mixed) From e8e42b364f25096a3f5fb7f6aad806863a5d3ec1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:39:17 -0700 Subject: [PATCH 06/17] fix(repo): ledger refuses a NaN alpha json.loads reads NaN, and the schema bound does not reject it. NaN compares false with every p value, so every rule came out insufficient-evidence with a reason that names the alpha. Refuse it at load time. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/ledger.py | 6 +++++- dokimi/tests/test_repo_ledger.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index e64424d8c..f91083554 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +import math from collections import Counter from dataclasses import dataclass from datetime import UTC, date, datetime @@ -75,7 +76,10 @@ def load_ledger(path: Path) -> Ledger: seen.add(rule.id) rules.append(rule) policy = raw.get("policy", {}) - return Ledger(tuple(rules), float(policy.get("alpha", ALPHA)), + alpha = float(policy.get("alpha", ALPHA)) + if not math.isfinite(alpha): # json.loads reads NaN; it would make every verdict insufficient-evidence + raise LedgerError(f"{path}: policy.alpha must be a finite number, not {alpha}") + return Ledger(tuple(rules), alpha, int(policy.get("review_within_days", DEFAULT_REVIEW_DAYS))) diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index 8c23a4ee6..36574b5fd 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -387,3 +387,14 @@ def test_a_mixed_rule_takes_no_slot_in_the_holm_family(tmp_path): mixed = [e for e in result["rules"] if e["rule"] == "A"] assert mixed and all(e["verdict"] == INSUFFICIENT and e["p_holm"] is None and "Holm family" in e["reason"] for e in mixed) + + +@pytest.mark.parametrize("alpha", ["NaN", "Infinity"]) +def test_a_non_finite_alpha_is_refused(tmp_path, alpha): + # json.loads accepts NaN and Infinity; NaN compares false with every p, so it would turn any rule insufficient + path = tmp_path / "ledger.json" + path.write_text('{"schema_version":"repo_ledger.v1","policy":{"alpha":%s},"rules":[{"id":"R1",' + '"hypothesis":"h","arm":"guidance:rules","cases":["t1"],"reviewed":"2026-10-30"}]}' % alpha, + encoding="utf-8") + with pytest.raises(LedgerError, match="alpha"): + load_ledger(path) From ef75b275c9dcd4868889e3448ad9b3fb207a81c1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:57:08 -0700 Subject: [PATCH 07/17] fix(repo): ledger alpha is strictly between 0 and 1 An alpha of 1 passed the schema (inclusive maximum), so any non-zero delta with two changed cases came out keep. Make the bound exclusive and test the edges: 0, 1, 2, -0.5, NaN and both infinities. Signed-off-by: Nikolay Petrov --- dokimi/docs/rule-ledger.md | 2 +- dokimi/dokimi/schemas/repo_ledger.v1.json | 2 +- dokimi/tests/test_repo_ledger.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index 2617b130e..4abc62462 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -90,7 +90,7 @@ Schema cannot). | `baseline` | Optional. Overrides the `--baseline` of the compare for this rule. | | `cases` | Task ids, at least one, unique. Ids with no finished cell are listed as missing. | | `reviewed` | `YYYY-MM-DD`, the last date a person confirmed the rule. | -| `policy.alpha` | Default 0.05. | +| `policy.alpha` | Default 0.05; a number strictly between 0 and 1. | | `policy.review_within_days` | Default 180. An older `reviewed` gets a warning. | ## Verdicts diff --git a/dokimi/dokimi/schemas/repo_ledger.v1.json b/dokimi/dokimi/schemas/repo_ledger.v1.json index 7968cc41f..99cb6e3af 100644 --- a/dokimi/dokimi/schemas/repo_ledger.v1.json +++ b/dokimi/dokimi/schemas/repo_ledger.v1.json @@ -17,7 +17,7 @@ "description": "Optional. alpha: the Holm-adjusted p below which a delta counts. review_within_days: a review older than this is flagged as stale.", "additionalProperties": false, "properties": { - "alpha": {"type": "number", "exclusiveMinimum": 0, "maximum": 1}, + "alpha": {"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1}, "review_within_days": {"type": "integer", "minimum": 1} } }, diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index 36574b5fd..8dad4a718 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -389,8 +389,8 @@ def test_a_mixed_rule_takes_no_slot_in_the_holm_family(tmp_path): for e in mixed) -@pytest.mark.parametrize("alpha", ["NaN", "Infinity"]) -def test_a_non_finite_alpha_is_refused(tmp_path, alpha): +@pytest.mark.parametrize("alpha", ["NaN", "Infinity", "-Infinity", "0", "1", "2", "-0.5"]) +def test_an_alpha_outside_the_open_interval_zero_one_is_refused(tmp_path, alpha): # json.loads accepts NaN and Infinity; NaN compares false with every p, so it would turn any rule insufficient path = tmp_path / "ledger.json" path.write_text('{"schema_version":"repo_ledger.v1","policy":{"alpha":%s},"rules":[{"id":"R1",' From cf3c65919dccb750d86119ed2cba24116e0552e7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:33:36 -0700 Subject: [PATCH 08/17] docs(ledger): a rule's example arm is one repo run can produce; --model applies The example used guidance:rules, which workspace._parse_guidance rejects, so a copied ledger gave "not measured". It now uses guidance:raw. The compare section says which flags reach the ledger: the ordinary comparison runs first (a --task or --arm it rejects exits 2), its filters do not change the ledger once it succeeds, and --model does apply, changing the Holm family. Thread: PRRT_kwDOVBg7g86yWhr1 Thread: PRRT_kwDOVBg7g86yWhr3 Signed-off-by: Nikolay Petrov --- dokimi/docs/rule-ledger.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index 4abc62462..f7a2f13e6 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -72,7 +72,7 @@ Schema cannot). "rule": "Revert edits to files the change does not need.", "source": "AGENTS.md#commits-and-pull-requests", "hypothesis": "With the rule, agents stop touching unrelated files on the replayed PRs.", - "arm": "guidance:rules", + "arm": "guidance:raw", "baseline": "guidance:none", "cases": ["review_replay_pr3580", "review_replay_pr3659", "fix_df_extratrees_reg"], "reviewed": "2026-10-01" @@ -124,9 +124,11 @@ The text output keeps the ordinary comparison and adds a `rule ledger` section. A ledger that does not validate, or a default `--baseline` the batches do not have, exits 2 with a `✗` line naming the file (and the rule, where the error is the rule's). A rule whose own `baseline` or `arm` has no cell among its cases is `not measured` for -that rule, and the command still exits 0. The -ordinary comparison's flags (`--arm`, `--task`, `--metric`, ...) do not change the -ledger, which uses the rules' own cases and arms. +that rule, and the command still exits 0. The ordinary comparison runs first, so a +`--task` or `--arm` it rejects (no cell matches) exits 2 before any ledger is evaluated. +Once it succeeds, its `--arm`, `--task` and `--metric` filters do not change the ledger, +which uses the rules' own cases and arms. `--model` does apply: it limits the ledger to +those models, which changes its Holm family and so its verdicts. ## Limits From b1af2947317f9257cc6dc9de937b554c6af4937b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:33:36 -0700 Subject: [PATCH 09/17] fix(ledger): verdicts read the unrounded p and mean; models keep their identity compare's sign-flip p was rounded to six places and the paired mean was taken from a bootstrap rounded to four, so a verdict could flip across alpha or read a tie on a small real change. Both are now unrounded; display rounds them. The ledger now counts only linked cases whose outcome (gate or score, or the whole task) did not pair, so a missing cost or turn count no longer warns. When no model pairs, each model that has an unpaired linked case gets its own insufficient-evidence result, and a rule with no finished case names every requested model. Score results say "mean score", not "pass rate". Regression tests cover the alpha boundary, small score deltas both ways, per-model unpaired and missing-case results, a missing cost, and the score label. The new tests fail on the previous code. Thread: PRRT_kwDOVBg7g86yWhr5 Thread: PRRT_kwDOVBg7g86yWhr7 Thread: PRRT_kwDOVBg7g86yWhr- Thread: PRRT_kwDOVBg7g86yWhsC Thread: PRRT_kwDOVBg7g86yWhsF Thread: PRRT_kwDOVBg7g86yWh0W Thread: PRRT_kwDOVBg7g86yWh0m Thread: PRRT_kwDOVBg7g86yWh08 Thread: PRRT_kwDOVBg7g86yWh1J Thread: PRRT_kwDOVBg7g86yWh1S Thread: PRRT_kwDOVBg7g86yWh1Z Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 6 +-- dokimi/dokimi/repos/ledger.py | 28 +++++++++---- dokimi/tests/test_repo_ledger.py | 67 +++++++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 308370da3..1ea8c9d29 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -537,7 +537,7 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), "treatment_mean": _round(_mean([p["treatment"] for p in pairs])), - "delta": {"mean": ci["mean"], "lo": lo, "hi": hi}, + "delta": {"mean": sum(deltas) / len(deltas), "lo": lo, "hi": hi}, # unrounded: a verdict reads its sign "p": p, "p_holm": None, "p_holm_floor": None, @@ -569,10 +569,10 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL if len(d) <= EXACT_MAX: hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed for signs in itertools.product((1, -1), repeat=len(d))) - return round(hits / 2 ** len(d), 6) + return hits / 2 ** len(d) rng = random.Random(seed) hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) - return round((hits + 1) / (n_resamples + 1), 6) + return (hits + 1) / (n_resamples + 1) def format_compare(summary: dict[str, Any]) -> str: diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index f91083554..74513c555 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -139,11 +139,12 @@ def _rule_entries(rule: Rule, batch_dirs: list[Path], default_baseline: str, mod present = [c for c in rule.cases if c in known] base = rule.baseline or default_baseline head = _head(rule, base, len(rule.cases), missing) - if not present: - return [{**head, "error": "no linked case has a finished cell"}] + requested = list(dict.fromkeys(models)) if models else [None] # a repeated --model is one comparison + if not present: # the requested models stay named, one result each + return [{**head, "model": model, "error": "no linked case has a finished cell"} for model in requested] entries: list[dict[str, Any]] = [] # One comparison per --model: a model with no cell among this rule's cases fails only its own comparison. - for model in dict.fromkeys(models) if models else [None]: # a repeated --model is one comparison + for model in requested: entries += _compared(rule, head, batch_dirs, base, [model] if model else None, present, seed, warnings) per_model = Counter(e["model"] for e in entries if not e["error"] and e["measure"] in PRIMARY) for entry in entries: @@ -160,11 +161,12 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st return [{**head, "model": models[0] if models else None, "error": str(exc)}] model = models[0] if models else None warnings.extend(summary["warnings"]) - linked = [u for u in summary["unpaired"] if u["task"] in present] + # Only a linked case whose outcome (the gate or score, or the whole task) did not pair counts: a missing + # cost or turn count is metadata, and does not weaken the verdict. + linked = [u for u in summary["unpaired"] if u["task"] in present and u["measure"] in (None, *PRIMARY)] rows = [r for r in summary["comparisons"] if r["measure"] in PRIMARY] if not rows: - reasons = sorted({u["reason"] for u in linked}) - return [{**head, "model": model, "error": "; ".join(reasons) or "no linked case has cells in both arms"}] + return _unmeasured(head, linked, model) out = [_entry(head, r) for r in rows] for entry in out: own = [u for u in linked if u["model"] == entry["model"]] @@ -177,6 +179,17 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st return out +def _unmeasured(head: dict[str, Any], linked: list[dict[str, Any]], model: str | None) -> list[dict[str, Any]]: + """The results of a comparison with no gate or score row: one per model its unpaired linked cases name, + so that models with no pair keep their identity, each with its own reason.""" + named = list(dict.fromkeys(u["model"] for u in linked)) or [model] + out = [] + for name in named: + reasons = sorted({u["reason"] for u in linked if u["model"] == name}) + out.append({**head, "model": name, "error": "; ".join(reasons) or "no linked case has cells in both arms"}) + return out + + def _entry(head: dict[str, Any], row: dict[str, Any]) -> dict[str, Any]: d = row["delta"] out = {**head, "model": row["model"], "measure": row["measure"], "paired": row["n_tasks"], @@ -232,7 +245,8 @@ def format_ledger(result: dict[str, Any]) -> str: if e["paired"]: d = e["delta"] ci = "-" if d["lo"] is None else f"[{d['lo']:+.3f}, {d['hi']:+.3f}]" - lines.append(f" paired {e['paired']} of {e['linked']} linked: pass rate {_num(e['baseline_mean'])} -> " + label = "mean score" if e["measure"] == "score" else "pass rate" # a score is not a success rate + lines.append(f" paired {e['paired']} of {e['linked']} linked: {label} {_num(e['baseline_mean'])} -> " f"{_num(e['treatment_mean'])}, delta {d['mean']:+.3f} {ci}, p {_p(e['p'])}, " f"p_holm {_p(e['p_holm'])}, up/down/tied {e['up']}/{e['down']}/{e['tied']}") lines.append(f" {e['reason']}") diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index 8dad4a718..ee736aba1 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -111,7 +111,7 @@ def test_all_cases_improved_is_keep_and_all_worse_is_review(tmp_path): up = result["rules"][0] assert (up["up"], up["down"], up["paired"]) == (8, 0, 8) assert (up["baseline_mean"], up["treatment_mean"]) == (0.0, 1.0) - assert up["p_holm"] == pytest.approx(2 * 2 / 2**8, abs=1e-5) # Holm over the two rules; p is rounded to 6 places + assert up["p_holm"] == pytest.approx(2 * 2 / 2**8, abs=1e-5) # Holm over the two rules; p is exact def test_few_changed_cases_are_insufficient_evidence_and_the_reason_says_why(tmp_path): @@ -398,3 +398,68 @@ def test_an_alpha_outside_the_open_interval_zero_one_is_refused(tmp_path, alpha) encoding="utf-8") with pytest.raises(LedgerError, match="alpha"): load_ledger(path) + + +def test_a_p_on_the_alpha_boundary_is_not_rounded_across_it(tmp_path): + # eight uniformly improved gate cases: the exact sign-flip p is 2 / 2**8 = 0.0078125, not 0.007812 + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + rules = [_rule("R1", tasks)] + below = evaluate(load_ledger(_ledger(tmp_path / "below.json", rules, alpha=0.0078124)), [batch], BASE, today=TODAY) + above = evaluate(load_ledger(_ledger(tmp_path / "above.json", rules, alpha=0.0078126)), [batch], BASE, today=TODAY) + assert _verdicts(below) == {("R1", "m-1"): INSUFFICIENT} + assert _verdicts(above) == {("R1", "m-1"): KEEP} + assert below["rules"][0]["p_holm"] == pytest.approx(2 / 2**8, abs=1e-12) + + +@pytest.mark.parametrize(("arm_score", "verdict"), [(0.20001, KEEP), (0.19999, REVIEW)]) +def test_a_small_score_change_keeps_its_sign_in_the_verdict(tmp_path, arm_score, verdict): + # the mean delta is 1e-5: rounded to four places it was 0.0, and the verdict read a tie + batch = tmp_path / "run" + for task in _ids("s", 8): + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=arm_score) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("s", 8))])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == verdict + assert entry["delta"]["mean"] == pytest.approx(arm_score - 0.2, abs=1e-12) + + +def test_a_score_rule_is_reported_as_a_mean_score_not_a_pass_rate(tmp_path): + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("s", 8))])), + [_score_batch(tmp_path)], BASE, today=TODAY) + text = format_ledger(result) + assert "mean score 0.200 -> 0.900" in text + assert "pass rate" not in text + + +def test_every_model_unpaired_on_the_rule_gets_its_own_result(tmp_path): + batch = tmp_path / "run" + _cell(batch, "t1", BASE, success=True, model="m-1") # the baseline ran t1 only on m-1 + _cell(batch, "t1", ARM, success=True, model="m-2") # the arm ran it only on m-2 + entries = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"])])), [batch], BASE, + today=TODAY)["rules"] + assert sorted(e["model"] for e in entries) == ["m-1", "m-2"] + assert all(e["verdict"] == INSUFFICIENT and e["error"] for e in entries) + + +def test_a_rule_with_no_finished_case_names_each_requested_model(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}) + entries = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["ghost"])])), [batch], BASE, + models=["m-1", "m-2", "m-1"], today=TODAY)["rules"] + assert [e["model"] for e in entries] == ["m-1", "m-2"] + assert all(e["verdict"] == INSUFFICIENT and "no linked case has a finished cell" in e["reason"] for e in entries) + + +def test_a_missing_cost_does_not_unpair_a_gate_case(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + meta = batch / f"{tasks[0]}__{BASE.replace(':', '-')}__m-1__r1" / "result.json" # cost gap, outcome paired + rec = json.loads(meta.read_text(encoding="utf-8")) + rec["agent"]["cost_usd"] = None + meta.write_text(json.dumps(rec), encoding="utf-8") + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == KEEP + assert entry["paired"] == 8 + assert not any("not paired" in w for w in entry["warnings"]) From eb2aa0ec65ae582c1f22fd50f0617cd3a3bf4bd9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:59:06 -0700 Subject: [PATCH 10/17] fix(ledger): exact statistics are opt-in for the ledger; a rule id ending in a newline is refused The ledger's verdicts read unrounded p and mean, but the change removed the rounding from compare's own output too, so plain `repo compare --json` changed for every user (delta.mean 0.1778 became 0.17777..., p gained digits). compare, _compare_model, _compare_measure, paired_summary and sign_flip_p now take exact=False by default, and only the ledger passes exact=True. The rule id pattern ends in `$`, which Python's re also matches before a trailing newline, so "R1\n" loaded as a rule distinct from "R1" and both printed as R1. load_ledger now checks the id with fullmatch. Red-then-green tests: test_the_printed_statistics_stay_rounded_and_full_precision_is_opt_in, test_a_rule_id_with_a_trailing_newline_is_refused_not_a_second_R1. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 35 ++++++++++++++++++------------- dokimi/dokimi/repos/ledger.py | 6 +++++- dokimi/tests/test_repo_compare.py | 14 ++++++++++++- dokimi/tests/test_repo_ledger.py | 7 +++++++ 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 1ea8c9d29..3e7dcccdb 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -174,7 +174,9 @@ def batch_stamp(batch_dir: Path) -> dict[str, str]: def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None = None, models: list[str] | None = None, metrics: tuple[str, ...] = (), seed: int = 0, - pass_k: int | None = None, path_stability: bool = False, tasks: list[str] | None = None) -> dict[str, Any]: + pass_k: int | None = None, path_stability: bool = False, tasks: list[str] | None = None, + *, exact: bool = False) -> dict[str, Any]: + """``exact``: the statistics at full precision (the ledger's verdicts read them); otherwise rounded as printed.""" metrics = tuple(dict.fromkeys(metrics)) if pass_k is not None and pass_k < 1: raise CompareError(f"--pass-k must be at least 1, got {pass_k}") @@ -204,7 +206,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None unpaired += _one_arm_models(text, base_vals, treat_vals, models) shared_models = sorted({m for m, _ in base_vals} & {m for m, _ in treat_vals}) for model in [m for m in shared_models if not models or m in models]: - skipped = _compare_model(text, model, base_vals, treat_vals, measures, seed, rows, unpaired) + skipped = _compare_model(text, model, base_vals, treat_vals, measures, seed, rows, unpaired, exact=exact) if skipped: one_arm.append(f"{text} {model}: only one arm reports {', '.join(skipped)} for any shared task; " "not compared") @@ -359,7 +361,7 @@ def _holm(rows: list[dict[str, Any]]) -> None: def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], measures: list[str], seed: int, - rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> list[str]: + rows: list[dict[str, Any]], unpaired: list[dict[str, Any]], *, exact: bool = False) -> list[str]: """Append one row per measure of one (treatment, model), and the tasks only one arm has. Returns the :data:`REPORTED` measures only one arm reports for any shared task: they are skipped, not unpaired. @@ -383,7 +385,7 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], if any(has): skipped.append(measure) continue - row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired) + row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired, exact=exact) if row is not None: rows.append(row) return skipped @@ -421,7 +423,7 @@ def _measure_pairs(treatment: str, model: str, shared: list[str], base_vals: dic def _compare_measure(treatment: str, model: str, measure: str, shared: list[str], base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], seed: int, - unpaired: list[dict[str, Any]]) -> dict[str, Any] | None: + unpaired: list[dict[str, Any]], *, exact: bool = False) -> dict[str, Any] | None: """The row of one measure over the shared tasks of one (treatment, model), or None with no pair.""" pairs = _measure_pairs(treatment, model, shared, base_vals, treat_vals, measure, unpaired) if not pairs: @@ -433,7 +435,7 @@ def _compare_measure(treatment: str, model: str, measure: str, shared: list[str] or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs] flag = headroom(pairs, tops, failed) return {"treatment": treatment, "model": model, "measure": measure, - **paired_summary(pairs, measure, seed=seed), "headroom": flag} + **paired_summary(pairs, measure, seed=seed, exact=exact), "headroom": flag} def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]], @@ -516,7 +518,8 @@ def _pair(task: str, base: dict[str, Any], treat: dict[str, Any], measure: str) "n_baseline": base["n"][measure], "n_treatment": treat["n"][measure]} -def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int = 0) -> dict[str, Any]: +def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int = 0, *, + exact: bool = False) -> dict[str, Any]: """Statistics over per-task pairs ``{task, baseline, treatment, delta, ...}``. The values are taken at full precision and rounded only in the output: rounded rates such as 1/3 and 2/3 @@ -530,14 +533,14 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int deltas = [p["delta"] for p in pairs] ci = (bootstrap_ci if measure in PRIMARY else mean_ci)(deltas, seed=seed) lo, hi = (float(ci["lo"]), float(ci["hi"])) if len(deltas) > 1 else (None, None) # mean_ci gives numpy floats - p = sign_flip_p(deltas, seed=seed) + p = sign_flip_p(deltas, seed=seed, exact=exact) sd = statistics.stdev(deltas) if len(deltas) > 1 else 0.0 reachable = min_sign_flip_p(sum(abs(d) > EPS for d in deltas)) < ALPHA # else no effect can be detected return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), "treatment_mean": _round(_mean([p["treatment"] for p in pairs])), - "delta": {"mean": sum(deltas) / len(deltas), "lo": lo, "hi": hi}, # unrounded: a verdict reads its sign + "delta": {"mean": sum(deltas) / len(deltas) if exact else ci["mean"], "lo": lo, "hi": hi}, "p": p, "p_holm": None, "p_holm_floor": None, @@ -557,10 +560,12 @@ def min_sign_flip_p(changed: int) -> float: return min(1.0, 2 / 2 ** changed) if changed <= EXACT_MAX else 1 / (N_RESAMPLES + 1) -def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES) -> float | None: +def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, + exact: bool = False) -> float | None: """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. ``None`` when every delta is a tie: there is no evidence either way, and 1.0 would read as a measured null. + Rounded to 6 places as printed, unless ``exact``. """ d = [x for x in deltas if abs(x) > EPS] if not d: @@ -569,10 +574,12 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL if len(d) <= EXACT_MAX: hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed for signs in itertools.product((1, -1), repeat=len(d))) - return hits / 2 ** len(d) - rng = random.Random(seed) - hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) - return (hits + 1) / (n_resamples + 1) + p = hits / 2 ** len(d) + else: + rng = random.Random(seed) + hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) + p = (hits + 1) / (n_resamples + 1) + return p if exact else round(p, 6) def format_compare(summary: dict[str, Any]) -> str: diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index 74513c555..79d2474d5 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -18,6 +18,7 @@ import json import math +import re from collections import Counter from dataclasses import dataclass from datetime import UTC, date, datetime @@ -34,6 +35,7 @@ KEEP, REVIEW, INSUFFICIENT = "keep", "review", "insufficient-evidence" DEFAULT_REVIEW_DAYS = 180 _EPS = 1e-9 # a mean delta this close to zero is a tie, as in compare +_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") # repo_ledger.v1's rule id class LedgerError(ValueError): @@ -70,6 +72,8 @@ def load_ledger(path: Path) -> Ledger: raise LedgerError(f"{path}: " + "; ".join(errors)) rules, seen = [], set() for entry in raw["rules"]: + if not _ID.fullmatch(entry["id"]): # the schema's pattern ends in `$`, which also matches before a newline + raise LedgerError(f"{path}: rule id {entry['id']!r} is not a valid id") rule = _rule(entry, path) if rule.id in seen: raise LedgerError(f"{path}: rule id {rule.id!r} is listed twice") @@ -156,7 +160,7 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st present: list[str], seed: int, warnings: list[str]) -> list[dict[str, Any]]: """The rule's results of one compare() call, with its linked cases that did not pair, as warnings.""" try: - summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present) + summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present, exact=True) except CompareError as exc: # the arm or the baseline has no cell among this rule's cases return [{**head, "model": models[0] if models else None, "error": str(exc)}] model = models[0] if models else None diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 9f8c6c098..3c0bff207 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -117,7 +117,7 @@ def test_seed_reaches_the_bootstrap_and_the_sign_flip(tmp_path, monkeypatch): seen = [] def spy(name, result): - return lambda deltas, seed: seen.append((name, seed)) or result + return lambda deltas, seed, **_: seen.append((name, seed)) or result monkeypatch.setattr(compare_mod, "bootstrap_ci", spy("bootstrap", {"mean": 0.0, "lo": -1.0, "hi": 1.0})) monkeypatch.setattr(compare_mod, "mean_ci", spy("t", {"mean": 0.0, "lo": -1.0, "hi": 1.0})) @@ -963,3 +963,15 @@ def test_arms_from_batches_with_different_built_in_graders_warn(tmp_path): _cell(new, "fix_a", "guidance:raw", 1) s = compare([old, new], "old/guidance:raw", ["new/guidance:raw"]) assert s["warnings"] == ["new/guidance:raw vs old/guidance:raw: the batches ran from different graders (g1, g2)"] + + +def test_the_printed_statistics_stay_rounded_and_full_precision_is_opt_in(): + # repo compare --json prints these as they were before the ledger; only the ledger asks for exact values. + from dokimi.repos.compare import paired_summary + + deltas = [1 / 3, 1 / 3, -0.1] + pairs = [{"task": f"t{i}", "baseline": 0.0, "treatment": d, "delta": d} for i, d in enumerate(deltas)] + plain, exact = paired_summary(pairs), paired_summary(pairs, exact=True) + assert plain["delta"]["mean"] == round(sum(deltas) / 3, 4) + assert exact["delta"]["mean"] == sum(deltas) / 3 + assert plain["p"] == round(exact["p"], 6) diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index ee736aba1..ed78ab405 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -95,6 +95,13 @@ def test_a_repeated_rule_id_is_refused(tmp_path): load_ledger(path) +def test_a_rule_id_with_a_trailing_newline_is_refused_not_a_second_R1(tmp_path): + # The schema's `$` matches before a final newline in Python; "R1\n" would otherwise load as its own rule. + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"]), _rule("R1\n", ["t2"])]) + with pytest.raises(LedgerError, match="R1"): + load_ledger(path) + + def test_an_unreadable_ledger_is_refused(tmp_path): path = tmp_path / "ledger.json" path.write_text("{not json", encoding="utf-8") From 3a7c9666f8725eb5bb8d0ee0c3b3f90c20feba10 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 23:37:28 -0700 Subject: [PATCH 11/17] chore(ledger): record compare.py over the 800-line cap with a rationale The ledger's exact statistics add 7 lines to repos/compare.py (807 lines), and the growth ratchet fails a new over-cap file without a baseline entry. The entry is the documented route; split the file in its own PR. Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d751c4643..8977b4ff8 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -5,6 +5,7 @@ "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1200, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/repos/compare.py": {"lines": 807, "rationale": "exact statistics for the ledger verdicts (#336) add 7 lines over the 800 cap; split compare.py in its own PR"}, "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1235, "rationale": "pre-existing when the gate was introduced"}, From f9ead263e23ac8469487732e263ce37c48af62e3 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 13:29:25 +0000 Subject: [PATCH 12/17] docs(ledger): state the 64-character limit on a rule id Signed-off-by: Anton Volkov --- dokimi/docs/rule-ledger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index f7a2f13e6..bda40b637 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -83,7 +83,7 @@ Schema cannot). | Field | Meaning | |---|---| -| `id` | Stable across edits, `[A-Za-z0-9][A-Za-z0-9._-]*`. | +| `id` | Stable across edits, `[A-Za-z0-9][A-Za-z0-9._-]{0,63}` (at most 64 characters). | | `rule`, `source` | Optional text and anchor, printed for the reader. | | `hypothesis` | Required. The claim the verdict tests, in words. | | `arm` | The treatment arm that carries the rule, `ARM` or `BATCH/ARM`. | From 6f94dbe68b3831cd39df73f41863cf43456786ac Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:37:41 -0800 Subject: [PATCH 13/17] fix(ledger): name each model when the arm or baseline has no linked cell A rule whose arm or baseline has no cell among its linked tasks went to insufficient-evidence with one result for the first --model or none, so models with linked cells on the other side lost their identity. Without --model, every model with a linked cell on either side now keeps its own result, each with the same reason. Pins the alpha boundary (p_holm equal to alpha is not a keep) and that compare --ledger with --model judges only that model. Refs PRRT_kwDOVBg7g86yX9Jj, PRRT_kwDOVBg7g86yavjT, PRRT_kwDOVBg7g86yavjW, PRRT_kwDOVBg7g86yavjZ Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/ledger.py | 14 ++++++++++-- dokimi/tests/test_repo_ledger.py | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index 79d2474d5..de40383ee 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -28,7 +28,7 @@ from dokimi.artifacts import schema_errors from dokimi.eval.stats import adjust_pvalues -from .compare import PRIMARY, compare, load_batches, min_sign_flip_p, parse_selector +from .compare import PRIMARY, compare, load_batches, min_sign_flip_p, parse_selector, selector_matches from .compare_reach import ALPHA from .compare_tasks import CompareError @@ -162,7 +162,9 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st try: summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present, exact=True) except CompareError as exc: # the arm or the baseline has no cell among this rule's cases - return [{**head, "model": models[0] if models else None, "error": str(exc)}] + # Without --model, every model that has a linked cell for either side keeps its own result. + named = models or _models_on_the_rule(batch_dirs, [rule.arm, base], present) or [None] + return [{**head, "model": model, "error": str(exc)} for model in named] model = models[0] if models else None warnings.extend(summary["warnings"]) # Only a linked case whose outcome (the gate or score, or the whole task) did not pair counts: a missing @@ -183,6 +185,14 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st return out +def _models_on_the_rule(batch_dirs: list[Path], selectors: list[str], tasks: list[str]) -> list[str]: + """The models with a finished cell on one of the rule's linked tasks that matches one of the selectors.""" + parsed = [parse_selector(text) for text in selectors] + cells, _ = load_batches(batch_dirs) + return sorted({c["model"] for c in cells if c["task"] in tasks + and any(selector_matches(sel, c) for sel in parsed)}) + + def _unmeasured(head: dict[str, Any], linked: list[dict[str, Any]], model: str | None) -> list[dict[str, Any]]: """The results of a comparison with no gate or score row: one per model its unpaired linked cases name, so that models with no pair keep their identity, each with its own reason.""" diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py index ed78ab405..b741bd5f2 100644 --- a/dokimi/tests/test_repo_ledger.py +++ b/dokimi/tests/test_repo_ledger.py @@ -470,3 +470,42 @@ def test_a_missing_cost_does_not_unpair_a_gate_case(tmp_path): assert entry["verdict"] == KEEP assert entry["paired"] == 8 assert not any("not paired" in w for w in entry["warnings"]) + + +@pytest.mark.parametrize("absent", ["arm", "baseline"]) +def test_a_rule_whose_arm_or_baseline_is_absent_names_each_model(tmp_path, absent): + batch = tmp_path / "run" + ran_t1 = BASE if absent == "arm" else ARM # the side that ran the linked task, for both models + other = ARM if absent == "arm" else BASE # the absent side ran only an unrelated task + for model in ("m-1", "m-2"): + _cell(batch, "t1", ran_t1, success=True, model=model) + _cell(batch, "t9", other, success=True, model=model) + entries = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"])])), [batch], BASE, + today=TODAY)["rules"] + assert sorted(e["model"] for e in entries) == ["m-1", "m-2"] + assert all(e["verdict"] == INSUFFICIENT and e["error"] for e in entries) + + +def test_a_p_exactly_on_alpha_is_insufficient_evidence(tmp_path): + # eight improved gate cases: the exact sign-flip p is 2 / 2**8, which is exactly representable + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)], alpha=2 / 2**8)), + [batch], BASE, today=TODAY) + assert result["rules"][0]["p_holm"] == 2 / 2**8 + assert _verdicts(result) == {("R1", "m-1"): INSUFFICIENT} + + +def test_compare_with_ledger_and_model_judges_only_that_model(tmp_path, capsys): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) # m-1 improves on every case + for t in tasks: # m-2 does not + _cell(batch, t, BASE, success=False, model="m-2") + _cell(batch, t, ARM, success=False, model="m-2") + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks)]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path), + "--model", "m-1", "--json"]) + args.func(args) + ledger = json.loads(capsys.readouterr().out)["ledger"] + assert {e["model"] for e in ledger["rules"]} == {"m-1"} + assert ledger["family"] == 1 # the Holm family counts the one model that was asked for From ca3645285440709b1d9fa9d685edad8985fbf1a5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:57:02 -0800 Subject: [PATCH 14/17] fix(ledger): type the named models list for mypy Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/ledger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py index de40383ee..d96014ede 100644 --- a/dokimi/dokimi/repos/ledger.py +++ b/dokimi/dokimi/repos/ledger.py @@ -163,7 +163,7 @@ def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: st summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present, exact=True) except CompareError as exc: # the arm or the baseline has no cell among this rule's cases # Without --model, every model that has a linked cell for either side keeps its own result. - named = models or _models_on_the_rule(batch_dirs, [rule.arm, base], present) or [None] + named: list[str | None] = list(models or _models_on_the_rule(batch_dirs, [rule.arm, base], present)) or [None] return [{**head, "model": model, "error": str(exc)} for model in named] model = models[0] if models else None warnings.extend(summary["warnings"]) From 45c34c2d5167fb832818a167c7dfaa82ebdbcfb6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:07:45 -0800 Subject: [PATCH 15/17] docs(ledger): repo run does not enforce the self-check results The new page said a case with a broken check cannot reach a paid run. repo run runs only an isolation probe; the per-task check is a step the user runs first. Signed-off-by: Nikolay Petrov --- dokimi/docs/rule-ledger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index bda40b637..b62dd6c4f 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -19,7 +19,7 @@ in [agentsmd/agents.md#213](https://github.com/agentsmd/agents.md/issues/213). |---|---|---| | Replay tasks from review threads | `dokimi/repos/replay/`, `repo tasks replay` ([contract §4.6](repo-evaluation-contract.md#46-replay-tasks-from-review-threads)) | One `dokimi:review_replay` task per merged PR, with the change as reviewed and the reviewers' threads as gold. No GitHub call. | | Deterministic review grader | `dokimi/repos/graders/review.py` | Matches the agent's findings JSON to gold threads by file and a line window (`window`, default 5). No LLM. | -| Executable tasks and self-check | `repo tasks check` (`dokimi/repos/selfcheck.py`, [contract §4.3](repo-evaluation-contract.md#43-self-check-tasks-check-no-llm)) | A task must fail on the untouched workspace and pass after its oracle, so a case with a broken check cannot reach a paid run. | +| Executable tasks and self-check | `repo tasks check` (`dokimi/repos/selfcheck.py`, [contract §4.3](repo-evaluation-contract.md#43-self-check-tasks-check-no-llm)) | Checks that a task fails on the untouched workspace and passes after its oracle. Run this check and resolve broken cases before starting paid runs; `repo run` does not enforce its results. | | Arms | `dokimi/repos/arms.py` | `guidance:none`, `guidance:raw`, `guidance:rev:`, `patch:` and the rest. | | Repetitions | `repo run --reps` | Several reps per (task, arm, model). | | Paired statistics | `repo compare` (`dokimi/repos/compare.py`) | Per-task deltas, sign-flip p, Holm over the gate and score rows of one invocation. | From f847271f9312be9acbd1f46f2c9d42fa42564296 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 12:58:54 -0800 Subject: [PATCH 16/17] docs(ledger): replay-only example; baseline error is not the ledger's The example rule mixed a score-only replay case with a gate task, which the ledger marks insufficient-evidence. Keep the replay cases only. A --baseline the batches do not have fails in compare() before the ledger loads, so its exit-2 line does not name the ledger file. Say so. Signed-off-by: Nikolay Petrov --- dokimi/docs/rule-ledger.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index b62dd6c4f..1e0cbc31f 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -74,7 +74,7 @@ Schema cannot). "hypothesis": "With the rule, agents stop touching unrelated files on the replayed PRs.", "arm": "guidance:raw", "baseline": "guidance:none", - "cases": ["review_replay_pr3580", "review_replay_pr3659", "fix_df_extratrees_reg"], + "cases": ["review_replay_pr3580", "review_replay_pr3659"], "reviewed": "2026-10-01" } ] @@ -121,8 +121,9 @@ python cli.py repo compare repo-runs/replay --baseline guidance:none --ledger re ``` The text output keeps the ordinary comparison and adds a `rule ledger` section. -A ledger that does not validate, or a default `--baseline` the batches do not have, -exits 2 with a `✗` line naming the file (and the rule, where the error is the rule's). +A ledger that does not validate exits 2 with a `✗` line naming the file (and the rule, +where the error is the rule's). A `--baseline` the batches do not have exits 2 with the +ordinary comparison's `✗` line, which does not name the ledger: the comparison runs first. A rule whose own `baseline` or `arm` has no cell among its cases is `not measured` for that rule, and the command still exits 0. The ordinary comparison runs first, so a `--task` or `--arm` it rejects (no cell matches) exits 2 before any ledger is evaluated. From 05371b955c0c28d1727a0d8c8377f8d056865260 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 21:12:15 +0000 Subject: [PATCH 17/17] docs(ledger): make the example hypothesis match the review-recall cases Signed-off-by: Anton Volkov --- dokimi/docs/rule-ledger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md index 1e0cbc31f..b006fb48c 100644 --- a/dokimi/docs/rule-ledger.md +++ b/dokimi/docs/rule-ledger.md @@ -71,7 +71,7 @@ Schema cannot). "id": "R13", "rule": "Revert edits to files the change does not need.", "source": "AGENTS.md#commits-and-pull-requests", - "hypothesis": "With the rule, agents stop touching unrelated files on the replayed PRs.", + "hypothesis": "With the rule, the agent's review findings land on more of the reviewers' threads on the replayed PRs.", "arm": "guidance:raw", "baseline": "guidance:none", "cases": ["review_replay_pr3580", "review_replay_pr3659"],