diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index b36efa75..753c06a0 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 37 + max_docs: 38 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md new file mode 100644 index 00000000..d5414e74 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-rule-ledger.md @@ -0,0 +1,3 @@ +### Added + +- **`repo compare --ledger FILE`** — gives each guidance rule of a `repo_ledger.v1` file a verdict: its arm against its baseline on the rule's linked cases (task ids), with the paired statistics of `repo compare`, Holm correction across the rules, and `keep`, `review` or `insufficient-evidence` with the reason. A rule whose `reviewed` date is older than `review_within_days` (default 180) gets a warning; its verdict is computed as usual. The ledger only computes verdicts; cutting a rule is left to a person. The design, the ledger format and the incident-to-case path are in `docs/rule-ledger.md`. `--json` adds a `ledger` key to the comparison. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 4133acaa..5903ad28 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -368,6 +368,10 @@ code. - [preparing-a-repository-evidence.md](preparing-a-repository-evidence.md) — the runs behind each figure on that page: task set, arms, model, cell count and raw count. +- [rule-ledger.md](rule-ledger.md) — one row per guidance rule: its + hypothesis, the replay or executable cases of the failures it prevents, and + the review date; `repo compare --ledger` gives each rule a keep, review or + insufficient-evidence verdict. The cut decision stays with a person. ## Extending Dokimi diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 5aafd5a8..f145f7cb 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -347,6 +347,10 @@ next release. what its `--task` selection keeps; `dokimi/repos/compare_reach.py` how small an effect a row can find (`mde`, an uncorrected t-based estimate, and the lowest `p_holm` Holm's family leaves it). +- `dokimi/repos/ledger.py` — `repo compare --ledger`: loads a `repo_ledger.v1` + file (rules with their hypothesis, arm, linked cases and review date) and gives + each rule a verdict from `repo compare` over its own cases, Holm-corrected + across the rules. [rule-ledger.md](rule-ledger.md). - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 4f613662..f7054d41 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10615,6 +10615,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--ledger", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_AppendAction", "choices": null, @@ -13121,6 +13137,7 @@ "'redteam'", "'repo_budget'", "'repo_environment'", + "'repo_ledger'", "'repo_readiness'", "'result'", "'skill_evidence'", diff --git a/dokimi/docs/rule-ledger.md b/dokimi/docs/rule-ledger.md new file mode 100644 index 00000000..b006fb48 --- /dev/null +++ b/dokimi/docs/rule-ledger.md @@ -0,0 +1,152 @@ + + +# Rule ledger: past failures as replay cases + +A guidance rule is a bet: the author expects an agent that reads it to make a +specific mistake less often. The ledger keeps one row per rule with that bet +written down, the cases that stand for the failures the rule was written to +prevent, and the date a person last confirmed it still earns its place. +`repo compare --ledger` then checks the bet on those cases only and gives a +verdict. It does not cut anything. This follows the held-out ablation proposal +in [agentsmd/agents.md#213](https://github.com/agentsmd/agents.md/issues/213). + +## What exists + +| Piece | Where | What it does | +|---|---|---| +| Replay tasks from review threads | `dokimi/repos/replay/`, `repo tasks replay` ([contract §4.6](repo-evaluation-contract.md#46-replay-tasks-from-review-threads)) | One `dokimi:review_replay` task per merged PR, with the change as reviewed and the reviewers' threads as gold. No GitHub call. | +| Deterministic review grader | `dokimi/repos/graders/review.py` | Matches the agent's findings JSON to gold threads by file and a line window (`window`, default 5). No LLM. | +| Executable tasks and self-check | `repo tasks check` (`dokimi/repos/selfcheck.py`, [contract §4.3](repo-evaluation-contract.md#43-self-check-tasks-check-no-llm)) | Checks that a task fails on the untouched workspace and passes after its oracle. Run this check and resolve broken cases before starting paid runs; `repo run` does not enforce its results. | +| Arms | `dokimi/repos/arms.py` | `guidance:none`, `guidance:raw`, `guidance:rev:`, `patch:` and the rest. | +| Repetitions | `repo run --reps` | Several reps per (task, arm, model). | +| Paired statistics | `repo compare` (`dokimi/repos/compare.py`) | Per-task deltas, sign-flip p, Holm over the gate and score rows of one invocation. | +| Rule vocabulary | [repo-evaluation-agent-modes.md §4](repo-evaluation-agent-modes.md#4-team-review-rules) | Mined `review_rules.json` entries (`R13`, `cut_date`). Method only: no code reads that file yet. | + +Before this change nothing tied a rule to the cases it was written from, and +`repo compare` had no notion of a rule. + +## From an incident to a held-out case + +An incident is a failure a rule should prevent. Three sources are in use or +planned: + +1. **A review comment on a merged PR.** Run `repo tasks replay` on the PR's + threads. The case is `review_replay_pr` and already exists. Its grader is + recall of the gold threads, so the check is deterministic. +2. **A failed agent run, or a bug a fix PR corrected.** Write an executable task + in the content repository: a test that fails on the base commit and passes + with the oracle. `repo tasks check` enforces both halves. No scaffold command + exists yet; the task is written by hand. +3. **A transcript slice.** A transcript shows what the agent did, not what + should have happened. It is evidence for case 1 or 2, not a case: a case + needs a check that does not read the transcript. + +Then, per rule: + +1. Record the incident as the rule's `source` (a file and anchor, or the + comment URL) and state the `hypothesis`: what the rule should change in + an answer on its cases. +2. Put the case ids in `cases`. Use cases the rule was not written from: a + rule checked on the failure it was mined from only shows it can repeat + itself. +3. Run `repo tasks check` on the cases, then the rule's arm and its baseline + with `--reps 3` to `5`. +4. Compare with `repo compare --ledger`. + +## Ledger format (`repo_ledger.v1`) + +`dokimi/schemas/repo_ledger.v1.json` is the schema. `rules` must not be +empty, and ids must be unique in the file (the loader checks this, since JSON +Schema cannot). + +```json +{ + "schema_version": "repo_ledger.v1", + "policy": {"alpha": 0.05, "review_within_days": 180}, + "rules": [ + { + "id": "R13", + "rule": "Revert edits to files the change does not need.", + "source": "AGENTS.md#commits-and-pull-requests", + "hypothesis": "With the rule, the agent's review findings land on more of the reviewers' threads on the replayed PRs.", + "arm": "guidance:raw", + "baseline": "guidance:none", + "cases": ["review_replay_pr3580", "review_replay_pr3659"], + "reviewed": "2026-10-01" + } + ] +} +``` + +| Field | Meaning | +|---|---| +| `id` | Stable across edits, `[A-Za-z0-9][A-Za-z0-9._-]{0,63}` (at most 64 characters). | +| `rule`, `source` | Optional text and anchor, printed for the reader. | +| `hypothesis` | Required. The claim the verdict tests, in words. | +| `arm` | The treatment arm that carries the rule, `ARM` or `BATCH/ARM`. | +| `baseline` | Optional. Overrides the `--baseline` of the compare for this rule. | +| `cases` | Task ids, at least one, unique. Ids with no finished cell are listed as missing. | +| `reviewed` | `YYYY-MM-DD`, the last date a person confirmed the rule. | +| `policy.alpha` | Default 0.05; a number strictly between 0 and 1. | +| `policy.review_within_days` | Default 180. An older `reviewed` gets a warning. | + +## Verdicts + +For each rule and model, `repo compare --ledger` runs the ordinary paired +comparison of the rule's arm against its baseline over the rule's linked cases +(`repo compare` itself, so the pairing, headroom and provenance warnings are +the same). The p values of the rule results are Holm-corrected together. A result whose +linked cases mix gate and score tasks has no single test, so it takes no slot in +that family and is `insufficient-evidence`. The rules' family is separate from the +family of the comparison's own rows. + +| Verdict | When | Meaning | +|---|---|---| +| `keep` | Mean delta above zero and Holm p below alpha | The arm beats its baseline on the linked cases. | +| `review` | Mean delta below zero and Holm p below alpha | The baseline beats the arm there. A person decides whether to fix or cut the rule. | +| `insufficient-evidence` | Anything else | Not measured, mixed kinds, every case tied, no significant difference, or too few changed cases for any outcome to reach alpha. The reason says which. | + +A review older than `review_within_days` adds a warning and leaves the verdict +alone. "No significant difference" is not a verdict to cut: the rule is not +shown to help, and whether that is enough to cut it is a decision for a person. + +## CLI + +```bash +python cli.py repo compare repo-runs/replay --baseline guidance:none --ledger repo-ledger.json +python cli.py repo compare repo-runs/replay --baseline guidance:none --ledger repo-ledger.json --json +``` + +The text output keeps the ordinary comparison and adds a `rule ledger` section. +A ledger that does not validate exits 2 with a `✗` line naming the file (and the rule, +where the error is the rule's). A `--baseline` the batches do not have exits 2 with the +ordinary comparison's `✗` line, which does not name the ledger: the comparison runs first. +A rule whose own `baseline` or `arm` has no cell among its cases is `not measured` for +that rule, and the command still exits 0. The ordinary comparison runs first, so a +`--task` or `--arm` it rejects (no cell matches) exits 2 before any ledger is evaluated. +Once it succeeds, its `--arm`, `--task` and `--metric` filters do not change the ledger, +which uses the rules' own cases and arms. `--model` does apply: it limits the ledger to +those models, which changes its Holm family and so its verdicts. + +## Limits + +- A rule's arm carries every rule that is in it. A verdict on one rule compares + the whole arm with its baseline, so it confounds the rule with its neighbours. + A per-rule claim needs an arm that differs by that rule alone (a `patch:` + arm that removes one rule, say). The ledger does not build those arms. +- Few cases give few verdicts. With fewer than six cases whose delta is not zero, + no outcome reaches p below 0.05, and the reason says so. +- Gate and score cases do not mix in one rule. + +## Not built + +- A scaffold that turns a failed run directory, or a description and a failing + check, into a task skeleton. The executable-task format needs an oracle and a + verifier, so a skeleton would still need hand work. +- A check that a rule's cases are held out from the PRs it was mined from. The + [mining leak guard](repo-evaluation-agent-modes.md#4-team-review-rules) + (`cut_date`) is the method; nothing enforces it for a ledger yet. +- Reading the mined `review_rules.json` into the ledger. diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index ce35de99..cf9b9135 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -61,6 +61,7 @@ "task_inspect": "task_inspect.v1", "redteam": "redteam.v1", "cli_routes": "cli_routes.v1", + "repo_ledger": "repo_ledger.v1", "need_graph": "need_graph.v1", "eval_coverage": "eval_coverage.v1", } diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index 9e265582..a592b9b6 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -242,6 +242,7 @@ class Concept: Flag("--answers", "FILE", "path"), Flag("--registry", "FILE", "path"), Flag("--from", "RUN", "text"), + Flag("--ledger", "FILE", "path"), ), banned=("--product", "--library", "--libraries", "--target", "--root", "--repo-root", "--tasks-root", "--subjects-root", "--questions-root", "--questions-from", diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 8c81f085..187bd4f4 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -352,8 +352,10 @@ def cmd_repo_report(args: argparse.Namespace) -> None: def cmd_repo_compare(args: argparse.Namespace) -> None: - """Per (treatment, model, measure): per-task deltas against the baseline, 95% CI, sign-flip p.""" + """Per (treatment, model, measure): per-task deltas against the baseline, 95% CI, sign-flip p; ``--ledger`` + adds one verdict per guidance rule.""" from dokimi.repos.compare import CompareError, compare, format_compare + from dokimi.repos.ledger import LedgerError, evaluate, format_ledger, load_ledger from dokimi.repos.report import CellFileError dirs = [Path(d) for d in args.batch_dirs] @@ -365,10 +367,16 @@ def cmd_repo_compare(args: argparse.Namespace) -> None: try: summary = compare(dirs, args.baseline, arms, args.model, tuple(args.metric or ()), pass_k=args.pass_k, path_stability=args.path_stability, tasks=args.task) - except (CompareError, CellFileError) as exc: # an unusable baseline, arm, metric or cell file: exit 2 + if args.ledger is not None: + summary["ledger"] = evaluate(load_ledger(Path(args.ledger)), dirs, args.baseline, args.model, + ledger_name=args.ledger) + except (CompareError, CellFileError, LedgerError) as exc: # an unusable baseline, arm, metric, cell or ledger print(f"✗ {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) - print(json.dumps(summary, indent=2) if args.json else format_compare(summary)) + text = format_compare(summary) + if "ledger" in summary: + text += "\n\n" + format_ledger(summary["ledger"]) + print(json.dumps(summary, indent=2) if args.json else text) def _usd(text: str) -> float: @@ -520,5 +528,8 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: help="Also compare pass^K: the chance K cells of a task all pass its gate") compare_p.add_argument("--path-stability", action="store_true", dest="path_stability", help="Also compare path_distance: how much the tool paths of a task's cells differ") + compare_p.add_argument("--ledger", default=None, metavar="FILE", + help="Also give each guidance rule of this repo_ledger.v1 file a verdict (keep, review or " + "insufficient-evidence) from its linked cases, against its baseline (dokimi/docs/rule-ledger.md)") compare_p.add_argument("--json", action="store_true", help="Print the comparison as JSON") compare_p.set_defaults(func=cmd_repo_compare) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 308370da..3e7dcccd 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -174,7 +174,9 @@ def batch_stamp(batch_dir: Path) -> dict[str, str]: def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None = None, models: list[str] | None = None, metrics: tuple[str, ...] = (), seed: int = 0, - pass_k: int | None = None, path_stability: bool = False, tasks: list[str] | None = None) -> dict[str, Any]: + pass_k: int | None = None, path_stability: bool = False, tasks: list[str] | None = None, + *, exact: bool = False) -> dict[str, Any]: + """``exact``: the statistics at full precision (the ledger's verdicts read them); otherwise rounded as printed.""" metrics = tuple(dict.fromkeys(metrics)) if pass_k is not None and pass_k < 1: raise CompareError(f"--pass-k must be at least 1, got {pass_k}") @@ -204,7 +206,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None unpaired += _one_arm_models(text, base_vals, treat_vals, models) shared_models = sorted({m for m, _ in base_vals} & {m for m, _ in treat_vals}) for model in [m for m in shared_models if not models or m in models]: - skipped = _compare_model(text, model, base_vals, treat_vals, measures, seed, rows, unpaired) + skipped = _compare_model(text, model, base_vals, treat_vals, measures, seed, rows, unpaired, exact=exact) if skipped: one_arm.append(f"{text} {model}: only one arm reports {', '.join(skipped)} for any shared task; " "not compared") @@ -359,7 +361,7 @@ def _holm(rows: list[dict[str, Any]]) -> None: def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], measures: list[str], seed: int, - rows: list[dict[str, Any]], unpaired: list[dict[str, Any]]) -> list[str]: + rows: list[dict[str, Any]], unpaired: list[dict[str, Any]], *, exact: bool = False) -> list[str]: """Append one row per measure of one (treatment, model), and the tasks only one arm has. Returns the :data:`REPORTED` measures only one arm reports for any shared task: they are skipped, not unpaired. @@ -383,7 +385,7 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], if any(has): skipped.append(measure) continue - row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired) + row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired, exact=exact) if row is not None: rows.append(row) return skipped @@ -421,7 +423,7 @@ def _measure_pairs(treatment: str, model: str, shared: list[str], base_vals: dic def _compare_measure(treatment: str, model: str, measure: str, shared: list[str], base_vals: dict[tuple[str, str], dict[str, Any]], treat_vals: dict[tuple[str, str], dict[str, Any]], seed: int, - unpaired: list[dict[str, Any]]) -> dict[str, Any] | None: + unpaired: list[dict[str, Any]], *, exact: bool = False) -> dict[str, Any] | None: """The row of one measure over the shared tasks of one (treatment, model), or None with no pair.""" pairs = _measure_pairs(treatment, model, shared, base_vals, treat_vals, measure, unpaired) if not pairs: @@ -433,7 +435,7 @@ def _compare_measure(treatment: str, model: str, measure: str, shared: list[str] or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs] flag = headroom(pairs, tops, failed) return {"treatment": treatment, "model": model, "measure": measure, - **paired_summary(pairs, measure, seed=seed), "headroom": flag} + **paired_summary(pairs, measure, seed=seed, exact=exact), "headroom": flag} def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]], @@ -516,7 +518,8 @@ def _pair(task: str, base: dict[str, Any], treat: dict[str, Any], measure: str) "n_baseline": base["n"][measure], "n_treatment": treat["n"][measure]} -def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int = 0) -> dict[str, Any]: +def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int = 0, *, + exact: bool = False) -> dict[str, Any]: """Statistics over per-task pairs ``{task, baseline, treatment, delta, ...}``. The values are taken at full precision and rounded only in the output: rounded rates such as 1/3 and 2/3 @@ -530,14 +533,14 @@ def paired_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: int deltas = [p["delta"] for p in pairs] ci = (bootstrap_ci if measure in PRIMARY else mean_ci)(deltas, seed=seed) lo, hi = (float(ci["lo"]), float(ci["hi"])) if len(deltas) > 1 else (None, None) # mean_ci gives numpy floats - p = sign_flip_p(deltas, seed=seed) + p = sign_flip_p(deltas, seed=seed, exact=exact) sd = statistics.stdev(deltas) if len(deltas) > 1 else 0.0 reachable = min_sign_flip_p(sum(abs(d) > EPS for d in deltas)) < ALPHA # else no effect can be detected return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), "treatment_mean": _round(_mean([p["treatment"] for p in pairs])), - "delta": {"mean": ci["mean"], "lo": lo, "hi": hi}, + "delta": {"mean": sum(deltas) / len(deltas) if exact else ci["mean"], "lo": lo, "hi": hi}, "p": p, "p_holm": None, "p_holm_floor": None, @@ -557,10 +560,12 @@ def min_sign_flip_p(changed: int) -> float: return min(1.0, 2 / 2 ** changed) if changed <= EXACT_MAX else 1 / (N_RESAMPLES + 1) -def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES) -> float | None: +def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, + exact: bool = False) -> float | None: """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. ``None`` when every delta is a tie: there is no evidence either way, and 1.0 would read as a measured null. + Rounded to 6 places as printed, unless ``exact``. """ d = [x for x in deltas if abs(x) > EPS] if not d: @@ -569,10 +574,12 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL if len(d) <= EXACT_MAX: hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed for signs in itertools.product((1, -1), repeat=len(d))) - return round(hits / 2 ** len(d), 6) - rng = random.Random(seed) - hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) - return round((hits + 1) / (n_resamples + 1), 6) + p = hits / 2 ** len(d) + else: + rng = random.Random(seed) + hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) + p = (hits + 1) / (n_resamples + 1) + return p if exact else round(p, 6) def format_compare(summary: dict[str, Any]) -> str: diff --git a/dokimi/dokimi/repos/ledger.py b/dokimi/dokimi/repos/ledger.py new file mode 100644 index 00000000..d96014ed --- /dev/null +++ b/dokimi/dokimi/repos/ledger.py @@ -0,0 +1,280 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``repo compare --ledger``: one verdict per guidance rule, from the cases it was written to prevent. + +A ledger (``repo_ledger.v1``, dokimi/docs/rule-ledger.md) gives each rule its hypothesis, the arm that +carries it, the task ids of the cases standing for the failures it prevents, and the date it was last +reviewed. Each rule is compared with its baseline by :func:`.compare.compare` over its own cases, so the +pairing, headroom and provenance warnings are those of ``repo compare``. The rules' p values are +Holm-corrected together here, not in the comparison's own family. + +Verdicts are computed, cutting is not: ``keep`` (the arm beats its baseline on the linked cases at +Holm-adjusted p below alpha), ``review`` (the baseline beats it at that level), ``insufficient-evidence`` +(anything else, with the reason). A review date older than the policy allows is a warning, not a verdict. +""" + +from __future__ import annotations + +import json +import math +import re +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Any + +from dokimi.artifacts import schema_errors +from dokimi.eval.stats import adjust_pvalues + +from .compare import PRIMARY, compare, load_batches, min_sign_flip_p, parse_selector, selector_matches +from .compare_reach import ALPHA +from .compare_tasks import CompareError + +KEEP, REVIEW, INSUFFICIENT = "keep", "review", "insufficient-evidence" +DEFAULT_REVIEW_DAYS = 180 +_EPS = 1e-9 # a mean delta this close to zero is a tie, as in compare +_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") # repo_ledger.v1's rule id + + +class LedgerError(ValueError): + """A ledger that cannot be read or does not validate; the message says where.""" + + +@dataclass(frozen=True) +class Rule: + id: str + hypothesis: str + arm: str + cases: tuple[str, ...] + reviewed: date + baseline: str | None + text: str | None + source: str | None + + +@dataclass(frozen=True) +class Ledger: + rules: tuple[Rule, ...] + alpha: float + review_within_days: int + + +def load_ledger(path: Path) -> Ledger: + """The ledger in ``path``, validated against ``repo_ledger.v1`` and checked for duplicate ids.""" + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise LedgerError(f"{path}: cannot read the ledger: {exc}") from exc + errors = schema_errors("repo_ledger.v1", raw) + if errors: + raise LedgerError(f"{path}: " + "; ".join(errors)) + rules, seen = [], set() + for entry in raw["rules"]: + if not _ID.fullmatch(entry["id"]): # the schema's pattern ends in `$`, which also matches before a newline + raise LedgerError(f"{path}: rule id {entry['id']!r} is not a valid id") + rule = _rule(entry, path) + if rule.id in seen: + raise LedgerError(f"{path}: rule id {rule.id!r} is listed twice") + seen.add(rule.id) + rules.append(rule) + policy = raw.get("policy", {}) + alpha = float(policy.get("alpha", ALPHA)) + if not math.isfinite(alpha): # json.loads reads NaN; it would make every verdict insufficient-evidence + raise LedgerError(f"{path}: policy.alpha must be a finite number, not {alpha}") + return Ledger(tuple(rules), alpha, + int(policy.get("review_within_days", DEFAULT_REVIEW_DAYS))) + + +def _rule(entry: dict[str, Any], path: Path) -> Rule: + try: + parse_selector(entry["arm"]) + if entry.get("baseline") is not None: + parse_selector(entry["baseline"]) + reviewed = date.fromisoformat(entry["reviewed"]) + except ValueError as exc: # a bad selector (CompareError is one) or an impossible date such as 2026-02-30 + raise LedgerError(f"{path}: rule {entry['id']!r}: {exc}") from exc + return Rule(id=entry["id"], hypothesis=entry["hypothesis"], arm=entry["arm"], cases=tuple(entry["cases"]), + reviewed=reviewed, baseline=entry.get("baseline"), text=entry.get("rule"), source=entry.get("source")) + + +def evaluate(ledger: Ledger, batch_dirs: list[Path], baseline: str, models: list[str] | None = None, + seed: int = 0, today: date | None = None, ledger_name: str = "") -> dict[str, Any]: + """One result per rule and model: paired statistics, Holm-adjusted p, verdict and reason.""" + today = today or datetime.now(UTC).date() + known = {c["task"] for c in load_batches(batch_dirs)[0]} + entries: list[dict[str, Any]] = [] + warnings: list[str] = [] + for rule in ledger.rules: + rule_entries = _rule_entries(rule, batch_dirs, baseline, models, known, seed, warnings) + age = (today - rule.reviewed).days + stale = age < 0 or age > ledger.review_within_days + if age < 0: + note = f"review date {rule.reviewed.isoformat()} is after {today.isoformat()}: check the date" + else: + note = (f"review date {rule.reviewed.isoformat()} is {age} days old (policy: " + f"{ledger.review_within_days}): re-review the rule") + for entry in rule_entries: + entry.update(reviewed=rule.reviewed.isoformat(), days_since_review=age, stale=stale) + if stale: # a new list: error entries of one rule share their head's warnings list + entry["warnings"] = [*entry["warnings"], note] + entries += rule_entries + _holm(entries) + for entry in entries: + entry["verdict"], entry["reason"] = _verdict(entry, ledger.alpha) + return {"ledger": ledger_name, "alpha": ledger.alpha, "review_within_days": ledger.review_within_days, + "today": today.isoformat(), "family": sum(_in_family(e) for e in entries), + "rules": entries, "warnings": list(dict.fromkeys(warnings))} + + +def _head(rule: Rule, baseline: str, linked: int, missing: list[str]) -> dict[str, Any]: + return {"rule": rule.id, "rule_text": rule.text, "source": rule.source, "hypothesis": rule.hypothesis, + "arm": rule.arm, "baseline": baseline, "linked": linked, "missing": missing, "model": None, + "measure": None, "paired": 0, "error": None, "mixed": False, "changed": 0, "headroom": None, + "baseline_mean": None, "treatment_mean": None, "delta": {"mean": None, "lo": None, "hi": None}, + "up": 0, "down": 0, "tied": 0, "p": None, "p_holm": None, "warnings": []} + + +def _rule_entries(rule: Rule, batch_dirs: list[Path], default_baseline: str, models: list[str] | None, + known: set[str], seed: int, warnings: list[str]) -> list[dict[str, Any]]: + """A rule's results: one per model with a paired gate or score measure, else one that says why not.""" + missing = [c for c in rule.cases if c not in known] + present = [c for c in rule.cases if c in known] + base = rule.baseline or default_baseline + head = _head(rule, base, len(rule.cases), missing) + requested = list(dict.fromkeys(models)) if models else [None] # a repeated --model is one comparison + if not present: # the requested models stay named, one result each + return [{**head, "model": model, "error": "no linked case has a finished cell"} for model in requested] + entries: list[dict[str, Any]] = [] + # One comparison per --model: a model with no cell among this rule's cases fails only its own comparison. + for model in requested: + entries += _compared(rule, head, batch_dirs, base, [model] if model else None, present, seed, warnings) + per_model = Counter(e["model"] for e in entries if not e["error"] and e["measure"] in PRIMARY) + for entry in entries: + entry["mixed"] = not entry["error"] and per_model[entry["model"]] > 1 + return entries + + +def _compared(rule: Rule, head: dict[str, Any], batch_dirs: list[Path], base: str, models: list[str] | None, + present: list[str], seed: int, warnings: list[str]) -> list[dict[str, Any]]: + """The rule's results of one compare() call, with its linked cases that did not pair, as warnings.""" + try: + summary = compare(batch_dirs, base, [rule.arm], models, (), seed=seed, tasks=present, exact=True) + except CompareError as exc: # the arm or the baseline has no cell among this rule's cases + # Without --model, every model that has a linked cell for either side keeps its own result. + named: list[str | None] = list(models or _models_on_the_rule(batch_dirs, [rule.arm, base], present)) or [None] + return [{**head, "model": model, "error": str(exc)} for model in named] + model = models[0] if models else None + warnings.extend(summary["warnings"]) + # Only a linked case whose outcome (the gate or score, or the whole task) did not pair counts: a missing + # cost or turn count is metadata, and does not weaken the verdict. + linked = [u for u in summary["unpaired"] if u["task"] in present and u["measure"] in (None, *PRIMARY)] + rows = [r for r in summary["comparisons"] if r["measure"] in PRIMARY] + if not rows: + return _unmeasured(head, linked, model) + out = [_entry(head, r) for r in rows] + for entry in out: + own = [u for u in linked if u["model"] == entry["model"]] + if own: + entry["warnings"].append(f"{len({u['task'] for u in own})} linked case(s) not paired: " + + "; ".join(sorted({u["reason"] for u in own}))) + for unpaired in sorted({u["model"] for u in linked} - {r["model"] for r in rows}): # not measured, and named + reasons = sorted({u["reason"] for u in linked if u["model"] == unpaired}) + out.append({**head, "model": unpaired, "error": "; ".join(reasons)}) + return out + + +def _models_on_the_rule(batch_dirs: list[Path], selectors: list[str], tasks: list[str]) -> list[str]: + """The models with a finished cell on one of the rule's linked tasks that matches one of the selectors.""" + parsed = [parse_selector(text) for text in selectors] + cells, _ = load_batches(batch_dirs) + return sorted({c["model"] for c in cells if c["task"] in tasks + and any(selector_matches(sel, c) for sel in parsed)}) + + +def _unmeasured(head: dict[str, Any], linked: list[dict[str, Any]], model: str | None) -> list[dict[str, Any]]: + """The results of a comparison with no gate or score row: one per model its unpaired linked cases name, + so that models with no pair keep their identity, each with its own reason.""" + named = list(dict.fromkeys(u["model"] for u in linked)) or [model] + out = [] + for name in named: + reasons = sorted({u["reason"] for u in linked if u["model"] == name}) + out.append({**head, "model": name, "error": "; ".join(reasons) or "no linked case has cells in both arms"}) + return out + + +def _entry(head: dict[str, Any], row: dict[str, Any]) -> dict[str, Any]: + d = row["delta"] + out = {**head, "model": row["model"], "measure": row["measure"], "paired": row["n_tasks"], + "baseline_mean": row["baseline_mean"], "treatment_mean": row["treatment_mean"], + "delta": {"mean": d["mean"], "lo": d["lo"], "hi": d["hi"]}, "up": row["up"], "down": row["down"], + "tied": row["tied"], "p": row["p"], "headroom": row["headroom"], "mixed": False, + "changed": row["up"] + row["down"], "warnings": []} + if row["headroom"]: + out["warnings"].append(f"{row['measure']} headroom: {row['headroom']} (the cases cannot show an effect)") + return out + + +def _in_family(entry: dict[str, Any]) -> bool: + """A result has a p and is not mixed: a gate-and-score rule has no single test, so it takes no slot.""" + return entry["p"] is not None and not entry["mixed"] + + +def _holm(entries: list[dict[str, Any]]) -> None: + """``p_holm`` over every result in the family (the exact adjusted value); the others get none.""" + scored = [e for e in entries if _in_family(e)] + for entry, adjusted in zip(scored, adjust_pvalues([e["p"] for e in scored], rounded=False), strict=True): + entry["p_holm"] = adjusted + + +def _verdict(entry: dict[str, Any], alpha: float) -> tuple[str, str]: + """The verdict of one result and why. Cutting a rule is left to a person.""" + if entry["error"] or not entry["paired"]: + return INSUFFICIENT, f"not measured: {entry['error'] or 'no linked case has cells in both arms'}" + if entry["mixed"]: + return INSUFFICIENT, ("the linked cases mix gate and score tasks: link one kind of task per rule " + "(not in the Holm family)") + if entry["p_holm"] is None: + return INSUFFICIENT, "every linked case ties: there is no difference to test" + p, mean = entry["p_holm"], entry["delta"]["mean"] + if p < alpha and abs(mean) > _EPS: + if mean > 0: + return KEEP, f"beats {entry['baseline']} on its linked cases (p_holm {p:.4f} < {alpha})" + return REVIEW, (f"{entry['baseline']} beats it on its linked cases (p_holm {p:.4f} < {alpha}): " + "a person decides whether to fix or cut the rule") + reason = f"no significant difference from {entry['baseline']} (p_holm {p:.4f}, alpha {alpha})" + if min_sign_flip_p(entry["changed"]) >= alpha: + reason += f"; with {entry['changed']} changed case(s) no outcome reaches p < {alpha}: link more cases" + return INSUFFICIENT, reason + + +def format_ledger(result: dict[str, Any]) -> str: + """The rule results as text: one block per rule and model, the verdict first.""" + lines = [f"rule ledger {result['ledger']}: alpha {result['alpha']}, Holm over {result['family']} " + f"result(s), reviews checked as of {result['today']}"] + for e in result["rules"]: + where = f"{e['rule']}: {e['arm']} vs {e['baseline']}" + (f", model {e['model']}" if e["model"] else "") + lines.append(f"{where}, {e['measure'] or 'not measured'}: {e['verdict']}") + if e["paired"]: + d = e["delta"] + ci = "-" if d["lo"] is None else f"[{d['lo']:+.3f}, {d['hi']:+.3f}]" + label = "mean score" if e["measure"] == "score" else "pass rate" # a score is not a success rate + lines.append(f" paired {e['paired']} of {e['linked']} linked: {label} {_num(e['baseline_mean'])} -> " + f"{_num(e['treatment_mean'])}, delta {d['mean']:+.3f} {ci}, p {_p(e['p'])}, " + f"p_holm {_p(e['p_holm'])}, up/down/tied {e['up']}/{e['down']}/{e['tied']}") + lines.append(f" {e['reason']}") + if e["missing"]: + lines.append(f" no finished cell for: {', '.join(e['missing'])}") + lines.append(f" hypothesis: {e['hypothesis']}") + lines += [f" warning: {w}" for w in e["warnings"]] + lines += [f"warning: {w}" for w in result["warnings"]] + return "\n".join(lines) + + +def _num(value: float | None) -> str: + return "-" if value is None else f"{value:.3f}" + + +def _p(value: float | None) -> str: + return "-" if value is None else f"{value:.4f}" diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index bfbe3c05..0952af17 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -67,6 +67,7 @@ class KindFields: "repo_budget": KindFields(subject=("tree",), counts=("entries",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "repo_ledger": KindFields(counts=("rules",)), "redteam": KindFields(subject=("questions_path",), model=("judge_model",), counts=("violations", "questions")), "need_graph": KindFields(subject=("artifact_name",), counts=("nodes", "edges")), diff --git a/dokimi/dokimi/schemas/repo_ledger.v1.json b/dokimi/dokimi/schemas/repo_ledger.v1.json new file mode 100644 index 00000000..99cb6e3a --- /dev/null +++ b/dokimi/dokimi/schemas/repo_ledger.v1.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/repo_ledger.v1.json", + "title": "Guidance rule ledger v1", + "description": "One row per guidance rule: the hypothesis it was written for, the replay or executable cases that stand for the failures it prevents, and the date it was last reviewed. `repo compare --ledger` pairs each rule's cases against its arm and a baseline and reports a verdict (keep, review or insufficient-evidence). Only the verdict is computed; cutting a rule is a human decision. Ids are unique within the file (checked by the loader, which JSON Schema cannot express).", + "type": "object", + "required": ["schema_version", "rules"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "repo_ledger.v1"}, + "policy": {"$ref": "#/$defs/policy"}, + "rules": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/rule"}} + }, + "$defs": { + "policy": { + "type": "object", + "description": "Optional. alpha: the Holm-adjusted p below which a delta counts. review_within_days: a review older than this is flagged as stale.", + "additionalProperties": false, + "properties": { + "alpha": {"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1}, + "review_within_days": {"type": "integer", "minimum": 1} + } + }, + "rule": { + "type": "object", + "required": ["id", "hypothesis", "arm", "cases", "reviewed"], + "additionalProperties": false, + "properties": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$", "description": "Stable across edits of the guidance, unique in the file."}, + "rule": {"type": "string", "minLength": 1, "description": "The guidance text, as written."}, + "source": {"type": "string", "minLength": 1, "description": "Where the rule lives or was mined from: a file and anchor, or a review comment URL."}, + "hypothesis": {"type": "string", "minLength": 1, "description": "What the rule should change in an agent's answer on its cases."}, + "arm": {"type": "string", "minLength": 1, "description": "The treatment arm that carries the rule, ARM or BATCH/ARM."}, + "baseline": {"type": "string", "minLength": 1, "description": "Optional baseline arm for this rule; default: the --baseline of the compare."}, + "cases": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}, "description": "Task ids of the cases that stand for the failures the rule prevents."}, + "reviewed": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "description": "The date (YYYY-MM-DD) a maintainer last confirmed the rule still earns its place."} + } + } + } +} diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 0f2d53b2..08384321 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -131,6 +131,7 @@ nav: - repo-evaluation-agent-modes.md - preparing-a-repository.md - preparing-a-repository-evidence.md + - rule-ledger.md - Extending Dokimi: - architecture.md - contributing-metric.md diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index b0b75b38..5762d1d6 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -5,6 +5,7 @@ "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1199, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/repos/compare.py": {"lines": 807, "rationale": "exact statistics for the ledger verdicts (#336) add 7 lines over the 800 cap; split compare.py in its own PR"}, "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1235, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/fixtures/artifacts/repo_ledger.v1.json b/dokimi/tests/fixtures/artifacts/repo_ledger.v1.json new file mode 100644 index 00000000..28ebff45 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/repo_ledger.v1.json @@ -0,0 +1,23 @@ +{ + "schema_version": "repo_ledger.v1", + "policy": {"alpha": 0.05, "review_within_days": 180}, + "rules": [ + { + "id": "R13", + "rule": "Revert edits to files the change does not need.", + "source": "AGENTS.md#commits-and-pull-requests", + "hypothesis": "With the rule, agents stop touching unrelated files on the replayed PRs.", + "arm": "guidance:rules", + "baseline": "guidance:none", + "cases": ["review_replay_pr3580", "review_replay_pr3659", "fix_df_extratrees_reg"], + "reviewed": "2026-10-01" + }, + { + "id": "R14", + "hypothesis": "With the rule, agents run the touched tests before they report done.", + "arm": "guidance:rules", + "cases": ["fix_df_extratrees_reg"], + "reviewed": "2026-09-15" + } + ] +} diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index 9f8c6c09..3c0bff20 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -117,7 +117,7 @@ def test_seed_reaches_the_bootstrap_and_the_sign_flip(tmp_path, monkeypatch): seen = [] def spy(name, result): - return lambda deltas, seed: seen.append((name, seed)) or result + return lambda deltas, seed, **_: seen.append((name, seed)) or result monkeypatch.setattr(compare_mod, "bootstrap_ci", spy("bootstrap", {"mean": 0.0, "lo": -1.0, "hi": 1.0})) monkeypatch.setattr(compare_mod, "mean_ci", spy("t", {"mean": 0.0, "lo": -1.0, "hi": 1.0})) @@ -963,3 +963,15 @@ def test_arms_from_batches_with_different_built_in_graders_warn(tmp_path): _cell(new, "fix_a", "guidance:raw", 1) s = compare([old, new], "old/guidance:raw", ["new/guidance:raw"]) assert s["warnings"] == ["new/guidance:raw vs old/guidance:raw: the batches ran from different graders (g1, g2)"] + + +def test_the_printed_statistics_stay_rounded_and_full_precision_is_opt_in(): + # repo compare --json prints these as they were before the ledger; only the ledger asks for exact values. + from dokimi.repos.compare import paired_summary + + deltas = [1 / 3, 1 / 3, -0.1] + pairs = [{"task": f"t{i}", "baseline": 0.0, "treatment": d, "delta": d} for i, d in enumerate(deltas)] + plain, exact = paired_summary(pairs), paired_summary(pairs, exact=True) + assert plain["delta"]["mean"] == round(sum(deltas) / 3, 4) + assert exact["delta"]["mean"] == sum(deltas) / 3 + assert plain["p"] == round(exact["p"], 6) diff --git a/dokimi/tests/test_repo_ledger.py b/dokimi/tests/test_repo_ledger.py new file mode 100644 index 00000000..b741bd5f --- /dev/null +++ b/dokimi/tests/test_repo_ledger.py @@ -0,0 +1,511 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``repo compare --ledger``: rule verdicts over hand-written batch directories. No docker, no LLM.""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.ledger import KEEP, REVIEW, INSUFFICIENT, LedgerError, evaluate, format_ledger, load_ledger + +TODAY = date(2026, 10, 31) +BASE, ARM = "guidance:none", "guidance:rules" + + +def _cell(batch: Path, task: str, arm: str, *, success: bool | None = None, score: float | None = None, + model: str = "m-1") -> None: + rd = batch / f"{task}__{arm.replace(':', '-')}__{model}__r1" + rd.mkdir(parents=True) + score_task = score is not None + grade = {"status": "graded", "pass": None if score_task else success, "success": None if score_task else success, + "score": score, "score_only": score_task, "reasons": []} + rec = {"task": task, "arm": arm, "model": model, "rep": 1, "score_task": score_task, "grade": grade, + "harness_version": "1.0", "score_max": 1.0 if score_task else None, "model_resolved": None, + "agent": {"cost_usd": 0.1, "num_turns": 10, "timed_out": False, "is_error": False, "rc": None, + "over_budget": False, "network": "none"}} + (rd / "meta.json").write_text("{}") + (rd / "result.json").write_text(json.dumps(rec)) + (rd / "grade.json").write_text(json.dumps({"metrics": {}})) + + +def _gate_batch(root: Path, outcomes: dict[str, tuple[bool, bool]], model: str = "m-1") -> Path: + """One batch; ``outcomes`` maps task -> (baseline passes, rules arm passes).""" + batch = root / "run" + for task, (base, treat) in outcomes.items(): + _cell(batch, task, BASE, success=base, model=model) + _cell(batch, task, ARM, success=treat, model=model) + return batch + + +def _ledger(path: Path, rules: list[dict], **policy) -> Path: + data: dict = {"schema_version": "repo_ledger.v1", "rules": rules} + if policy: + data["policy"] = policy + path.write_text(json.dumps(data), encoding="utf-8") + return path + + +def _rule(rid: str, cases: list[str], reviewed: str = "2026-10-30", **extra) -> dict: + return {"id": rid, "hypothesis": f"{rid} changes the answer", "arm": ARM, "cases": cases, + "reviewed": reviewed, **extra} + + +def _ids(tasks: str, n: int) -> list[str]: + return [f"{tasks}_{i}" for i in range(n)] + + +def _verdicts(result: dict) -> dict[str, str]: + return {(e["rule"], e["model"]): e["verdict"] for e in result["rules"]} + + +def test_a_valid_ledger_loads_with_the_policy_defaults(tmp_path): + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"], source="AGENTS.md#commits", rule="Say why.")]) + ledger = load_ledger(path) + assert [r.id for r in ledger.rules] == ["R1"] + assert (ledger.alpha, ledger.review_within_days) == (0.05, 180) + assert ledger.rules[0].reviewed == date(2026, 10, 30) + assert ledger.rules[0].source == "AGENTS.md#commits" + + +@pytest.mark.parametrize("bad", [ + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": ["t1"], "reviewed": "2026-02-30"}, + {"id": "R1", "hypothesis": "h", "arm": "guidance:rules/", "cases": ["t1"], "reviewed": "2026-01-01"}, + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": [], "reviewed": "2026-01-01"}, + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": ["t1", "t1"], "reviewed": "2026-01-01"}, + {"id": "R 1", "hypothesis": "h", "arm": ARM, "cases": ["t1"], "reviewed": "2026-01-01"}, + {"id": "R1", "arm": ARM, "cases": ["t1"], "reviewed": "2026-01-01"}, + {"id": "R1", "hypothesis": "h", "arm": ARM, "cases": ["t1"], "reviewed": "2026-01-01", "verdict": "cut"}, +], ids=["impossible-date", "bad-selector", "no-cases", "repeated-case", "id-with-space", "no-hypothesis", + "unknown-key"]) +def test_a_rule_that_does_not_validate_is_refused(tmp_path, bad): + path = _ledger(tmp_path / "ledger.json", [bad]) + with pytest.raises(LedgerError): + load_ledger(path) + + +def test_a_repeated_rule_id_is_refused(tmp_path): + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"]), _rule("R1", ["t2"])]) + with pytest.raises(LedgerError, match="R1"): + load_ledger(path) + + +def test_a_rule_id_with_a_trailing_newline_is_refused_not_a_second_R1(tmp_path): + # The schema's `$` matches before a final newline in Python; "R1\n" would otherwise load as its own rule. + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"]), _rule("R1\n", ["t2"])]) + with pytest.raises(LedgerError, match="R1"): + load_ledger(path) + + +def test_an_unreadable_ledger_is_refused(tmp_path): + path = tmp_path / "ledger.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(LedgerError, match="cannot read"): + load_ledger(path) + + +def test_all_cases_improved_is_keep_and_all_worse_is_review(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks} | {u: (True, False) for u in _ids("u", 8)}) + ledger = load_ledger(_ledger(tmp_path / "ledger.json", [_rule("up", tasks), _rule("down", _ids("u", 8))])) + result = evaluate(ledger, [batch], BASE, today=TODAY) + assert _verdicts(result) == {("up", "m-1"): KEEP, ("down", "m-1"): REVIEW} + up = result["rules"][0] + assert (up["up"], up["down"], up["paired"]) == (8, 0, 8) + assert (up["baseline_mean"], up["treatment_mean"]) == (0.0, 1.0) + assert up["p_holm"] == pytest.approx(2 * 2 / 2**8, abs=1e-5) # Holm over the two rules; p is exact + + +def test_few_changed_cases_are_insufficient_evidence_and_the_reason_says_why(tmp_path): + tasks = _ids("t", 3) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY) + entry = result["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert "no outcome reaches p < 0.05" in entry["reason"] + + +def test_holm_corrects_the_rules_together(tmp_path): + # each rule alone reaches p = 2/64 = 0.031 (six cases all improved); two rules together give Holm 0.0625 + first, second = _ids("a", 6), _ids("b", 6) + batch = _gate_batch(tmp_path, {t: (False, True) for t in first + second}) + alone = evaluate(load_ledger(_ledger(tmp_path / "one.json", [_rule("A", first)])), [batch], BASE, today=TODAY) + both = evaluate(load_ledger(_ledger(tmp_path / "two.json", [_rule("A", first), _rule("B", second)])), + [batch], BASE, today=TODAY) + assert alone["rules"][0]["verdict"] == KEEP + assert [e["verdict"] for e in both["rules"]] == [INSUFFICIENT, INSUFFICIENT] + assert both["rules"][0]["p_holm"] == pytest.approx(2 * alone["rules"][0]["p"]) + assert both["family"] == 2 + + +def test_a_missing_case_is_listed_and_the_rest_are_measured(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + rule = _rule("R1", tasks + ["never_ran"]) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, today=TODAY)["rules"][0] + assert entry["missing"] == ["never_ran"] + assert (entry["linked"], entry["paired"], entry["verdict"]) == (9, 8, KEEP) + + +def test_a_rule_with_no_case_that_ran_is_not_measured(tmp_path): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["ghost"])])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert entry["reason"].startswith("not measured") + assert entry["missing"] == ["ghost"] + + +def test_an_arm_with_no_cell_on_the_linked_cases_is_not_measured(tmp_path): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + _cell(batch, "t2", BASE, success=False) # the baseline ran t2, the rules arm did not + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["t2"])])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert "not measured" in entry["reason"] + + +def test_a_tie_on_every_linked_case_has_nothing_to_test(tmp_path): + tasks = _ids("t", 4) + batch = _gate_batch(tmp_path, {t: (True, True) for t in tasks}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == INSUFFICIENT + assert "every linked case ties" in entry["reason"] + + +def test_gate_and_score_cases_in_one_rule_are_refused_as_mixed(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("g", 8)}) + for task in _ids("s", 8): + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=0.9) + rule = _rule("R1", _ids("g", 8) + _ids("s", 8)) + results = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, today=TODAY)["rules"] + assert {e["measure"] for e in results} == {"gate", "score"} + assert all(e["verdict"] == INSUFFICIENT and "mix gate and score" in e["reason"] for e in results) + + +def test_a_score_rule_is_measured_on_score(tmp_path): + batch = _score_batch(tmp_path) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("s", 8))])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["measure"] == "score" + assert entry["verdict"] == KEEP + assert entry["delta"]["mean"] == pytest.approx(0.7) + + +def _score_batch(root: Path) -> Path: + batch = root / "run" + for task in _ids("s", 8): + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=0.9) + return batch + + +def test_results_are_per_model(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}, model="m-1") + for task in _ids("t", 8): # a second model where the rule does nothing + _cell(batch, task, BASE, success=True, model="m-2") + _cell(batch, task, ARM, success=True, model="m-2") + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("t", 8))])), [batch], BASE, + today=TODAY) + assert _verdicts(result) == {("R1", "m-1"): KEEP, ("R1", "m-2"): INSUFFICIENT} + + +def test_a_review_older_than_the_policy_is_stale_but_keeps_its_verdict(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks, reviewed="2026-01-01")], review_within_days=180) + entry = evaluate(load_ledger(path), [batch], BASE, today=TODAY)["rules"][0] + assert entry["stale"] is True + assert entry["days_since_review"] == (TODAY - date(2026, 1, 1)).days + assert entry["verdict"] == KEEP + assert any("re-review" in w for w in entry["warnings"]) + + +def test_a_recent_review_is_not_stale(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["stale"] is False + assert not any("re-review" in w for w in entry["warnings"]) + + +def test_a_rule_can_name_its_own_baseline(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "t_0", "guidance:raw", success=True) # the rule's baseline has a cell for one of its cases only + rule = _rule("R1", tasks, baseline="guidance:raw") + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, today=TODAY)["rules"][0] + assert entry["baseline"] == "guidance:raw" + assert entry["paired"] == 1 # only t_0 has a cell in both arms + + +def test_format_names_the_verdict_the_hypothesis_and_the_stale_review(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks, reviewed="2026-01-01")]) + text = format_ledger(evaluate(load_ledger(path), [batch], BASE, today=TODAY, ledger_name="ledger.json")) + assert "R1: guidance:rules vs guidance:none, model m-1, gate: keep" in text + assert "hypothesis: R1 changes the answer" in text + assert "re-review" in text + + +def test_compare_with_ledger_adds_the_rule_results_to_the_json(tmp_path, capsys): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks)]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path), + "--json"]) + args.func(args) + data = json.loads(capsys.readouterr().out) + assert data["ledger"]["ledger"] == str(path) + assert [(e["rule"], e["verdict"]) for e in data["ledger"]["rules"]] == [("R1", KEEP)] + + +def test_compare_text_prints_the_rule_section(tmp_path, capsys): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks)]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path)]) + args.func(args) + out = capsys.readouterr().out + assert "rule ledger" in out + assert "R1: guidance:rules vs guidance:none, model m-1, gate: keep" in out + + +def test_a_bad_ledger_exits_with_usage_error(tmp_path, capsys): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + path = _ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"], reviewed="2026-02-30")]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path)]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "R1" in capsys.readouterr().err + + +def test_a_model_with_no_cell_on_the_rule_does_not_hide_the_others(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "elsewhere", BASE, success=True, model="m-2") # m-2 ran a task this rule does not link + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + models=["m-1", "m-2"], today=TODAY) + assert _verdicts(result) == {("R1", "m-1"): KEEP, ("R1", "m-2"): INSUFFICIENT} + (other,) = [e for e in result["rules"] if e["model"] == "m-2"] + assert "no compared cells for --model m-2" in other["reason"] + + +def test_a_linked_case_that_does_not_pair_is_named_on_the_rule(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "t_8", BASE, success=False) # the baseline's kind is gate, the rule's arm's is score + _cell(batch, "t_8", ARM, score=0.9) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks + ["t_8"])])), [batch], BASE, + today=TODAY)["rules"][0] + assert (entry["paired"], entry["linked"]) == (8, 9) + assert any("1 linked case(s) not paired" in w for w in entry["warnings"]) + + +def test_an_empty_ledger_path_is_refused_not_ignored(tmp_path, capsys): + batch = _gate_batch(tmp_path, {"t1": (False, True)}) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", ""]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + + +def test_a_model_whose_linked_cases_do_not_pair_is_named_when_others_pair(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + for task in tasks: # m-2 has a baseline cell on each linked case, and no arm cell + _cell(batch, task, BASE, success=True, model="m-2") + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY) + assert _verdicts(result) == {("R1", "m-1"): KEEP, ("R1", "m-2"): INSUFFICIENT} + + +def test_a_model_error_names_the_model(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}) + _cell(batch, "elsewhere", BASE, success=True, model="m-2") + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("t", 8))])), [batch], BASE, + models=["m-2"], today=TODAY)["rules"][0] + assert entry["model"] == "m-2" + assert entry["verdict"] == INSUFFICIENT + + +def test_a_stale_review_is_warned_once_per_result(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}) + rule = _rule("R1", _ids("t", 8), reviewed="2026-01-01") + results = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [rule])), [batch], BASE, models=["m-2", "m-3"], + today=TODAY)["rules"] + assert len(results) == 2 + for entry in results: + assert sum("re-review" in w for w in entry["warnings"]) == 1 + + +def test_a_review_dated_after_today_is_stale_with_a_date_warning(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks, reviewed="2036-10-01")])), + [batch], BASE, today=TODAY)["rules"][0] + assert entry["stale"] is True + assert any("after 2026-10-31" in w for w in entry["warnings"]) + + +def test_a_not_measured_model_does_not_carry_another_models_warning(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + _cell(batch, "t_0", BASE, success=True, model="m-2") # m-2 pairs nothing on this rule: its baseline only + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY) + (other,) = [e for e in result["rules"] if e["model"] == "m-2"] + assert other["verdict"] == INSUFFICIENT + assert not any("not paired" in w for w in other["warnings"]) + + +def test_a_repeated_model_is_compared_once(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + models=["m-1", "m-1"], today=TODAY) + assert [(e["model"], e["verdict"]) for e in result["rules"]] == [("m-1", KEEP)] + + +def test_a_mixed_rule_takes_no_slot_in_the_holm_family(tmp_path): + # A mixes gate and score (both improve); B is six gate cases, all improved. Alone B reaches p = 2/64. + gate_a, score_a, gate_b = _ids("ga", 6), _ids("sa", 6), _ids("gb", 6) + batch = _gate_batch(tmp_path, {t: (False, True) for t in gate_a + gate_b}) + for task in score_a: + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=0.9) + rules = [_rule("A", gate_a + score_a), _rule("B", gate_b)] + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", rules)), [batch], BASE, today=TODAY) + verdicts = {e["rule"]: e for e in result["rules"] if e["rule"] == "B"} + assert result["family"] == 1 + assert verdicts["B"]["verdict"] == KEEP + assert verdicts["B"]["p_holm"] == pytest.approx(2 / 2**6, abs=1e-5) + mixed = [e for e in result["rules"] if e["rule"] == "A"] + assert mixed and all(e["verdict"] == INSUFFICIENT and e["p_holm"] is None and "Holm family" in e["reason"] + for e in mixed) + + +@pytest.mark.parametrize("alpha", ["NaN", "Infinity", "-Infinity", "0", "1", "2", "-0.5"]) +def test_an_alpha_outside_the_open_interval_zero_one_is_refused(tmp_path, alpha): + # json.loads accepts NaN and Infinity; NaN compares false with every p, so it would turn any rule insufficient + path = tmp_path / "ledger.json" + path.write_text('{"schema_version":"repo_ledger.v1","policy":{"alpha":%s},"rules":[{"id":"R1",' + '"hypothesis":"h","arm":"guidance:rules","cases":["t1"],"reviewed":"2026-10-30"}]}' % alpha, + encoding="utf-8") + with pytest.raises(LedgerError, match="alpha"): + load_ledger(path) + + +def test_a_p_on_the_alpha_boundary_is_not_rounded_across_it(tmp_path): + # eight uniformly improved gate cases: the exact sign-flip p is 2 / 2**8 = 0.0078125, not 0.007812 + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + rules = [_rule("R1", tasks)] + below = evaluate(load_ledger(_ledger(tmp_path / "below.json", rules, alpha=0.0078124)), [batch], BASE, today=TODAY) + above = evaluate(load_ledger(_ledger(tmp_path / "above.json", rules, alpha=0.0078126)), [batch], BASE, today=TODAY) + assert _verdicts(below) == {("R1", "m-1"): INSUFFICIENT} + assert _verdicts(above) == {("R1", "m-1"): KEEP} + assert below["rules"][0]["p_holm"] == pytest.approx(2 / 2**8, abs=1e-12) + + +@pytest.mark.parametrize(("arm_score", "verdict"), [(0.20001, KEEP), (0.19999, REVIEW)]) +def test_a_small_score_change_keeps_its_sign_in_the_verdict(tmp_path, arm_score, verdict): + # the mean delta is 1e-5: rounded to four places it was 0.0, and the verdict read a tie + batch = tmp_path / "run" + for task in _ids("s", 8): + _cell(batch, task, BASE, score=0.2) + _cell(batch, task, ARM, score=arm_score) + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("s", 8))])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == verdict + assert entry["delta"]["mean"] == pytest.approx(arm_score - 0.2, abs=1e-12) + + +def test_a_score_rule_is_reported_as_a_mean_score_not_a_pass_rate(tmp_path): + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", _ids("s", 8))])), + [_score_batch(tmp_path)], BASE, today=TODAY) + text = format_ledger(result) + assert "mean score 0.200 -> 0.900" in text + assert "pass rate" not in text + + +def test_every_model_unpaired_on_the_rule_gets_its_own_result(tmp_path): + batch = tmp_path / "run" + _cell(batch, "t1", BASE, success=True, model="m-1") # the baseline ran t1 only on m-1 + _cell(batch, "t1", ARM, success=True, model="m-2") # the arm ran it only on m-2 + entries = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"])])), [batch], BASE, + today=TODAY)["rules"] + assert sorted(e["model"] for e in entries) == ["m-1", "m-2"] + assert all(e["verdict"] == INSUFFICIENT and e["error"] for e in entries) + + +def test_a_rule_with_no_finished_case_names_each_requested_model(tmp_path): + batch = _gate_batch(tmp_path, {t: (False, True) for t in _ids("t", 8)}) + entries = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["ghost"])])), [batch], BASE, + models=["m-1", "m-2", "m-1"], today=TODAY)["rules"] + assert [e["model"] for e in entries] == ["m-1", "m-2"] + assert all(e["verdict"] == INSUFFICIENT and "no linked case has a finished cell" in e["reason"] for e in entries) + + +def test_a_missing_cost_does_not_unpair_a_gate_case(tmp_path): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + meta = batch / f"{tasks[0]}__{BASE.replace(':', '-')}__m-1__r1" / "result.json" # cost gap, outcome paired + rec = json.loads(meta.read_text(encoding="utf-8")) + rec["agent"]["cost_usd"] = None + meta.write_text(json.dumps(rec), encoding="utf-8") + entry = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)])), [batch], BASE, + today=TODAY)["rules"][0] + assert entry["verdict"] == KEEP + assert entry["paired"] == 8 + assert not any("not paired" in w for w in entry["warnings"]) + + +@pytest.mark.parametrize("absent", ["arm", "baseline"]) +def test_a_rule_whose_arm_or_baseline_is_absent_names_each_model(tmp_path, absent): + batch = tmp_path / "run" + ran_t1 = BASE if absent == "arm" else ARM # the side that ran the linked task, for both models + other = ARM if absent == "arm" else BASE # the absent side ran only an unrelated task + for model in ("m-1", "m-2"): + _cell(batch, "t1", ran_t1, success=True, model=model) + _cell(batch, "t9", other, success=True, model=model) + entries = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", ["t1"])])), [batch], BASE, + today=TODAY)["rules"] + assert sorted(e["model"] for e in entries) == ["m-1", "m-2"] + assert all(e["verdict"] == INSUFFICIENT and e["error"] for e in entries) + + +def test_a_p_exactly_on_alpha_is_insufficient_evidence(tmp_path): + # eight improved gate cases: the exact sign-flip p is 2 / 2**8, which is exactly representable + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) + result = evaluate(load_ledger(_ledger(tmp_path / "ledger.json", [_rule("R1", tasks)], alpha=2 / 2**8)), + [batch], BASE, today=TODAY) + assert result["rules"][0]["p_holm"] == 2 / 2**8 + assert _verdicts(result) == {("R1", "m-1"): INSUFFICIENT} + + +def test_compare_with_ledger_and_model_judges_only_that_model(tmp_path, capsys): + tasks = _ids("t", 8) + batch = _gate_batch(tmp_path, {t: (False, True) for t in tasks}) # m-1 improves on every case + for t in tasks: # m-2 does not + _cell(batch, t, BASE, success=False, model="m-2") + _cell(batch, t, ARM, success=False, model="m-2") + path = _ledger(tmp_path / "ledger.json", [_rule("R1", tasks)]) + args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", BASE, "--ledger", str(path), + "--model", "m-1", "--json"]) + args.func(args) + ledger = json.loads(capsys.readouterr().out)["ledger"] + assert {e["model"] for e in ledger["rules"]} == {"m-1"} + assert ledger["family"] == 1 # the Holm family counts the one model that was asked for diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9bd3e9d9..cdff1ea9 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -49,6 +49,7 @@ "repo_budget.v1": ({}, "toy", None, None), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "repo_ledger.v1": ({}, None, None, None), "redteam.v1": ({}, "questions.json", None, None), "need_graph.v1": ({}, "dpnp-io", None, None), "eval_coverage.v1": ({}, "dpnp-io", None, None), diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 3e607b07..1e17e7e1 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -38,6 +38,7 @@ "subject_scorecard.v2": {"cells": 0}, "skill_evidence.v1": {"results": 2, "warnings": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "repo_ledger.v1": {"rules": 2}, "redteam.v1": {"violations": 2, "questions": 2}, "eval_coverage.v1": {"gaps": 1, "evals": 2, "problems": 0}, "need_graph.v1": {"nodes": 3, "edges": 1},