From 989ef4437dcc16d6c148aabc919fffd2b04499ee Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 16:32:51 +0000 Subject: [PATCH] test(eval): pin check error and baseline delta; fix harness docs Settles three points the reviews of #116 and #280 left open. #116: preparing-a-repository.md said no loading behavior was measured for Codex and Copilot, while the built-in codex and copilot profiles record measurements on codex-cli 0.160 and Copilot CLI 1.0.91 in their guidance.notes. The sentence now points at those profiles; a harness without a built-in profile still needs its own harness_profile.v1 and a dokimi repo reach measurement. #280: two outputs had no test that failed when they changed. - A judge check whose call fails carries its "error" in the cell's checks entry, and a graded check has no "error" key (CheckResult.as_dict); the new test goes through eval reference's score(). - The baseline arm's per_arm delta_vs_baseline is present and null, not missing and not 0.0 (ArmRunner._summarize). Signed-off-by: Anton Volkov --- dokimi/docs/preparing-a-repository.md | 6 ++++-- dokimi/tests/test_arm_cell_errors.py | 3 +++ dokimi/tests/test_reference_checks.py | 9 +++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/preparing-a-repository.md b/dokimi/docs/preparing-a-repository.md index e52910f23..a2c1c842d 100644 --- a/dokimi/docs/preparing-a-repository.md +++ b/dokimi/docs/preparing-a-repository.md @@ -199,8 +199,10 @@ Recurring review comments are review knowledge a repository already has. |---|---| | Claude Code 2.1.286 (measured 2026-10) | `CLAUDE.md`. `AGENTS.md` only when no `CLAUDE.md` is in its start-up chain. Subdirectory files lazily, when it reads a file there (a subdirectory `AGENTS.md` only if no `CLAUDE.md` is in the chain). | -Other harnesses (Codex, Copilot and others) are not listed because no loading -behavior was measured for them. Each needs its own `harness_profile.v1` and a +Codex (codex-cli 0.160) and Copilot CLI (1.0.91) were measured too: what each +loaded is in the `guidance.notes` of the built-in `codex` and `copilot` profiles +(`dokimi/repos/profiles/codex.yaml`, `dokimi/repos/profiles/copilot.yaml`). A +harness without a built-in profile needs its own `harness_profile.v1` and a measurement with `dokimi repo reach`; the `text`-trace limit below applies. Put a `CLAUDE.md` containing `@AGENTS.md` next to each `AGENTS.md`, including in diff --git a/dokimi/tests/test_arm_cell_errors.py b/dokimi/tests/test_arm_cell_errors.py index 65fa331d6..6debfb543 100644 --- a/dokimi/tests/test_arm_cell_errors.py +++ b/dokimi/tests/test_arm_cell_errors.py @@ -117,6 +117,9 @@ def test_delta_vs_baseline_is_paired_over_the_questions_both_arms_scored(): summary = runner._summarize(evaluations, "baseline") assert summary["per_arm"]["skill"]["delta_vs_baseline"] == -10.0 assert summary["per_arm"]["skill"]["avg_aggregate"] == 55.0 # the unpaired mean is still reported + # The baseline has no delta against itself: the key is there, as null, not 0.0. + assert "delta_vs_baseline" in summary["per_arm"]["baseline"] + assert summary["per_arm"]["baseline"]["delta_vs_baseline"] is None @pytest.mark.parametrize("reply", ["", " \n\t", None]) diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index d9a18b734..983b1a09c 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -474,6 +474,15 @@ def test_eval_reference_counts_a_judge_failure_as_an_error_for_every_arm(): assert (cell["passed"], cell["aggregate"], cell["error"]) == (None, None, "judge: TimeoutError") +def test_the_failed_judge_check_names_its_error_in_the_cell_and_the_graded_check_has_none(): + data = {"baseline_arm": "baseline", "arms": ["baseline"], "answers": [ + {"question_id": "q1", "question_text": "Capital?", "arms": {"baseline": {"answer": "Paris"}}}]} + result = score(data, [{"id": "q1", "question": "Capital?", "grading": _MIXED}], ask=_judge_raises) + match, judge = result.evaluations[0]["scores"]["baseline"]["checks"] + assert (judge["type"], judge["error"]) == ("judge", "judge: TimeoutError") + assert match == {"type": "match", "passed": True, "score": 1.0} + + def test_a_reference_cell_carries_its_answer_error_as_an_arm_cell_does(tmp_path): data = {"baseline_arm": "baseline", "arms": ["baseline", "skill"], "answers": [ {"question_id": "q1", "question_text": "Capital?",