From 137a80dbf9cd9454eead83e8b6aa17d2a017018c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:44:46 -0700 Subject: [PATCH 01/29] feat(repo): effective-instructions receipt for repo reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `repo reach --format receipt` prints, and writes to effective_instructions.json in --out, the effective-instructions/v0 receipt (agentsmd/agents.md#213) of each probed scenario. It joins the probe's marker tokens with budget's start-up rules (Claude Code only): per source the path, sha256, scope, expected state, observed state, observation_method, the harness name and version, the displacements seen in the run, and the unknowns. followed is always unknown. The mapping and its limits are in docs/repo-evaluation-contract.md §7.1. The receipt is validated against schemas/effective_instructions.v1.json and registered in artifacts.SCHEMA_VERSIONS. The default output of repo reach is unchanged. Signed-off-by: Nikolay Petrov --- ...napetrov_effective-instructions-receipt.md | 8 + dokimi/docs/reference/cli-interface.json | 20 ++ dokimi/docs/repo-evaluation-contract.md | 64 +++++ dokimi/dokimi/artifacts.py | 4 +- dokimi/dokimi/commands/repo.py | 30 ++- dokimi/dokimi/repos/receipt.py | 229 ++++++++++++++++++ dokimi/dokimi/runs/kinds.py | 2 + .../schemas/effective_instructions.v1.json | 89 +++++++ .../artifacts/effective_instructions.v1.json | 104 ++++++++ dokimi/tests/test_repo_receipt.py | 197 +++++++++++++++ dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + 12 files changed, 745 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md create mode 100644 dokimi/dokimi/repos/receipt.py create mode 100644 dokimi/dokimi/schemas/effective_instructions.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/effective_instructions.v1.json create mode 100644 dokimi/tests/test_repo_receipt.py diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md new file mode 100644 index 000000000..0261c8a5e --- /dev/null +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -0,0 +1,8 @@ + + +### Added + +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`prompt_capture`), the harness's name and version, the displacements seen in the run, and the unknowns. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..550c2ec10 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10751,6 +10751,25 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": [ + "'matrix'", + "'receipt'" + ], + "const": null, + "default": "matrix", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--format", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -12884,6 +12903,7 @@ "'answers'", "'arms'", "'cli_routes'", + "'effective_instructions'", "'eval'", "'explore_runs'", "'guidance_reach'", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c2b873922..856b4616f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1049,6 +1049,70 @@ per scenario). token was in the request bytes. This settles cells where the model under-reports, which is how a small model produces false `truncated` cells. +### 7.1 Effective-instructions receipt (`effective_instructions.v1`) + +`repo reach --format receipt` prints, and writes to `effective_instructions.json` +in `--out`, a receipt in the shape of `effective-instructions/v0` +(agentsmd/agents.md#213). It is built from the reach payload and from the +start-up chain of `repo budget` (`repos/budget.py`). It makes no +LLM call. The schema is `schemas/effective_instructions.v1.json`; the code is +`repos/receipt.py`. + +One receipt is written per reach scenario, with `run_id` `:`. +Each source carries: + +| Receipt field | From | Meaning | +|---|---|---| +| `path`, `alias_of` | reach `files`, `aliases` | a link is `adapter` scope and takes its target's tokens | +| `sha256` | the text dokimi reads (LF-normalised UTF-8) | null when the file is not readable as text | +| `scope` | path | `repo` at the root or in `.claude/`; `subtree` for a nested AGENTS.md, CLAUDE.md or CLAUDE.local.md; else `unknown` | +| `expected` | budget `when` | see the table below | +| `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | +| `observation_method` | | `prompt_capture` for every observed source; `unknown` for an error | +| `states.accessed` | the Read tool | `true` if a Read-tool read was seen, `false` with tools off, else `unknown` | +| `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | +| `states.loaded` | the class | `true` for loaded, `false` for not_loaded, else `unknown`; a fetched file is not loaded | +| `states.followed` | | always `unknown` (the schema's `const`) | + +`runtime` and `model` are the harness's own name and version, as reach records +them. `observation.method` is `prompt_capture`: the model reports the tokens it +can see, and the request sent to the provider is not captured. A reported token +is evidence of loading, not a copy of the bytes received. + +**Expected state.** budget's start-up rules are Claude Code's, so they apply +only to `claude-code` runs; for other harnesses every `expected` is `unknown` +and the receipt says so. + +| budget `when` | `expected` | Reason | +|---|---|---| +| `start` | `true` | loaded at start-up in every run | +| `off` | `false` | a root CLAUDE\*.md turns AGENTS.md off | +| `lazy`, run directory is the file's directory or inside it | `true` | loaded when the work reaches that directory | +| `lazy`, run directory outside it | `false` | not reached by this run | +| `on demand` or absent | `unknown` | budget does not say | + +**Precedence.** `precedence_observed` lists a displacement only: a root +CLAUDE\*.md (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md`) observed +loaded in a run, with an `off` AGENTS.md observed not_loaded in that run. It +does not give an order of entry; budget's displacement rule is applied as +written, so nested AGENTS.md files that budget marks `off` appear too. + +**What the marker method can and cannot show.** It shows which tokens were +reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as +observations of that run. It cannot show: + +- that an instruction was followed (`followed` is always `unknown`); +- a read through a shell (`cat`, `rg`, `grep`), which the `Read` tool check + does not see; +- when a source entered the request, or in what order (no timestamps); +- start-up imports and skills from budget, which reach does not probe: they + appear with `observed: unknown` and a run-level unknown saying so; +- anything a model under-reports; a prose-heavy answer is flagged. + +Each run lists its own `unknowns` (no usable answer, a source with one token +reported, prose above the limit, unseeded tokens). The receipt-level `unknowns` +repeat the limits above. + ## 8. L0 scan (`repo_readiness.v1`) `repo scan ` checks every mechanically checkable claim in the guidance diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..893731d14 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -12,7 +12,8 @@ what moves it), ``repo_environment`` (static inspection of what a repository offers an agent), ``plan`` (the cells a run promises to measure), ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts -into an agent's context) and ``task_inspect`` (which eval-task format each +into an agent's context), ``effective_instructions`` (the effective-instructions/v0 +receipt of a reach probe) and ``task_inspect`` (which eval-task format each path under a tree is). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay @@ -56,6 +57,7 @@ "plan": "plan.v1", "result": "result.v1", "repo_budget": "repo_budget.v1", + "effective_instructions": "effective_instructions.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", } diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 88f64ad09..7a526da73 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -269,7 +269,6 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: from dokimi.repos.source import SourceArgumentError from dokimi.repos.reach import ( ReachError, - format_matrix, planned, probe_files, run_reach, @@ -325,12 +324,34 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) path = save_reach(payload, out_dir) - print(format_matrix(payload)) - print(f"wrote {path}") + _print_reach_output(tree, profile, payload, out_dir, path, receipt=args.format == "receipt") if any(s["error"] for s in payload["scenarios"]): sys.exit(1) +def _print_reach_output(tree, profile, payload: dict, out_dir: Path, path: Path, receipt: bool) -> None: + """The matrix on stdout; with ``receipt``, the effective-instructions receipt on stdout and the note on stderr.""" + from dokimi.repos.reach import format_matrix + + if receipt: + _print_receipt(tree, profile, payload, out_dir) + print(f"wrote {path}", file=sys.stderr) + else: + print(format_matrix(payload)) + print(f"wrote {path}") + + +def _print_receipt(tree, profile, payload: dict, out_dir: Path) -> None: + """The effective-instructions receipt of a probe: budget's start-up chain, reach's observed states.""" + from dokimi.artifacts import save_artifact + from dokimi.repos.budget import budget + from dokimi.repos.receipt import build_receipt + + receipt = build_receipt(payload, budget(tree, profile.guidance_names, profile.import_max_depth), tree) + written = save_artifact("effective_instructions", receipt, out_dir / "effective_instructions.json") + print(json.dumps(written, indent=2)) + + def cmd_repo_budget(args: argparse.Namespace) -> None: """What the repository puts into an agent's context at start-up, lazily and on demand.""" from dokimi.artifacts import save_artifact, stamp @@ -535,6 +556,9 @@ def register(sub, positive_int) -> None: reach_p.add_argument("--budget", type=float, default=0.10, help="Per-scenario USD cap (default: 0.10)") reach_p.add_argument("--timeout", type=positive_int, default=300, help="Per-scenario seconds") reach_p.add_argument("--keep", action="store_true", help="Keep the temporary workspace") + reach_p.add_argument("--format", choices=["matrix", "receipt"], default="matrix", + help="matrix: the table (default). receipt: print the effective-instructions/v0 receipt as JSON and " + "write it to effective_instructions.json in --out (docs/repo-evaluation-contract.md §7.1)") reach_p.add_argument("--dry-run", action="store_true", dest="dry_run", help="Print the files and the exact argv of every scenario; run nothing") reach_p.add_argument("--cache-dir", default=None, dest="cache_dir", help=cache_help) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py new file mode 100644 index 000000000..04531dc4c --- /dev/null +++ b/dokimi/dokimi/repos/receipt.py @@ -0,0 +1,229 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The effective-instructions receipt (``effective-instructions/v0``, agentsmd/agents.md#213) of one reach probe. + +Built from a ``guidance_reach.v1`` payload and the start-up chain of ``repo_budget.v1``. Per run (one reach +scenario) and per guidance source it gives the expected state (budget's start-up rules), the observed state (the +probe's marker tokens) and the states ``accessed``, ``selected``, ``loaded`` and ``followed``. What the probe +cannot establish is ``unknown``; ``followed`` always is. docs/repo-evaluation-contract.md §7.1 holds the mapping +and its limits. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + +from .reach import PROSE_LIMIT +from .source import RepoTree + +SCHEMA = "effective-instructions/v0" +SCHEMA_VERSION = "effective_instructions.v1" +UNKNOWN = "unknown" +# The probe's method: the model reports the marker tokens it can see; the request itself is not captured. +METHOD = "prompt_capture" +# Root-level files that turn AGENTS.md off in Claude Code (budget.py): the displacing set. +CLAUDE_ROOT = ("CLAUDE.md", "CLAUDE.local.md", ".claude/CLAUDE.md") +ROOT_NESTED = (".claude/CLAUDE.md", ".claude/AGENTS.md") # project memory in .claude/ applies repo-wide +SUBTREE_NAMES = ("AGENTS.md", "CLAUDE.md", "CLAUDE.local.md") +_STATE = {"loaded": "loaded", "fetched": "fetched", "not_loaded": "not_loaded"} +_LOADED: dict[str | None, bool] = {"loaded": True, "not_loaded": False} +DETAIL = ("the model is asked to report the marker tokens it can see; the request sent to the provider is not " + "captured, so a reported token is evidence of loading, not a copy of the bytes it received") +NOT_PROBED = "start-up imports and skills from repo_budget are not probed by reach; their observed state is unknown" + + +@dataclass(frozen=True) +class _Run: + row: dict[str, Any] # the reach scenario row + records: dict[tuple[str, str], dict[str, Any]] # (file, scenario id) -> record + when_of: dict[str, str] # canonical path -> budget's when + digests: dict[str, str | None] + workdir: str # the directory the run works in + rules: bool # budget's rules apply (Claude Code) + + +def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) -> dict[str, Any]: + """The ``effective_instructions.v1`` artifact: one ``effective-instructions/v0`` receipt per reach scenario.""" + harness = reach["harness"] + rules = harness["name"] == "claude-code" # budget's start-up rules are Claude Code's + when_of = _budget_when(reach, budget) if rules else {} + start_up = [p for p, when in when_of.items() if when in ("start", "off") and p not in _known(reach)] + digests = source_digests(tree, [p for p, _ in _probed(reach)] + start_up) + records = {(r["file"], r["scenario"]): r for r in reach["records"]} + runtime = {"name": harness["name"], "version": harness["version"]} + receipts = [] + for row in reach["scenarios"]: + run = _Run(row, records, when_of, digests, _workdir(row), rules) + receipts.append(_receipt(reach, run, runtime, start_up)) + return { + "schema_version": SCHEMA_VERSION, + "generated_at": reach["generated_at"], + "repo": reach["repo"], + "runtime": runtime, + "model": reach["model"], + "observation": {"method": METHOD, "detail": DETAIL}, + "unknowns": _global_unknowns(harness["name"], rules), + "receipts": receipts, + } + + +def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: + """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" + out: dict[str, str | None] = {} + for path in dict.fromkeys(paths): + try: + out[path] = hashlib.sha256(tree.read_text(path).encode("utf-8")).hexdigest() + except (OSError, ValueError, RuntimeError): # SourceError and UnicodeDecodeError are ValueErrors + out[path] = None + return out + + +def _probed(reach: dict[str, Any]) -> list[tuple[str, str | None]]: + """(path, alias_of) for every probed path: each guidance file, then the links to it.""" + out: list[tuple[str, str | None]] = [] + for item in reach["files"]: + out.append((item["path"], None)) + out.extend((alias, item["path"]) for alias in item.get("aliases", [])) + return out + + +def _known(reach: dict[str, Any]) -> set[str]: + return {path for path, _ in _probed(reach)} + + +def _budget_when(reach: dict[str, Any], budget: dict[str, Any]) -> dict[str, str]: + """budget's ``when`` of each guidance file and import, keyed by the file the path resolves to.""" + target_of = {alias: item["path"] for item in reach["files"] for alias in item.get("aliases", [])} + return {target_of.get(e["path"], e["path"]): e["when"] + for e in budget["entries"] if e["kind"] in ("guidance", "import")} + + +def _dir(path: str) -> str: + parent = str(PurePosixPath(path).parent) + return "" if parent == "." else parent + + +def _workdir(row: dict[str, Any]) -> str: + """The directory a run works in: the one holding the file it is told to read, else where it starts.""" + return _dir(row["read"]) if row.get("read") else row["cwd"] + + +def _receipt(reach: dict[str, Any], run: _Run, runtime: dict[str, str], start_up: list[str]) -> dict[str, Any]: + sources = [_source(path, alias_of, run) for path, alias_of in _probed(reach)] + sources += [_budget_only(path, run) for path in start_up] + off = {p for p, when in run.when_of.items() if when == "off"} + row = run.row + return { + "schema": SCHEMA, + "run_id": f"{reach['salt']}:{row['id']}", + "runtime": runtime, + "cwd": row["cwd"] or ".", + "target": row.get("read"), + "sources": sources, + "precedence_observed": _precedence(sources, off), + "unknowns": _run_unknowns(row, sources), + } + + +def _source(path: str, alias_of: str | None, run: _Run) -> dict[str, Any]: + target = alias_of or path + rec = run.records.get((target, run.row["id"])) + cls = rec["class"] if rec else None + observed, method = _observation(cls) + return { + "path": path, + **({"alias_of": alias_of} if alias_of else {}), + "sha256": run.digests.get(path), + "scope": _scope(path, alias_of is not None), + "expected": _expected(target, run.when_of.get(target), run), + "observed": observed, + "observation_method": method, + "states": _states(cls, run.row["tools"] == "none", bool(rec and rec["read_by_tool"])), + } + + +def _budget_only(path: str, run: _Run) -> dict[str, Any]: + """A start-up import or guidance file budget found and reach did not probe: expected, never observed.""" + return {"path": path, "sha256": run.digests.get(path), "scope": _scope(path, False), + "expected": _expected(path, run.when_of[path], run), "observed": UNKNOWN, + "observation_method": UNKNOWN, "states": _states(None, False, False)} + + +def _observation(cls: str | None) -> tuple[str, str]: + """(observed state, observation_method) of one reach class. A truncated cell saw one token: an observation, + no verdict.""" + if cls in _STATE: + return _STATE[cls], METHOD + if cls == "truncated": + return UNKNOWN, METHOD + return UNKNOWN, UNKNOWN # error, or no record (not probed in this run) + + +def _states(cls: str | None, tools_off: bool, read_by_tool: bool) -> dict[str, Any]: + """accessed: a Read-tool read was seen (false with no tools at all); selected: the probe saw the source enter the + request, by the harness or by the agent's own read; loaded: at the harness's own load step, not through a read.""" + accessed: bool | str = True if read_by_tool else (False if tools_off else UNKNOWN) + selected: bool | str = True if cls in ("loaded", "fetched") else UNKNOWN + return {"accessed": accessed, "selected": selected, "loaded": _LOADED.get(cls, UNKNOWN), "followed": UNKNOWN} + + +def _expected(target: str, when: str | None, run: _Run) -> bool | str: + """Start-up files are expected in every run; off ones in none; a nested lazy file in a run whose directory is its + subtree (or inside it), since the harness loads it when the work reaches that directory.""" + if not run.rules or when is None: + return UNKNOWN + if when in ("start", "off"): + return when == "start" + if when == "lazy": + d = _dir(target) + return run.workdir == d or run.workdir.startswith(d + "/") + return UNKNOWN + + +def _scope(path: str, alias: bool) -> str: + if alias: + return "adapter" # a link to another guidance file: its tokens are the target's + if "/" not in path or path in ROOT_NESTED: + return "repo" + return "subtree" if path.rsplit("/", 1)[-1] in SUBTREE_NAMES else UNKNOWN + + +def _precedence(sources: list[dict[str, Any]], off: set[str]) -> list[dict[str, str]]: + """A file budget says a root CLAUDE* file displaces, seen not loaded while that file was seen loaded in the run.""" + winners = sorted(s["path"] for s in sources if s["path"] in CLAUDE_ROOT and s["observed"] == "loaded") + if not winners: + return [] + return [{"winner": w, "displaced": s["path"]} + for s in sources if (s.get("alias_of") or s["path"]) in off and s["observed"] == "not_loaded" + for w in winners] + + +def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]]) -> list[str]: + out = [] + if row.get("error"): + out.append("no usable answer from this run: every observed state is unknown") + out += [f"{s['path']}: one of its two marker tokens was reported; loaded is not established" + for s in sources if s["observed"] == UNKNOWN and s["observation_method"] == METHOD] + if (row.get("prose") or 0) > PROSE_LIMIT: + out.append(f"answered with {row['prose']} words besides tokens; its not_loaded states may be under-reports") + if row.get("unknown"): + out.append("reported tokens that were never seeded: " + ", ".join(row["unknown"])) + return out + + +def _global_unknowns(name: str, rules: bool) -> list[str]: + out = [ + "followed is not observed: no marker shows what the model did with a loaded source", + "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", + "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", + "no load timestamps: the probe does not record when a source entered the request", + "the order in which sources entered the request is not observed; precedence_observed lists displacements only", + NOT_PROBED, + ] + if not rules: + out.append(f"budget's start-up rules are Claude Code's: no source has an expected state for {name}") + return out diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..b9b3d2cb1 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -63,6 +63,8 @@ class KindFields: counts=("cells",)), "result": KindFields(time=("generated_at",), counts=("cells",)), "repo_budget": KindFields(subject=("tree",), counts=("entries",)), + "effective_instructions": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",), + counts=("receipts",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), } diff --git a/dokimi/dokimi/schemas/effective_instructions.v1.json b/dokimi/dokimi/schemas/effective_instructions.v1.json new file mode 100644 index 000000000..b31064f18 --- /dev/null +++ b/dokimi/dokimi/schemas/effective_instructions.v1.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/effective_instructions.v1.json", + "title": "Effective-instructions receipt v1", + "description": "The effective-instructions/v0 receipt (agentsmd/agents.md#213) of one `repo reach` probe (docs/repo-evaluation-contract.md §7.1): one receipt per scenario, each source's expected state from budget's start-up rules and its observed state from the probe's marker tokens. `followed` is always unknown: reach never proves it.", + "type": "object", + "required": ["schema_version", "generated_at", "repo", "runtime", "model", "observation", "unknowns", "receipts"], + "properties": { + "schema_version": {"const": "effective_instructions.v1"}, + "generated_at": {"type": "string"}, + "repo": { + "type": "object", + "required": ["id", "tree"], + "properties": { + "id": {"type": "string"}, + "tree": {"type": "string"}, + "sha": {"type": ["string", "null"]}, + "manifest": {"type": "string"} + } + }, + "runtime": {"$ref": "#/$defs/runtime"}, + "model": {"type": "string"}, + "observation": { + "type": "object", + "required": ["method", "detail"], + "properties": { + "method": {"enum": ["prompt_capture"], "description": "The probe's method for every observed source."}, + "detail": {"type": "string"} + } + }, + "unknowns": {"type": "array", "items": {"type": "string"}}, + "receipts": { + "type": "array", + "items": { + "type": "object", + "required": ["schema", "run_id", "runtime", "cwd", "target", "sources", "precedence_observed", "unknowns"], + "properties": { + "schema": {"const": "effective-instructions/v0"}, + "run_id": {"type": "string", "description": "The reach salt and the scenario id, so the receipt names its record in reach.json."}, + "runtime": {"$ref": "#/$defs/runtime"}, + "cwd": {"type": "string", "description": "The directory the run started in; `.` is the repository root."}, + "target": {"type": ["string", "null"], "description": "The file the run was told to read (the probe file of a read= scenario), else null."}, + "sources": {"type": "array", "items": {"$ref": "#/$defs/source"}}, + "precedence_observed": { + "type": "array", + "description": "Displacements seen in this run: a root CLAUDE* file loaded while the AGENTS.md it displaces was not.", + "items": { + "type": "object", + "required": ["winner", "displaced"], + "properties": {"winner": {"type": "string"}, "displaced": {"type": "string"}} + } + }, + "unknowns": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "$defs": { + "runtime": { + "type": "object", + "required": ["name", "version"], + "properties": {"name": {"type": "string"}, "version": {"type": "string"}} + }, + "tristate": {"enum": [true, false, "unknown"]}, + "source": { + "type": "object", + "required": ["path", "sha256", "scope", "expected", "observed", "observation_method", "states"], + "properties": { + "path": {"type": "string"}, + "alias_of": {"type": "string", "description": "Set when the path is a link to another probed file; its tokens and its observation are the target's."}, + "sha256": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$", "description": "Null when the file cannot be read as text."}, + "scope": {"enum": ["repo", "subtree", "user", "adapter", "unknown"]}, + "expected": {"$ref": "#/$defs/tristate", "description": "Budget's start-up rules: true for start, false for off, true for a lazy file whose subtree the run works in. unknown without Claude Code's rules."}, + "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, + "observation_method": {"enum": ["runtime_event", "prompt_capture", "hook", "unknown"]}, + "states": { + "type": "object", + "required": ["accessed", "selected", "loaded", "followed"], + "properties": { + "accessed": {"$ref": "#/$defs/tristate"}, + "selected": {"$ref": "#/$defs/tristate"}, + "loaded": {"$ref": "#/$defs/tristate"}, + "followed": {"const": "unknown", "description": "Reach never proves that an instruction was followed."} + } + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json new file mode 100644 index 000000000..eee6afc56 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json @@ -0,0 +1,104 @@ +{ + "schema_version": "effective_instructions.v1", + "generated_at": "2026-10-31T00:00:00+00:00", + "repo": { + "id": "toy", + "tree": "toy", + "sha": null, + "manifest": "evals/repo-eval.yaml" + }, + "runtime": { + "name": "claude-code", + "version": "2.1.0" + }, + "model": "m", + "observation": { + "method": "prompt_capture", + "detail": "the model is asked to report the marker tokens it can see; the request sent to the provider is not captured, so a reported token is evidence of loading, not a copy of the bytes it received" + }, + "unknowns": [ + "followed is not observed: no marker shows what the model did with a loaded source", + "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", + "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", + "no load timestamps: the probe does not record when a source entered the request", + "the order in which sources entered the request is not observed; precedence_observed lists displacements only", + "start-up imports and skills from repo_budget are not probed by reach; their observed state is unknown" + ], + "receipts": [ + { + "schema": "effective-instructions/v0", + "run_id": "salt:cwd=.", + "runtime": { + "name": "claude-code", + "version": "2.1.0" + }, + "cwd": ".", + "target": null, + "sources": [ + { + "path": "AGENTS.md", + "sha256": "87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7", + "scope": "repo", + "expected": false, + "observed": "not_loaded", + "observation_method": "prompt_capture", + "states": { + "accessed": false, + "selected": "unknown", + "loaded": false, + "followed": "unknown" + } + }, + { + "path": "CLAUDE.md", + "sha256": "a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478", + "scope": "repo", + "expected": true, + "observed": "loaded", + "observation_method": "prompt_capture", + "states": { + "accessed": false, + "selected": true, + "loaded": true, + "followed": "unknown" + } + }, + { + "path": "sub/AGENTS.md", + "sha256": "cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556", + "scope": "subtree", + "expected": false, + "observed": "not_loaded", + "observation_method": "prompt_capture", + "states": { + "accessed": false, + "selected": "unknown", + "loaded": false, + "followed": "unknown" + } + }, + { + "path": "docs/style.md", + "sha256": "8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be", + "scope": "unknown", + "expected": true, + "observed": "unknown", + "observation_method": "unknown", + "states": { + "accessed": "unknown", + "selected": "unknown", + "loaded": "unknown", + "followed": "unknown" + } + } + ], + "precedence_observed": [ + { + "winner": "CLAUDE.md", + "displaced": "AGENTS.md" + } + ], + "unknowns": [] + } + ] +} diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py new file mode 100644 index 000000000..ebd8c6833 --- /dev/null +++ b/dokimi/tests/test_repo_receipt.py @@ -0,0 +1,197 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The effective-instructions receipt of a reach probe (no LLM): its mapping, its schema and the CLI.""" + +from __future__ import annotations + +import hashlib +import json + +import pytest + +from dokimi.artifacts import ArtifactValidationError, validate_artifact +from dokimi.cli import build_parser +from dokimi.repos import WorkTree +from dokimi.repos.receipt import build_receipt, source_digests +from tests.test_repo_reach import _profile, _repo + +ROOT_ROW = {"id": "cwd=.", "cwd": "", "tools": "none", "read": None, "error": None, "prose": 0, "unknown": []} +SUB_ROW = {"id": "read=sub", "cwd": "", "tools": "read", "read": "sub/REACH_PROBE.txt", "error": None, + "prose": 0, "unknown": []} +BUDGET = {"entries": [ + {"path": "CLAUDE.md", "kind": "guidance", "when": "start", "chars": 10}, + {"path": "AGENTS.md", "kind": "guidance", "when": "off", "chars": 10}, + {"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "docs/style.md", "kind": "import", "when": "start", "chars": 10}, +]} + + +def _record(file: str, scenario: str, cls: str, read_by_tool: bool = False) -> dict: + return {"file": file, "scenario": scenario, "harness": "claude-code", "harness_version": "2.1.0", + "model": "m", "head": cls in ("loaded", "fetched", "truncated"), "tail": cls in ("loaded", "fetched"), + "read_by_tool": read_by_tool, "class": cls} + + +def _reach(records: list[dict], scenarios: list[dict], harness: str = "claude-code", + files: list[dict] | None = None) -> dict: + files = files or [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + return {"generated_at": "2026-10-31T00:00:00+00:00", "repo": {"id": "toy", "tree": "toy", "sha": None, + "manifest": "evals/repo-eval.yaml"}, "harness": {"name": harness, "version": "2.1.0"}, "model": "m", + "salt": "salt", "files": files, "scenarios": scenarios, "records": records, "summary": {}} + + +def _tree(tmp_path, files: dict[str, str | bytes]) -> WorkTree: + for rel, text in files.items(): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + if isinstance(text, bytes): + (tmp_path / rel).write_bytes(text) + else: + (tmp_path / rel).write_text(text, encoding="utf-8") + return WorkTree(tmp_path) + + +def _sources(receipt: dict, run_id: str) -> dict[str, dict]: + run = next(r for r in receipt["receipts"] if r["run_id"].endswith(run_id)) + return {s["path"]: s for s in run["sources"]} + + +CLASS_MAP = { # reach class -> (observed, observation_method, selected, loaded) + "loaded": ("loaded", "prompt_capture", True, True), + "fetched": ("fetched", "prompt_capture", True, "unknown"), + "not_loaded": ("not_loaded", "prompt_capture", "unknown", False), + "truncated": ("unknown", "prompt_capture", "unknown", "unknown"), + "error": ("unknown", "unknown", "unknown", "unknown"), +} + + +@pytest.mark.parametrize("cls", sorted(CLASS_MAP)) +def test_each_reach_class_maps_to_its_observed_and_thread_states(tmp_path, cls): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", cls, read_by_tool=cls == "fetched")], [ROOT_ROW]) + src = _sources(build_receipt(reach, {"entries": []}, tree), "cwd=.")["AGENTS.md"] + observed, method, selected, loaded = CLASS_MAP[cls] + assert (src["observed"], src["observation_method"]) == (observed, method) + assert (src["states"]["selected"], src["states"]["loaded"]) == (selected, loaded) + assert src["states"]["followed"] == "unknown" + + +def test_start_up_expectations_follow_budget_and_the_run_directory(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n", "docs/style.md": "d\n"}) + records = [_record(f, s, "not_loaded") for f in ("AGENTS.md", "CLAUDE.md", "sub/AGENTS.md") + for s in ("cwd=.", "read=sub")] + reach = _reach(records, [ROOT_ROW, SUB_ROW]) + receipt = build_receipt(reach, BUDGET, tree) + root = _sources(receipt, "cwd=.") + assert root["CLAUDE.md"]["expected"] is True # start-up + assert root["AGENTS.md"]["expected"] is False # off: a root CLAUDE.md displaces it + assert root["sub/AGENTS.md"]["expected"] is False # lazy, and the run starts at the root + assert root["docs/style.md"]["expected"] is True and root["docs/style.md"]["observed"] == "unknown" + assert _sources(receipt, "read=sub")["sub/AGENTS.md"]["expected"] is True # the run reads inside its subtree + + +def test_a_displacement_seen_in_the_run_is_precedence_observed(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + records = [_record("CLAUDE.md", "cwd=.", "loaded"), _record("AGENTS.md", "cwd=.", "not_loaded")] + receipt = build_receipt(_reach(records, [ROOT_ROW]), BUDGET, tree) + run = receipt["receipts"][0] + assert run["precedence_observed"] == [{"winner": "CLAUDE.md", "displaced": "AGENTS.md"}] + + +def test_no_displacement_claim_without_the_winner_loaded(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + records = [_record("CLAUDE.md", "cwd=.", "not_loaded"), _record("AGENTS.md", "cwd=.", "not_loaded")] + assert build_receipt(_reach(records, [ROOT_ROW]), BUDGET, tree)["receipts"][0]["precedence_observed"] == [] + + +def test_other_harnesses_get_no_expectations_and_say_so(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + records = [_record("AGENTS.md", "cwd=.", "loaded")] + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + receipt = build_receipt(_reach(records, [ROOT_ROW], harness="codex", files=files), BUDGET, tree) + src = _sources(receipt, "cwd=.") + assert set(src) == {"AGENTS.md", "CLAUDE.md"} # budget's start-up chain is Claude Code's, so none is added + assert all(s["expected"] == "unknown" for s in src.values()) + assert any("no source has an expected state for codex" in u for u in receipt["unknowns"]) + + +def test_a_link_is_an_adapter_with_the_targets_observation(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "sub/AGENTS.md": "s\n"}) + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002", "aliases": ["CLAUDE.md"]}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + records = [_record("AGENTS.md", "cwd=.", "loaded")] + src = _sources(build_receipt(_reach(records, [ROOT_ROW], files=files), {"entries": []}, tree), "cwd=.") + assert src["CLAUDE.md"]["alias_of"] == "AGENTS.md" + assert src["CLAUDE.md"]["scope"] == "adapter" and src["CLAUDE.md"]["observed"] == "loaded" + assert src["sub/AGENTS.md"]["scope"] == "subtree" and src["sub/AGENTS.md"]["observed"] == "unknown" + + +def test_a_scenario_with_no_answer_is_unknown_and_says_why(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + row = dict(ROOT_ROW, error="exit 1") + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "error")], [row]), {"entries": []}, tree) + run = receipt["receipts"][0] + assert all(s["observed"] == "unknown" for s in run["sources"]) + assert any("no usable answer" in u for u in run["unknowns"]) + + +def test_a_prose_heavy_or_unseeded_answer_is_flagged(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + row = dict(ROOT_ROW, prose=40, unknown=["RC0000FF"]) + run = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "not_loaded")], [row]), {"entries": []}, tree) + text = " ".join(run["receipts"][0]["unknowns"]) + assert "under-reports" in text and "RC0000FF" in text + + +def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\r\nb\n", "blob.md": b"\xff\xfe\x00bin"}) + digests = source_digests(tree, ["AGENTS.md", "blob.md", "missing.md"]) + assert digests["AGENTS.md"] == hashlib.sha256("a\nb\n".encode("utf-8")).hexdigest() # LF, as dokimi reads it + assert digests["blob.md"] is None and digests["missing.md"] is None + + +def test_the_receipt_is_a_valid_artifact_and_a_schema_rejects_a_followed_claim(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + validate_artifact("effective_instructions", receipt) + receipt["receipts"][0]["sources"][0]["states"]["followed"] = "loaded" + with pytest.raises(ArtifactValidationError, match="followed"): + validate_artifact("effective_instructions", receipt) + + +def _end_to_end_args(tmp_path, monkeypatch, *extra): + monkeypatch.setenv("FAKE_MODE", "ok") + root = _repo(tmp_path) + profile = _profile(tmp_path, name="claude-code", guidance={"names": ["CLAUDE.md", "AGENTS.md"]}) + work = tmp_path / "work" + work.mkdir() + return ["repo", "reach", str(root), "--manifest", "evals/repo-eval.yaml", "--harness", str(profile), + "--model", "m", "--out", str(tmp_path / "out"), "--work-dir", str(work), "--salt", "s", + "--scenarios", "root", *extra] + + +def test_cli_receipt_format_prints_and_writes_a_valid_receipt(tmp_path, monkeypatch, capsys): + args = build_parser().parse_args(_end_to_end_args(tmp_path, monkeypatch, "--format", "receipt")) + args.func(args) + out = capsys.readouterr().out + data = json.loads(out) + validate_artifact("effective_instructions", data) + assert json.loads((tmp_path / "out" / "effective_instructions.json").read_text(encoding="utf-8")) == data + run = data["receipts"][0] + assert run["run_id"] == "s:cwd=." and run["runtime"] == {"name": "claude-code", "version": "9.8.7"} + states = {s["path"]: s["observed"] for s in run["sources"]} + assert states["CLAUDE.md"] == "loaded" and states["AGENTS.md"] == "not_loaded" + # budget turns off every AGENTS.md while a root CLAUDE.md exists, nested ones included + assert sorted(p["displaced"] for p in run["precedence_observed"]) == ["AGENTS.md", "sub/AGENTS.md"] + assert all(p["winner"] == "CLAUDE.md" for p in run["precedence_observed"]) + assert out.lstrip().startswith("{") and "wrote" not in out # the matrix is not printed in receipt mode + + +def test_default_format_still_prints_the_matrix(tmp_path, monkeypatch, capsys): + args = build_parser().parse_args(_end_to_end_args(tmp_path, monkeypatch)) + args.func(args) + out = capsys.readouterr().out + assert "wrote" in out and not (tmp_path / "out" / "effective_instructions.json").exists() diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..3a24afd19 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -46,6 +46,7 @@ "plan.v1": ({}, "skill:onetbb", "claude-sonnet-4-6", "2026-10-21T00:00:00Z"), "result.v1": ({}, None, None, "2026-10-21T01:00:00Z"), "repo_budget.v1": ({}, "toy", None, None), + "effective_instructions.v1": ({}, "toy", "m", "2026-10-31T00:00:00+00:00"), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..170c1138b 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -32,6 +32,7 @@ "plugin_delta.v1": {"arms": 2}, "questions.v2": {"questions": 3}, "repo_budget.v1": {"entries": 9}, + "effective_instructions.v1": {"receipts": 1}, "repo_environment.v1": {"instructions": 4, "findings": 13}, "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6}, From dce67d1aa48d3f00b0845d8d26758b8873cec49e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:46:50 -0700 Subject: [PATCH 02/29] fix(repo): keep local directories out of the receipt's tree label A local source's tree label is its absolute path, and the receipt copied it verbatim: a shared effective-instructions receipt then carried the home path. An absolute or drive-letter or ~ label now keeps only its last directory; URLs and relative labels are unchanged. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/receipt.py | 14 ++++++++++++-- dokimi/tests/test_repo_receipt.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index 04531dc4c..b3f972815 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -14,7 +14,7 @@ import hashlib from dataclasses import dataclass -from pathlib import PurePosixPath +from pathlib import PurePosixPath, PureWindowsPath from typing import Any from .reach import PROSE_LIMIT @@ -62,7 +62,7 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) return { "schema_version": SCHEMA_VERSION, "generated_at": reach["generated_at"], - "repo": reach["repo"], + "repo": {**reach["repo"], "tree": _shareable_tree(reach["repo"]["tree"])}, "runtime": runtime, "model": reach["model"], "observation": {"method": METHOD, "detail": DETAIL}, @@ -71,6 +71,16 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) } +def _shareable_tree(label: str) -> str: + """The tree label with a local directory cut to its last component, so no home path reaches a shared receipt.""" + name, at, sha = label.rpartition("@") + if not at: + name, sha = label, "" + if not (PureWindowsPath(name).drive or PurePosixPath(name).is_absolute() or name.startswith("~")): + return label + return (PurePosixPath(name.replace("\\", "/")).name or "tree") + at + sha + + def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" out: dict[str, str | None] = {} diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index ebd8c6833..73e5d4cdc 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -153,6 +153,22 @@ def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): assert digests["blob.md"] is None and digests["missing.md"] is None +@pytest.mark.parametrize("label, shared", [ + ("/home/alice/src/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("C:\\Users\\alice\\toy", "toy"), + ("~alice/toy", "toy"), + ("https://example.com/org/toy.git@4ff761e2d943", "https://example.com/org/toy.git@4ff761e2d943"), + ("toy@4ff761e2d943", "toy@4ff761e2d943"), +]) +def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["tree"] = label + receipt = build_receipt(reach, {"entries": []}, tree) + assert receipt["repo"]["tree"] == shared + assert "alice" not in json.dumps(receipt) + + def test_the_receipt_is_a_valid_artifact_and_a_schema_rejects_a_followed_claim(tmp_path): tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) From b37b9fd960ebc3829c1d5ac768a46fad27c7bc6a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:31:41 -0700 Subject: [PATCH 03/29] feat(repo): reach conformance suite and a nightly agent x version matrix The reach probe says what a harness loaded in one run. This adds the suite that runs it over a set of fixture repositories for several harnesses and versions, and compares each observed state with what the harness's vendor documentation says. - data/reach-conformance/: eight fixture repositories (root, nested, CLAUDE.md with AGENTS.md, a Claude Code @ import, an override file, a subdirectory shadow, a file over the size cap, a symbolic link), a guidance-only manifest that probes every guidance name for every harness, and expectations.yaml (reach_conformance.v1) with the documented state per file and scenario; anything undocumented is unknown and reported as no_claim. - repos/conformance.py: harness registry, availability (a harness with no model, credential or CLI is a skipped run), the agreement rules, one run per fixture from the effective-instructions receipt, and the merge and markdown rendering of reach_matrix.v1. - scripts/reach_conformance.py: check (expectations against the tree), run (paid calls, only for harnesses that can run here) and merge. - .github/workflows/reach-matrix.yml: nightly and workflow_dispatch, one job per harness with only its own secret, then a merge job that publishes the matrix as an artifact. Not a pull-request check. - docs/reach-matrix.md: design, limits and how to run and extend it. - reach_matrix.v1 is registered as an artifact kind and added to the runs views and the CLI snapshot. Offline tests cover the rules, the run outcomes with the harness faked, merge refusal, rendering, drift checks and the CLI. Signed-off-by: Nikolay Petrov --- .github/workflows/reach-matrix.yml | 182 +++++++ dokimi/benchmarks/spec.v1.yaml | 2 +- .../20261031_160000_napetrov_reach-matrix.md | 8 + .../claude-and-agents/AGENTS.md | 3 + .../claude-and-agents/CLAUDE.md | 3 + .../claude-imports-agents/AGENTS.md | 3 + .../claude-imports-agents/CLAUDE.md | 5 + .../data/reach-conformance/expectations.yaml | 120 +++++ dokimi/data/reach-conformance/guidance.yaml | 13 + .../reach-conformance/large-file/AGENTS.md | 460 ++++++++++++++++++ .../reach-conformance/nested-agents/AGENTS.md | 3 + .../nested-agents/packages/app/AGENTS.md | 3 + .../reach-conformance/override-file/AGENTS.md | 3 + .../override-file/AGENTS.override.md | 3 + .../reach-conformance/root-agents/AGENTS.md | 3 + .../subtree-claude-shadow/sub/AGENTS.md | 3 + .../subtree-claude-shadow/sub/CLAUDE.md | 3 + .../symlinked-file/AGENTS.md | 3 + .../symlinked-file/CLAUDE.md | 1 + dokimi/docs/README.md | 4 + dokimi/docs/reach-matrix.md | 140 ++++++ dokimi/docs/reference/cli-interface.json | 1 + dokimi/docs/repo-evaluation-contract.md | 4 + dokimi/dokimi/artifacts.py | 3 +- dokimi/dokimi/repos/conformance.py | 294 +++++++++++ dokimi/dokimi/runs/kinds.py | 1 + .../dokimi/schemas/reach_conformance.v1.json | 40 ++ dokimi/dokimi/schemas/reach_matrix.v1.json | 68 +++ dokimi/mkdocs.yml | 1 + dokimi/scripts/reach_conformance.py | 108 ++++ .../artifacts/reach_conformance.v1.json | 22 + .../fixtures/artifacts/reach_matrix.v1.json | 66 +++ dokimi/tests/test_reach_conformance.py | 302 ++++++++++++ dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + 35 files changed, 1878 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/reach-matrix.yml create mode 100644 dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md create mode 100644 dokimi/data/reach-conformance/claude-and-agents/AGENTS.md create mode 100644 dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md create mode 100644 dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md create mode 100644 dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md create mode 100644 dokimi/data/reach-conformance/expectations.yaml create mode 100644 dokimi/data/reach-conformance/guidance.yaml create mode 100644 dokimi/data/reach-conformance/large-file/AGENTS.md create mode 100644 dokimi/data/reach-conformance/nested-agents/AGENTS.md create mode 100644 dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md create mode 100644 dokimi/data/reach-conformance/override-file/AGENTS.md create mode 100644 dokimi/data/reach-conformance/override-file/AGENTS.override.md create mode 100644 dokimi/data/reach-conformance/root-agents/AGENTS.md create mode 100644 dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md create mode 100644 dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md create mode 100644 dokimi/data/reach-conformance/symlinked-file/AGENTS.md create mode 120000 dokimi/data/reach-conformance/symlinked-file/CLAUDE.md create mode 100644 dokimi/docs/reach-matrix.md create mode 100644 dokimi/dokimi/repos/conformance.py create mode 100644 dokimi/dokimi/schemas/reach_conformance.v1.json create mode 100644 dokimi/dokimi/schemas/reach_matrix.v1.json create mode 100644 dokimi/scripts/reach_conformance.py create mode 100644 dokimi/tests/fixtures/artifacts/reach_conformance.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/reach_matrix.v1.json create mode 100644 dokimi/tests/test_reach_conformance.py diff --git a/.github/workflows/reach-matrix.yml b/.github/workflows/reach-matrix.yml new file mode 100644 index 000000000..c954587c4 --- /dev/null +++ b/.github/workflows/reach-matrix.yml @@ -0,0 +1,182 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: reach-matrix + +# The nightly reach conformance matrix (dokimi/docs/reach-matrix.md): each harness runs `repo reach` over the +# fixtures in dokimi/data/reach-conformance, and the three matrices merge into one page. This makes paid model +# calls, so it is not a check on pull requests. A harness with no model variable, no credential or no CLI on +# the runner is recorded as skipped; it does not fail the run. +on: + schedule: + - cron: '37 4 * * *' + workflow_dispatch: + +permissions: {} + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + claude-code: + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - name: Check the credential + id: creds + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + run: | + if [ -n "$ANTHROPIC_API_KEY$AWS_BEARER_TOKEN_BEDROCK" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi + - name: Install the CLI + if: steps.creds.outputs.present == 'true' + run: npm install -g @anthropic-ai/claude-code@2.1.316 + - name: Run reach over the fixtures + working-directory: dokimi + env: + PYTHONPATH: . + REACH_MODEL: ${{ vars.REACH_MODEL_CLAUDE_CODE }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + run: python scripts/reach_conformance.py run --harness "claude-code=$REACH_MODEL" --out reach-out + - name: Upload the matrix + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix-claude-code + path: dokimi/reach-out/claude-code.matrix.json + if-no-files-found: warn + retention-days: 30 + + codex: + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - name: Check the credential + id: creds + env: + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + run: | + if [ -n "$CODEX_API_KEY" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi + - name: Install the CLI + if: steps.creds.outputs.present == 'true' + run: npm install -g @openai/codex@0.166.0 + - name: Run reach over the fixtures + working-directory: dokimi + env: + PYTHONPATH: . + REACH_MODEL: ${{ vars.REACH_MODEL_CODEX }} + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + run: python scripts/reach_conformance.py run --harness "codex=$REACH_MODEL" --out reach-out + - name: Upload the matrix + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix-codex + path: dokimi/reach-out/codex.matrix.json + if-no-files-found: warn + retention-days: 30 + + copilot: + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - name: Check the credential + id: creds + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + run: | + if [ -n "$COPILOT_GITHUB_TOKEN" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi + - name: Install the CLI + if: steps.creds.outputs.present == 'true' + run: npm install -g @github/copilot@1.0.105 + - name: Run reach over the fixtures + working-directory: dokimi + env: + PYTHONPATH: . + REACH_MODEL: ${{ vars.REACH_MODEL_COPILOT }} + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + run: python scripts/reach_conformance.py run --harness "copilot=$REACH_MODEL" --out reach-out + - name: Upload the matrix + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix-copilot + path: dokimi/reach-out/copilot.matrix.json + if-no-files-found: warn + retention-days: 30 + + matrix: + needs: [claude-code, codex, copilot] + if: always() + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: reach-matrix-* + merge-multiple: true + path: dokimi/reach-in + - name: Merge the matrices + working-directory: dokimi + env: + PYTHONPATH: . + run: | + python scripts/reach_conformance.py merge reach-in/*.matrix.json \ + --out reach-out/reach-matrix.json --md reach-out/reach-matrix.md + cat reach-out/reach-matrix.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload the matrix page + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix + path: | + dokimi/reach-out/reach-matrix.json + dokimi/reach-out/reach-matrix.md + if-no-files-found: error + retention-days: 90 diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index b36efa758..753c06a0d 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 37 + max_docs: 38 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md b/dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md new file mode 100644 index 000000000..14892b22e --- /dev/null +++ b/dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md @@ -0,0 +1,8 @@ + + +### Added + +- **A reach conformance suite and a nightly agent × version matrix** — `data/reach-conformance/` holds eight fixture repositories (root, nested, `CLAUDE.md` with `AGENTS.md`, a Claude Code `@` import, an `AGENTS.override.md`, a subdirectory shadow, a file over the size cap, a symbolic link), and `expectations.yaml` (`reach_conformance.v1`) gives what each harness's vendor documentation says each file loads in each scenario, or `unknown`. `scripts/reach_conformance.py` checks the expectations against the fixtures, runs `repo reach --format receipt` over them for each harness with a model, credential and CLI present (the rest are recorded as skipped), and merges the per-harness `reach_matrix.v1` JSON into one matrix and one markdown page. `.github/workflows/reach-matrix.yml` runs it nightly and on dispatch, not as a pull-request check, and uploads the matrix as an artifact. The design, the agreement rules and the limits are in `docs/reach-matrix.md`. diff --git a/dokimi/data/reach-conformance/claude-and-agents/AGENTS.md b/dokimi/data/reach-conformance/claude-and-agents/AGENTS.md new file mode 100644 index 000000000..ea44db060 --- /dev/null +++ b/dokimi/data/reach-conformance/claude-and-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +These instructions are for the other coding agents. diff --git a/dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md b/dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md new file mode 100644 index 000000000..62571eb4a --- /dev/null +++ b/dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md @@ -0,0 +1,3 @@ +# Claude instructions + +These instructions are for Claude Code. diff --git a/dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md b/dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md new file mode 100644 index 000000000..9806e4047 --- /dev/null +++ b/dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +Shared rules for every coding agent. diff --git a/dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md b/dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md new file mode 100644 index 000000000..8dc3d3719 --- /dev/null +++ b/dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md @@ -0,0 +1,5 @@ +# Claude instructions + +Shared rules live in AGENTS.md. + +@AGENTS.md diff --git a/dokimi/data/reach-conformance/expectations.yaml b/dokimi/data/reach-conformance/expectations.yaml new file mode 100644 index 000000000..d3692af42 --- /dev/null +++ b/dokimi/data/reach-conformance/expectations.yaml @@ -0,0 +1,120 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# What each harness's vendor documentation says each fixture loads, per reach scenario +# (cwd=. is the root, cwd= starts there, read= starts at the root and reads a file in ). +# A file or scenario not listed is `unknown`: the docs do not say, so the matrix reports the +# observation without a claim. Sources: code.claude.com/docs/en/memory (Claude Code, "AGENTS.md" +# and "How CLAUDE.md files load") and learn.chatgpt.com/docs/agent-configuration/agents-md (Codex). +schema: reach_conformance.v1 +cases: + root-agents: + summary: AGENTS.md at the repository root, and nothing else. + claude-code: + why: With no CLAUDE.md in the working directory or above, Claude Code reads AGENTS.md. + files: + AGENTS.md: {"cwd=.": loaded} + codex: + why: Codex checks AGENTS.override.md then AGENTS.md in each directory from the git root to cwd. + files: + AGENTS.md: {"cwd=.": loaded} + + nested-agents: + summary: A root AGENTS.md and a packages/app/AGENTS.md below it. + claude-code: + why: >- + Every AGENTS.md in the working directory and above loads at start-up. A subdirectory's + AGENTS.md loads when Claude reads a file there with the Read tool. + files: + AGENTS.md: {"cwd=.": loaded, "cwd=packages/app": loaded, "read=packages/app": loaded} + packages/app/AGENTS.md: {"cwd=.": not_loaded, "cwd=packages/app": loaded, "read=packages/app": loaded} + codex: + why: >- + Codex walks from the git root down to cwd, one file per directory. A directory off that path + is not read; the docs do not describe a read-triggered load, so read= is unknown. + files: + AGENTS.md: {"cwd=.": loaded, "cwd=packages/app": loaded} + packages/app/AGENTS.md: {"cwd=.": not_loaded, "cwd=packages/app": loaded} + + claude-and-agents: + summary: A root CLAUDE.md beside a root AGENTS.md. + claude-code: + why: >- + AGENTS.md is read only when there is no CLAUDE.md in the working directory or above; the + default setting (claude-md-or-agents-md) reads the CLAUDE.md files alone. + files: + CLAUDE.md: {"cwd=.": loaded} + AGENTS.md: {"cwd=.": not_loaded} + codex: + why: >- + Codex checks AGENTS.override.md, AGENTS.md and the project_doc_fallback_filenames list only, + and CLAUDE.md is not on that list by default. + files: + AGENTS.md: {"cwd=.": loaded} + CLAUDE.md: {"cwd=.": not_loaded} + + claude-imports-agents: + summary: A root CLAUDE.md that imports AGENTS.md with @AGENTS.md. + claude-code: + why: CLAUDE.md @path imports are expanded and loaded with the file that references them. + files: + CLAUDE.md: {"cwd=.": loaded} + AGENTS.md: {"cwd=.": loaded} + codex: + why: Codex does not read CLAUDE.md by default, so the import is never followed. + files: + AGENTS.md: {"cwd=.": loaded} + CLAUDE.md: {"cwd=.": not_loaded} + + override-file: + summary: A root AGENTS.md beside an AGENTS.override.md. + claude-code: + why: The Claude Code AGENTS.md rules list AGENTS.override.md among the files it does not read. + files: + AGENTS.md: {"cwd=.": loaded} + AGENTS.override.md: {"cwd=.": not_loaded} + codex: + why: >- + In each directory AGENTS.override.md is used in place of AGENTS.md, and at most one of the + two is read. + files: + AGENTS.override.md: {"cwd=.": loaded} + AGENTS.md: {"cwd=.": not_loaded} + + subtree-claude-shadow: + summary: sub/AGENTS.md and sub/CLAUDE.md in the same subdirectory. + claude-code: + why: >- + A CLAUDE.md in the working directory loads at start-up. AGENTS.md is not read in a directory + that has a CLAUDE.md of its own. + files: + sub/CLAUDE.md: {"cwd=.": not_loaded, "cwd=sub": loaded, "read=sub": loaded} + sub/AGENTS.md: {"cwd=.": not_loaded, "cwd=sub": not_loaded, "read=sub": not_loaded} + codex: + why: Codex reads sub/AGENTS.md when cwd is sub, and never reads CLAUDE.md by default. + files: + sub/AGENTS.md: {"cwd=.": not_loaded, "cwd=sub": loaded} + sub/CLAUDE.md: {"cwd=.": not_loaded, "cwd=sub": not_loaded} + + large-file: + summary: A root AGENTS.md of about 40 KB, above Codex's 32 KiB default. + claude-code: + why: >- + No size limit is documented. The memory page advises under 200 lines per file, which is a + target and not a cut-off. So the state is unknown. + files: {} + codex: + why: >- + project_doc_max_bytes (default 32 KiB) caps the combined size of the chain. The docs do not + say whether a file that crosses the cap is cut, so the state is unknown. + files: {} + + symlinked-file: + summary: CLAUDE.md is a symbolic link to AGENTS.md. + claude-code: + why: The docs do not say how a symbolic link named CLAUDE.md is read, so the state is unknown. + files: {} + codex: + why: Codex reads AGENTS.md itself, and the link's name is not one it checks. + files: + AGENTS.md: {"cwd=.": loaded} diff --git a/dokimi/data/reach-conformance/guidance.yaml b/dokimi/data/reach-conformance/guidance.yaml new file mode 100644 index 000000000..f53cef4ff --- /dev/null +++ b/dokimi/data/reach-conformance/guidance.yaml @@ -0,0 +1,13 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# The guidance-only manifest every conformance case is probed with (`repo reach --manifest-file`). +# AGENTS.override.md is named here so every harness probes it, not only the ones whose profile names it. +schema: repo_manifest.v1 +id: reach-conformance +guidance: + globs: + - "**/AGENTS.md" + - "**/AGENTS.override.md" + - "**/CLAUDE.md" + - "**/CLAUDE.local.md" diff --git a/dokimi/data/reach-conformance/large-file/AGENTS.md b/dokimi/data/reach-conformance/large-file/AGENTS.md new file mode 100644 index 000000000..56b874120 --- /dev/null +++ b/dokimi/data/reach-conformance/large-file/AGENTS.md @@ -0,0 +1,460 @@ +Filler line 1: padding for the size case, not an instruction, nothing to act on here. +Filler line 2: padding for the size case, not an instruction, nothing to act on here. +Filler line 3: padding for the size case, not an instruction, nothing to act on here. +Filler line 4: padding for the size case, not an instruction, nothing to act on here. +Filler line 5: padding for the size case, not an instruction, nothing to act on here. +Filler line 6: padding for the size case, not an instruction, nothing to act on here. +Filler line 7: padding for the size case, not an instruction, nothing to act on here. +Filler line 8: padding for the size case, not an instruction, nothing to act on here. +Filler line 9: padding for the size case, not an instruction, nothing to act on here. +Filler line 10: padding for the size case, not an instruction, nothing to act on here. +Filler line 11: padding for the size case, not an instruction, nothing to act on here. +Filler line 12: padding for the size case, not an instruction, nothing to act on here. +Filler line 13: padding for the size case, not an instruction, nothing to act on here. +Filler line 14: padding for the size case, not an instruction, nothing to act on here. +Filler line 15: padding for the size case, not an instruction, nothing to act on here. +Filler line 16: padding for the size case, not an instruction, nothing to act on here. +Filler line 17: padding for the size case, not an instruction, nothing to act on here. +Filler line 18: padding for the size case, not an instruction, nothing to act on here. +Filler line 19: padding for the size case, not an instruction, nothing to act on here. +Filler line 20: padding for the size case, not an instruction, nothing to act on here. +Filler line 21: padding for the size case, not an instruction, nothing to act on here. +Filler line 22: padding for the size case, not an instruction, nothing to act on here. +Filler line 23: padding for the size case, not an instruction, nothing to act on here. +Filler line 24: padding for the size case, not an instruction, nothing to act on here. +Filler line 25: padding for the size case, not an instruction, nothing to act on here. +Filler line 26: padding for the size case, not an instruction, nothing to act on here. +Filler line 27: padding for the size case, not an instruction, nothing to act on here. +Filler line 28: padding for the size case, not an instruction, nothing to act on here. +Filler line 29: padding for the size case, not an instruction, nothing to act on here. +Filler line 30: padding for the size case, not an instruction, nothing to act on here. +Filler line 31: padding for the size case, not an instruction, nothing to act on here. +Filler line 32: padding for the size case, not an instruction, nothing to act on here. +Filler line 33: padding for the size case, not an instruction, nothing to act on here. +Filler line 34: padding for the size case, not an instruction, nothing to act on here. +Filler line 35: padding for the size case, not an instruction, nothing to act on here. +Filler line 36: padding for the size case, not an instruction, nothing to act on here. +Filler line 37: padding for the size case, not an instruction, nothing to act on here. +Filler line 38: padding for the size case, not an instruction, nothing to act on here. +Filler line 39: padding for the size case, not an instruction, nothing to act on here. +Filler line 40: padding for the size case, not an instruction, nothing to act on here. +Filler line 41: padding for the size case, not an instruction, nothing to act on here. +Filler line 42: padding for the size case, not an instruction, nothing to act on here. +Filler line 43: padding for the size case, not an instruction, nothing to act on here. +Filler line 44: padding for the size case, not an instruction, nothing to act on here. +Filler line 45: padding for the size case, not an instruction, nothing to act on here. +Filler line 46: padding for the size case, not an instruction, nothing to act on here. +Filler line 47: padding for the size case, not an instruction, nothing to act on here. +Filler line 48: padding for the size case, not an instruction, nothing to act on here. +Filler line 49: padding for the size case, not an instruction, nothing to act on here. +Filler line 50: padding for the size case, not an instruction, nothing to act on here. +Filler line 51: padding for the size case, not an instruction, nothing to act on here. +Filler line 52: padding for the size case, not an instruction, nothing to act on here. +Filler line 53: padding for the size case, not an instruction, nothing to act on here. +Filler line 54: padding for the size case, not an instruction, nothing to act on here. +Filler line 55: padding for the size case, not an instruction, nothing to act on here. +Filler line 56: padding for the size case, not an instruction, nothing to act on here. +Filler line 57: padding for the size case, not an instruction, nothing to act on here. +Filler line 58: padding for the size case, not an instruction, nothing to act on here. +Filler line 59: padding for the size case, not an instruction, nothing to act on here. +Filler line 60: padding for the size case, not an instruction, nothing to act on here. +Filler line 61: padding for the size case, not an instruction, nothing to act on here. +Filler line 62: padding for the size case, not an instruction, nothing to act on here. +Filler line 63: padding for the size case, not an instruction, nothing to act on here. +Filler line 64: padding for the size case, not an instruction, nothing to act on here. +Filler line 65: padding for the size case, not an instruction, nothing to act on here. +Filler line 66: padding for the size case, not an instruction, nothing to act on here. +Filler line 67: padding for the size case, not an instruction, nothing to act on here. +Filler line 68: padding for the size case, not an instruction, nothing to act on here. +Filler line 69: padding for the size case, not an instruction, nothing to act on here. +Filler line 70: padding for the size case, not an instruction, nothing to act on here. +Filler line 71: padding for the size case, not an instruction, nothing to act on here. +Filler line 72: padding for the size case, not an instruction, nothing to act on here. +Filler line 73: padding for the size case, not an instruction, nothing to act on here. +Filler line 74: padding for the size case, not an instruction, nothing to act on here. +Filler line 75: padding for the size case, not an instruction, nothing to act on here. +Filler line 76: padding for the size case, not an instruction, nothing to act on here. +Filler line 77: padding for the size case, not an instruction, nothing to act on here. +Filler line 78: padding for the size case, not an instruction, nothing to act on here. +Filler line 79: padding for the size case, not an instruction, nothing to act on here. +Filler line 80: padding for the size case, not an instruction, nothing to act on here. +Filler line 81: padding for the size case, not an instruction, nothing to act on here. +Filler line 82: padding for the size case, not an instruction, nothing to act on here. +Filler line 83: padding for the size case, not an instruction, nothing to act on here. +Filler line 84: padding for the size case, not an instruction, nothing to act on here. +Filler line 85: padding for the size case, not an instruction, nothing to act on here. +Filler line 86: padding for the size case, not an instruction, nothing to act on here. +Filler line 87: padding for the size case, not an instruction, nothing to act on here. +Filler line 88: padding for the size case, not an instruction, nothing to act on here. +Filler line 89: padding for the size case, not an instruction, nothing to act on here. +Filler line 90: padding for the size case, not an instruction, nothing to act on here. +Filler line 91: padding for the size case, not an instruction, nothing to act on here. +Filler line 92: padding for the size case, not an instruction, nothing to act on here. +Filler line 93: padding for the size case, not an instruction, nothing to act on here. +Filler line 94: padding for the size case, not an instruction, nothing to act on here. +Filler line 95: padding for the size case, not an instruction, nothing to act on here. +Filler line 96: padding for the size case, not an instruction, nothing to act on here. +Filler line 97: padding for the size case, not an instruction, nothing to act on here. +Filler line 98: padding for the size case, not an instruction, nothing to act on here. +Filler line 99: padding for the size case, not an instruction, nothing to act on here. +Filler line 100: padding for the size case, not an instruction, nothing to act on here. +Filler line 101: padding for the size case, not an instruction, nothing to act on here. +Filler line 102: padding for the size case, not an instruction, nothing to act on here. +Filler line 103: padding for the size case, not an instruction, nothing to act on here. +Filler line 104: padding for the size case, not an instruction, nothing to act on here. +Filler line 105: padding for the size case, not an instruction, nothing to act on here. +Filler line 106: padding for the size case, not an instruction, nothing to act on here. +Filler line 107: padding for the size case, not an instruction, nothing to act on here. +Filler line 108: padding for the size case, not an instruction, nothing to act on here. +Filler line 109: padding for the size case, not an instruction, nothing to act on here. +Filler line 110: padding for the size case, not an instruction, nothing to act on here. +Filler line 111: padding for the size case, not an instruction, nothing to act on here. +Filler line 112: padding for the size case, not an instruction, nothing to act on here. +Filler line 113: padding for the size case, not an instruction, nothing to act on here. +Filler line 114: padding for the size case, not an instruction, nothing to act on here. +Filler line 115: padding for the size case, not an instruction, nothing to act on here. +Filler line 116: padding for the size case, not an instruction, nothing to act on here. +Filler line 117: padding for the size case, not an instruction, nothing to act on here. +Filler line 118: padding for the size case, not an instruction, nothing to act on here. +Filler line 119: padding for the size case, not an instruction, nothing to act on here. +Filler line 120: padding for the size case, not an instruction, nothing to act on here. +Filler line 121: padding for the size case, not an instruction, nothing to act on here. +Filler line 122: padding for the size case, not an instruction, nothing to act on here. +Filler line 123: padding for the size case, not an instruction, nothing to act on here. +Filler line 124: padding for the size case, not an instruction, nothing to act on here. +Filler line 125: padding for the size case, not an instruction, nothing to act on here. +Filler line 126: padding for the size case, not an instruction, nothing to act on here. +Filler line 127: padding for the size case, not an instruction, nothing to act on here. +Filler line 128: padding for the size case, not an instruction, nothing to act on here. +Filler line 129: padding for the size case, not an instruction, nothing to act on here. +Filler line 130: padding for the size case, not an instruction, nothing to act on here. +Filler line 131: padding for the size case, not an instruction, nothing to act on here. +Filler line 132: padding for the size case, not an instruction, nothing to act on here. +Filler line 133: padding for the size case, not an instruction, nothing to act on here. +Filler line 134: padding for the size case, not an instruction, nothing to act on here. +Filler line 135: padding for the size case, not an instruction, nothing to act on here. +Filler line 136: padding for the size case, not an instruction, nothing to act on here. +Filler line 137: padding for the size case, not an instruction, nothing to act on here. +Filler line 138: padding for the size case, not an instruction, nothing to act on here. +Filler line 139: padding for the size case, not an instruction, nothing to act on here. +Filler line 140: padding for the size case, not an instruction, nothing to act on here. +Filler line 141: padding for the size case, not an instruction, nothing to act on here. +Filler line 142: padding for the size case, not an instruction, nothing to act on here. +Filler line 143: padding for the size case, not an instruction, nothing to act on here. +Filler line 144: padding for the size case, not an instruction, nothing to act on here. +Filler line 145: padding for the size case, not an instruction, nothing to act on here. +Filler line 146: padding for the size case, not an instruction, nothing to act on here. +Filler line 147: padding for the size case, not an instruction, nothing to act on here. +Filler line 148: padding for the size case, not an instruction, nothing to act on here. +Filler line 149: padding for the size case, not an instruction, nothing to act on here. +Filler line 150: padding for the size case, not an instruction, nothing to act on here. +Filler line 151: padding for the size case, not an instruction, nothing to act on here. +Filler line 152: padding for the size case, not an instruction, nothing to act on here. +Filler line 153: padding for the size case, not an instruction, nothing to act on here. +Filler line 154: padding for the size case, not an instruction, nothing to act on here. +Filler line 155: padding for the size case, not an instruction, nothing to act on here. +Filler line 156: padding for the size case, not an instruction, nothing to act on here. +Filler line 157: padding for the size case, not an instruction, nothing to act on here. +Filler line 158: padding for the size case, not an instruction, nothing to act on here. +Filler line 159: padding for the size case, not an instruction, nothing to act on here. +Filler line 160: padding for the size case, not an instruction, nothing to act on here. +Filler line 161: padding for the size case, not an instruction, nothing to act on here. +Filler line 162: padding for the size case, not an instruction, nothing to act on here. +Filler line 163: padding for the size case, not an instruction, nothing to act on here. +Filler line 164: padding for the size case, not an instruction, nothing to act on here. +Filler line 165: padding for the size case, not an instruction, nothing to act on here. +Filler line 166: padding for the size case, not an instruction, nothing to act on here. +Filler line 167: padding for the size case, not an instruction, nothing to act on here. +Filler line 168: padding for the size case, not an instruction, nothing to act on here. +Filler line 169: padding for the size case, not an instruction, nothing to act on here. +Filler line 170: padding for the size case, not an instruction, nothing to act on here. +Filler line 171: padding for the size case, not an instruction, nothing to act on here. +Filler line 172: padding for the size case, not an instruction, nothing to act on here. +Filler line 173: padding for the size case, not an instruction, nothing to act on here. +Filler line 174: padding for the size case, not an instruction, nothing to act on here. +Filler line 175: padding for the size case, not an instruction, nothing to act on here. +Filler line 176: padding for the size case, not an instruction, nothing to act on here. +Filler line 177: padding for the size case, not an instruction, nothing to act on here. +Filler line 178: padding for the size case, not an instruction, nothing to act on here. +Filler line 179: padding for the size case, not an instruction, nothing to act on here. +Filler line 180: padding for the size case, not an instruction, nothing to act on here. +Filler line 181: padding for the size case, not an instruction, nothing to act on here. +Filler line 182: padding for the size case, not an instruction, nothing to act on here. +Filler line 183: padding for the size case, not an instruction, nothing to act on here. +Filler line 184: padding for the size case, not an instruction, nothing to act on here. +Filler line 185: padding for the size case, not an instruction, nothing to act on here. +Filler line 186: padding for the size case, not an instruction, nothing to act on here. +Filler line 187: padding for the size case, not an instruction, nothing to act on here. +Filler line 188: padding for the size case, not an instruction, nothing to act on here. +Filler line 189: padding for the size case, not an instruction, nothing to act on here. +Filler line 190: padding for the size case, not an instruction, nothing to act on here. +Filler line 191: padding for the size case, not an instruction, nothing to act on here. +Filler line 192: padding for the size case, not an instruction, nothing to act on here. +Filler line 193: padding for the size case, not an instruction, nothing to act on here. +Filler line 194: padding for the size case, not an instruction, nothing to act on here. +Filler line 195: padding for the size case, not an instruction, nothing to act on here. +Filler line 196: padding for the size case, not an instruction, nothing to act on here. +Filler line 197: padding for the size case, not an instruction, nothing to act on here. +Filler line 198: padding for the size case, not an instruction, nothing to act on here. +Filler line 199: padding for the size case, not an instruction, nothing to act on here. +Filler line 200: padding for the size case, not an instruction, nothing to act on here. +Filler line 201: padding for the size case, not an instruction, nothing to act on here. +Filler line 202: padding for the size case, not an instruction, nothing to act on here. +Filler line 203: padding for the size case, not an instruction, nothing to act on here. +Filler line 204: padding for the size case, not an instruction, nothing to act on here. +Filler line 205: padding for the size case, not an instruction, nothing to act on here. +Filler line 206: padding for the size case, not an instruction, nothing to act on here. +Filler line 207: padding for the size case, not an instruction, nothing to act on here. +Filler line 208: padding for the size case, not an instruction, nothing to act on here. +Filler line 209: padding for the size case, not an instruction, nothing to act on here. +Filler line 210: padding for the size case, not an instruction, nothing to act on here. +Filler line 211: padding for the size case, not an instruction, nothing to act on here. +Filler line 212: padding for the size case, not an instruction, nothing to act on here. +Filler line 213: padding for the size case, not an instruction, nothing to act on here. +Filler line 214: padding for the size case, not an instruction, nothing to act on here. +Filler line 215: padding for the size case, not an instruction, nothing to act on here. +Filler line 216: padding for the size case, not an instruction, nothing to act on here. +Filler line 217: padding for the size case, not an instruction, nothing to act on here. +Filler line 218: padding for the size case, not an instruction, nothing to act on here. +Filler line 219: padding for the size case, not an instruction, nothing to act on here. +Filler line 220: padding for the size case, not an instruction, nothing to act on here. +Filler line 221: padding for the size case, not an instruction, nothing to act on here. +Filler line 222: padding for the size case, not an instruction, nothing to act on here. +Filler line 223: padding for the size case, not an instruction, nothing to act on here. +Filler line 224: padding for the size case, not an instruction, nothing to act on here. +Filler line 225: padding for the size case, not an instruction, nothing to act on here. +Filler line 226: padding for the size case, not an instruction, nothing to act on here. +Filler line 227: padding for the size case, not an instruction, nothing to act on here. +Filler line 228: padding for the size case, not an instruction, nothing to act on here. +Filler line 229: padding for the size case, not an instruction, nothing to act on here. +Filler line 230: padding for the size case, not an instruction, nothing to act on here. +Filler line 231: padding for the size case, not an instruction, nothing to act on here. +Filler line 232: padding for the size case, not an instruction, nothing to act on here. +Filler line 233: padding for the size case, not an instruction, nothing to act on here. +Filler line 234: padding for the size case, not an instruction, nothing to act on here. +Filler line 235: padding for the size case, not an instruction, nothing to act on here. +Filler line 236: padding for the size case, not an instruction, nothing to act on here. +Filler line 237: padding for the size case, not an instruction, nothing to act on here. +Filler line 238: padding for the size case, not an instruction, nothing to act on here. +Filler line 239: padding for the size case, not an instruction, nothing to act on here. +Filler line 240: padding for the size case, not an instruction, nothing to act on here. +Filler line 241: padding for the size case, not an instruction, nothing to act on here. +Filler line 242: padding for the size case, not an instruction, nothing to act on here. +Filler line 243: padding for the size case, not an instruction, nothing to act on here. +Filler line 244: padding for the size case, not an instruction, nothing to act on here. +Filler line 245: padding for the size case, not an instruction, nothing to act on here. +Filler line 246: padding for the size case, not an instruction, nothing to act on here. +Filler line 247: padding for the size case, not an instruction, nothing to act on here. +Filler line 248: padding for the size case, not an instruction, nothing to act on here. +Filler line 249: padding for the size case, not an instruction, nothing to act on here. +Filler line 250: padding for the size case, not an instruction, nothing to act on here. +Filler line 251: padding for the size case, not an instruction, nothing to act on here. +Filler line 252: padding for the size case, not an instruction, nothing to act on here. +Filler line 253: padding for the size case, not an instruction, nothing to act on here. +Filler line 254: padding for the size case, not an instruction, nothing to act on here. +Filler line 255: padding for the size case, not an instruction, nothing to act on here. +Filler line 256: padding for the size case, not an instruction, nothing to act on here. +Filler line 257: padding for the size case, not an instruction, nothing to act on here. +Filler line 258: padding for the size case, not an instruction, nothing to act on here. +Filler line 259: padding for the size case, not an instruction, nothing to act on here. +Filler line 260: padding for the size case, not an instruction, nothing to act on here. +Filler line 261: padding for the size case, not an instruction, nothing to act on here. +Filler line 262: padding for the size case, not an instruction, nothing to act on here. +Filler line 263: padding for the size case, not an instruction, nothing to act on here. +Filler line 264: padding for the size case, not an instruction, nothing to act on here. +Filler line 265: padding for the size case, not an instruction, nothing to act on here. +Filler line 266: padding for the size case, not an instruction, nothing to act on here. +Filler line 267: padding for the size case, not an instruction, nothing to act on here. +Filler line 268: padding for the size case, not an instruction, nothing to act on here. +Filler line 269: padding for the size case, not an instruction, nothing to act on here. +Filler line 270: padding for the size case, not an instruction, nothing to act on here. +Filler line 271: padding for the size case, not an instruction, nothing to act on here. +Filler line 272: padding for the size case, not an instruction, nothing to act on here. +Filler line 273: padding for the size case, not an instruction, nothing to act on here. +Filler line 274: padding for the size case, not an instruction, nothing to act on here. +Filler line 275: padding for the size case, not an instruction, nothing to act on here. +Filler line 276: padding for the size case, not an instruction, nothing to act on here. +Filler line 277: padding for the size case, not an instruction, nothing to act on here. +Filler line 278: padding for the size case, not an instruction, nothing to act on here. +Filler line 279: padding for the size case, not an instruction, nothing to act on here. +Filler line 280: padding for the size case, not an instruction, nothing to act on here. +Filler line 281: padding for the size case, not an instruction, nothing to act on here. +Filler line 282: padding for the size case, not an instruction, nothing to act on here. +Filler line 283: padding for the size case, not an instruction, nothing to act on here. +Filler line 284: padding for the size case, not an instruction, nothing to act on here. +Filler line 285: padding for the size case, not an instruction, nothing to act on here. +Filler line 286: padding for the size case, not an instruction, nothing to act on here. +Filler line 287: padding for the size case, not an instruction, nothing to act on here. +Filler line 288: padding for the size case, not an instruction, nothing to act on here. +Filler line 289: padding for the size case, not an instruction, nothing to act on here. +Filler line 290: padding for the size case, not an instruction, nothing to act on here. +Filler line 291: padding for the size case, not an instruction, nothing to act on here. +Filler line 292: padding for the size case, not an instruction, nothing to act on here. +Filler line 293: padding for the size case, not an instruction, nothing to act on here. +Filler line 294: padding for the size case, not an instruction, nothing to act on here. +Filler line 295: padding for the size case, not an instruction, nothing to act on here. +Filler line 296: padding for the size case, not an instruction, nothing to act on here. +Filler line 297: padding for the size case, not an instruction, nothing to act on here. +Filler line 298: padding for the size case, not an instruction, nothing to act on here. +Filler line 299: padding for the size case, not an instruction, nothing to act on here. +Filler line 300: padding for the size case, not an instruction, nothing to act on here. +Filler line 301: padding for the size case, not an instruction, nothing to act on here. +Filler line 302: padding for the size case, not an instruction, nothing to act on here. +Filler line 303: padding for the size case, not an instruction, nothing to act on here. +Filler line 304: padding for the size case, not an instruction, nothing to act on here. +Filler line 305: padding for the size case, not an instruction, nothing to act on here. +Filler line 306: padding for the size case, not an instruction, nothing to act on here. +Filler line 307: padding for the size case, not an instruction, nothing to act on here. +Filler line 308: padding for the size case, not an instruction, nothing to act on here. +Filler line 309: padding for the size case, not an instruction, nothing to act on here. +Filler line 310: padding for the size case, not an instruction, nothing to act on here. +Filler line 311: padding for the size case, not an instruction, nothing to act on here. +Filler line 312: padding for the size case, not an instruction, nothing to act on here. +Filler line 313: padding for the size case, not an instruction, nothing to act on here. +Filler line 314: padding for the size case, not an instruction, nothing to act on here. +Filler line 315: padding for the size case, not an instruction, nothing to act on here. +Filler line 316: padding for the size case, not an instruction, nothing to act on here. +Filler line 317: padding for the size case, not an instruction, nothing to act on here. +Filler line 318: padding for the size case, not an instruction, nothing to act on here. +Filler line 319: padding for the size case, not an instruction, nothing to act on here. +Filler line 320: padding for the size case, not an instruction, nothing to act on here. +Filler line 321: padding for the size case, not an instruction, nothing to act on here. +Filler line 322: padding for the size case, not an instruction, nothing to act on here. +Filler line 323: padding for the size case, not an instruction, nothing to act on here. +Filler line 324: padding for the size case, not an instruction, nothing to act on here. +Filler line 325: padding for the size case, not an instruction, nothing to act on here. +Filler line 326: padding for the size case, not an instruction, nothing to act on here. +Filler line 327: padding for the size case, not an instruction, nothing to act on here. +Filler line 328: padding for the size case, not an instruction, nothing to act on here. +Filler line 329: padding for the size case, not an instruction, nothing to act on here. +Filler line 330: padding for the size case, not an instruction, nothing to act on here. +Filler line 331: padding for the size case, not an instruction, nothing to act on here. +Filler line 332: padding for the size case, not an instruction, nothing to act on here. +Filler line 333: padding for the size case, not an instruction, nothing to act on here. +Filler line 334: padding for the size case, not an instruction, nothing to act on here. +Filler line 335: padding for the size case, not an instruction, nothing to act on here. +Filler line 336: padding for the size case, not an instruction, nothing to act on here. +Filler line 337: padding for the size case, not an instruction, nothing to act on here. +Filler line 338: padding for the size case, not an instruction, nothing to act on here. +Filler line 339: padding for the size case, not an instruction, nothing to act on here. +Filler line 340: padding for the size case, not an instruction, nothing to act on here. +Filler line 341: padding for the size case, not an instruction, nothing to act on here. +Filler line 342: padding for the size case, not an instruction, nothing to act on here. +Filler line 343: padding for the size case, not an instruction, nothing to act on here. +Filler line 344: padding for the size case, not an instruction, nothing to act on here. +Filler line 345: padding for the size case, not an instruction, nothing to act on here. +Filler line 346: padding for the size case, not an instruction, nothing to act on here. +Filler line 347: padding for the size case, not an instruction, nothing to act on here. +Filler line 348: padding for the size case, not an instruction, nothing to act on here. +Filler line 349: padding for the size case, not an instruction, nothing to act on here. +Filler line 350: padding for the size case, not an instruction, nothing to act on here. +Filler line 351: padding for the size case, not an instruction, nothing to act on here. +Filler line 352: padding for the size case, not an instruction, nothing to act on here. +Filler line 353: padding for the size case, not an instruction, nothing to act on here. +Filler line 354: padding for the size case, not an instruction, nothing to act on here. +Filler line 355: padding for the size case, not an instruction, nothing to act on here. +Filler line 356: padding for the size case, not an instruction, nothing to act on here. +Filler line 357: padding for the size case, not an instruction, nothing to act on here. +Filler line 358: padding for the size case, not an instruction, nothing to act on here. +Filler line 359: padding for the size case, not an instruction, nothing to act on here. +Filler line 360: padding for the size case, not an instruction, nothing to act on here. +Filler line 361: padding for the size case, not an instruction, nothing to act on here. +Filler line 362: padding for the size case, not an instruction, nothing to act on here. +Filler line 363: padding for the size case, not an instruction, nothing to act on here. +Filler line 364: padding for the size case, not an instruction, nothing to act on here. +Filler line 365: padding for the size case, not an instruction, nothing to act on here. +Filler line 366: padding for the size case, not an instruction, nothing to act on here. +Filler line 367: padding for the size case, not an instruction, nothing to act on here. +Filler line 368: padding for the size case, not an instruction, nothing to act on here. +Filler line 369: padding for the size case, not an instruction, nothing to act on here. +Filler line 370: padding for the size case, not an instruction, nothing to act on here. +Filler line 371: padding for the size case, not an instruction, nothing to act on here. +Filler line 372: padding for the size case, not an instruction, nothing to act on here. +Filler line 373: padding for the size case, not an instruction, nothing to act on here. +Filler line 374: padding for the size case, not an instruction, nothing to act on here. +Filler line 375: padding for the size case, not an instruction, nothing to act on here. +Filler line 376: padding for the size case, not an instruction, nothing to act on here. +Filler line 377: padding for the size case, not an instruction, nothing to act on here. +Filler line 378: padding for the size case, not an instruction, nothing to act on here. +Filler line 379: padding for the size case, not an instruction, nothing to act on here. +Filler line 380: padding for the size case, not an instruction, nothing to act on here. +Filler line 381: padding for the size case, not an instruction, nothing to act on here. +Filler line 382: padding for the size case, not an instruction, nothing to act on here. +Filler line 383: padding for the size case, not an instruction, nothing to act on here. +Filler line 384: padding for the size case, not an instruction, nothing to act on here. +Filler line 385: padding for the size case, not an instruction, nothing to act on here. +Filler line 386: padding for the size case, not an instruction, nothing to act on here. +Filler line 387: padding for the size case, not an instruction, nothing to act on here. +Filler line 388: padding for the size case, not an instruction, nothing to act on here. +Filler line 389: padding for the size case, not an instruction, nothing to act on here. +Filler line 390: padding for the size case, not an instruction, nothing to act on here. +Filler line 391: padding for the size case, not an instruction, nothing to act on here. +Filler line 392: padding for the size case, not an instruction, nothing to act on here. +Filler line 393: padding for the size case, not an instruction, nothing to act on here. +Filler line 394: padding for the size case, not an instruction, nothing to act on here. +Filler line 395: padding for the size case, not an instruction, nothing to act on here. +Filler line 396: padding for the size case, not an instruction, nothing to act on here. +Filler line 397: padding for the size case, not an instruction, nothing to act on here. +Filler line 398: padding for the size case, not an instruction, nothing to act on here. +Filler line 399: padding for the size case, not an instruction, nothing to act on here. +Filler line 400: padding for the size case, not an instruction, nothing to act on here. +Filler line 401: padding for the size case, not an instruction, nothing to act on here. +Filler line 402: padding for the size case, not an instruction, nothing to act on here. +Filler line 403: padding for the size case, not an instruction, nothing to act on here. +Filler line 404: padding for the size case, not an instruction, nothing to act on here. +Filler line 405: padding for the size case, not an instruction, nothing to act on here. +Filler line 406: padding for the size case, not an instruction, nothing to act on here. +Filler line 407: padding for the size case, not an instruction, nothing to act on here. +Filler line 408: padding for the size case, not an instruction, nothing to act on here. +Filler line 409: padding for the size case, not an instruction, nothing to act on here. +Filler line 410: padding for the size case, not an instruction, nothing to act on here. +Filler line 411: padding for the size case, not an instruction, nothing to act on here. +Filler line 412: padding for the size case, not an instruction, nothing to act on here. +Filler line 413: padding for the size case, not an instruction, nothing to act on here. +Filler line 414: padding for the size case, not an instruction, nothing to act on here. +Filler line 415: padding for the size case, not an instruction, nothing to act on here. +Filler line 416: padding for the size case, not an instruction, nothing to act on here. +Filler line 417: padding for the size case, not an instruction, nothing to act on here. +Filler line 418: padding for the size case, not an instruction, nothing to act on here. +Filler line 419: padding for the size case, not an instruction, nothing to act on here. +Filler line 420: padding for the size case, not an instruction, nothing to act on here. +Filler line 421: padding for the size case, not an instruction, nothing to act on here. +Filler line 422: padding for the size case, not an instruction, nothing to act on here. +Filler line 423: padding for the size case, not an instruction, nothing to act on here. +Filler line 424: padding for the size case, not an instruction, nothing to act on here. +Filler line 425: padding for the size case, not an instruction, nothing to act on here. +Filler line 426: padding for the size case, not an instruction, nothing to act on here. +Filler line 427: padding for the size case, not an instruction, nothing to act on here. +Filler line 428: padding for the size case, not an instruction, nothing to act on here. +Filler line 429: padding for the size case, not an instruction, nothing to act on here. +Filler line 430: padding for the size case, not an instruction, nothing to act on here. +Filler line 431: padding for the size case, not an instruction, nothing to act on here. +Filler line 432: padding for the size case, not an instruction, nothing to act on here. +Filler line 433: padding for the size case, not an instruction, nothing to act on here. +Filler line 434: padding for the size case, not an instruction, nothing to act on here. +Filler line 435: padding for the size case, not an instruction, nothing to act on here. +Filler line 436: padding for the size case, not an instruction, nothing to act on here. +Filler line 437: padding for the size case, not an instruction, nothing to act on here. +Filler line 438: padding for the size case, not an instruction, nothing to act on here. +Filler line 439: padding for the size case, not an instruction, nothing to act on here. +Filler line 440: padding for the size case, not an instruction, nothing to act on here. +Filler line 441: padding for the size case, not an instruction, nothing to act on here. +Filler line 442: padding for the size case, not an instruction, nothing to act on here. +Filler line 443: padding for the size case, not an instruction, nothing to act on here. +Filler line 444: padding for the size case, not an instruction, nothing to act on here. +Filler line 445: padding for the size case, not an instruction, nothing to act on here. +Filler line 446: padding for the size case, not an instruction, nothing to act on here. +Filler line 447: padding for the size case, not an instruction, nothing to act on here. +Filler line 448: padding for the size case, not an instruction, nothing to act on here. +Filler line 449: padding for the size case, not an instruction, nothing to act on here. +Filler line 450: padding for the size case, not an instruction, nothing to act on here. +Filler line 451: padding for the size case, not an instruction, nothing to act on here. +Filler line 452: padding for the size case, not an instruction, nothing to act on here. +Filler line 453: padding for the size case, not an instruction, nothing to act on here. +Filler line 454: padding for the size case, not an instruction, nothing to act on here. +Filler line 455: padding for the size case, not an instruction, nothing to act on here. +Filler line 456: padding for the size case, not an instruction, nothing to act on here. +Filler line 457: padding for the size case, not an instruction, nothing to act on here. +Filler line 458: padding for the size case, not an instruction, nothing to act on here. +Filler line 459: padding for the size case, not an instruction, nothing to act on here. +Filler line 460: padding for the size case, not an instruction, nothing to act on here. diff --git a/dokimi/data/reach-conformance/nested-agents/AGENTS.md b/dokimi/data/reach-conformance/nested-agents/AGENTS.md new file mode 100644 index 000000000..9f772df7b --- /dev/null +++ b/dokimi/data/reach-conformance/nested-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Repository guide + +Code under packages/ has its own guide next to it. diff --git a/dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md b/dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md new file mode 100644 index 000000000..09d2d628e --- /dev/null +++ b/dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md @@ -0,0 +1,3 @@ +# App guide + +The app package uses pytest. Keep tests next to the code. diff --git a/dokimi/data/reach-conformance/override-file/AGENTS.md b/dokimi/data/reach-conformance/override-file/AGENTS.md new file mode 100644 index 000000000..6ee948ce1 --- /dev/null +++ b/dokimi/data/reach-conformance/override-file/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +The base rules, used where no override file sits. diff --git a/dokimi/data/reach-conformance/override-file/AGENTS.override.md b/dokimi/data/reach-conformance/override-file/AGENTS.override.md new file mode 100644 index 000000000..4bff2cc4a --- /dev/null +++ b/dokimi/data/reach-conformance/override-file/AGENTS.override.md @@ -0,0 +1,3 @@ +# Override guide + +The rules that replace the base file in this directory. diff --git a/dokimi/data/reach-conformance/root-agents/AGENTS.md b/dokimi/data/reach-conformance/root-agents/AGENTS.md new file mode 100644 index 000000000..76e6e8dd5 --- /dev/null +++ b/dokimi/data/reach-conformance/root-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +Run `make test` from the repository root. Nothing else applies. diff --git a/dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md new file mode 100644 index 000000000..01e8f4ba6 --- /dev/null +++ b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md @@ -0,0 +1,3 @@ +# Subtree agent guide + +Used only where the directory has no CLAUDE.md. diff --git a/dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md new file mode 100644 index 000000000..73e174a85 --- /dev/null +++ b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md @@ -0,0 +1,3 @@ +# Subtree Claude guide + +This directory has its own CLAUDE.md. diff --git a/dokimi/data/reach-conformance/symlinked-file/AGENTS.md b/dokimi/data/reach-conformance/symlinked-file/AGENTS.md new file mode 100644 index 000000000..483863b75 --- /dev/null +++ b/dokimi/data/reach-conformance/symlinked-file/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +The file CLAUDE.md links to this one. diff --git a/dokimi/data/reach-conformance/symlinked-file/CLAUDE.md b/dokimi/data/reach-conformance/symlinked-file/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/dokimi/data/reach-conformance/symlinked-file/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 4133acaa6..0125d2d4e 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -353,6 +353,10 @@ code. harness profiles, and the guidance reach probe; and how to run its task self-check, agent runs, report and arm comparison (`repo tasks check`, `repo run`, `repo report`, `repo compare`). Tasks and graders stay in the scanned repository. +- [reach-matrix.md](reach-matrix.md) — the reach conformance suite: fixture + repositories, what each harness's vendor documentation says they load, the + `scripts/reach_conformance.py` run and merge that produce the agent × version + matrix, and the nightly workflow that publishes it. - [repo-evaluation-mergeability.md](repo-evaluation-mergeability.md) — *(proposed)* grading whether a maintainer would merge a test-passing change: `rubric.json` (blockers, non-blockers, scope), deterministic scope and diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md new file mode 100644 index 000000000..a9aa71d54 --- /dev/null +++ b/dokimi/docs/reach-matrix.md @@ -0,0 +1,140 @@ +# Reach conformance matrix + +The reach conformance suite asks, for each agent harness and version, which guidance files it loads in a set of +small fixture repositories, and compares what it observed with what the harness's vendor documentation says. A +nightly workflow runs it and publishes the result as the **reach matrix**: harness, version, fixture, file, +scenario, observed state, documented state. + +It answers the question behind agents.md issues #211, #213 and #135: which file a given harness actually reads, +and where its documentation and its behaviour differ. + +## What a run is + +A run is one harness, at one version and one model, over one fixture. It is `repo reach` (see +[repo-evaluation-contract.md](repo-evaluation-contract.md) §7) with `--format receipt`: the effective-instructions +receipt of §7.1 is written to the run's output directory, and each source's observed state is compared with the +documented one. A harness that cannot run here gets a `skipped` run and no cells. + +The matrix has two kinds of record: + +- `runs`: one per harness and fixture, with a status. `ran` means every scenario answered; `errors` means a + receipt came back with a scenario that gave no usable answer; `failed` means no receipt (a timeout, or a + crash before the receipt was written); `skipped` means the harness could not run here, with the reason. +- `cells`: one per source file per scenario, with the observed state, the documented state and an agreement: + `agrees`, `disagrees`, `inconclusive` (observed unknown or only fetched), `no_claim` (the docs say nothing), + or `alias` (a link, whose state is its target's and is not compared). + +The schema is `reach_matrix.v1` (`dokimi/schemas/reach_matrix.v1.json`). + +## The fixtures + +Each directory under `data/reach-conformance/` is one case. Each has a root `AGENTS.md` or `CLAUDE.md` +arrangement that exercises one loading rule. + +| case | what it holds | the rule it tests | +|---|---|---| +| `root-agents` | `AGENTS.md` at the root | the baseline: a root file is read at the root | +| `nested-agents` | a root `AGENTS.md` and `packages/app/AGENTS.md` | a subdirectory file loads when cwd is inside it, or (Claude Code) when a file there is read | +| `claude-and-agents` | `CLAUDE.md` beside `AGENTS.md` | Claude Code reads `AGENTS.md` only when no `CLAUDE.md` is in scope | +| `claude-imports-agents` | `CLAUDE.md` containing `@AGENTS.md` | a Claude Code `@` import is followed; Codex does not read `CLAUDE.md` | +| `override-file` | `AGENTS.md` beside `AGENTS.override.md` | Codex uses the override in place of `AGENTS.md` | +| `subtree-claude-shadow` | `sub/AGENTS.md` beside `sub/CLAUDE.md` | a directory's `CLAUDE.md` shadows its `AGENTS.md` in Claude Code | +| `large-file` | a root `AGENTS.md` of about 40 KB | the size cap; the docs give no cut-off for Claude Code, and 32 KiB for Codex | +| `symlinked-file` | `CLAUDE.md` as a symbolic link to `AGENTS.md` | how a link is named and read; the docs do not say | + +The fixtures are plain directories. `guidance.yaml` next to them is a `repo_manifest.v1` that names every +guidance file glob (`AGENTS.override.md` included), so every harness probes the same set of files whatever its +own profile names. Each fixture holds only guidance files: no tasks and no code. + +## Where the documented states come from + +`data/reach-conformance/expectations.yaml` (`reach_conformance.v1`) holds, for each case and harness, the +documented state of each file in each scenario, and a `why` in words. The sources are the vendor pages: + +- Claude Code: `code.claude.com/docs/en/memory`, the "AGENTS.md" and "How CLAUDE.md files load" sections. +- Codex: `learn.chatgpt.com/docs/agent-configuration/agents-md`, and the `project_doc_max_bytes` and + `project_doc_fallback_filenames` settings. + +A file or scenario the expectations do not list is `unknown`. Then the cell's agreement is `no_claim`: the +observation is reported, and nothing is claimed about it. Copilot has no documented start-up behaviour for its +CLI, so every Copilot cell is `no_claim`; a Copilot run still records what it observed. + +Scenario ids are `cwd=.` (start in the root), `cwd=` (start in a subdirectory) and `read=` (start in +the root and have the model read a file in ``). + +`python scripts/reach_conformance.py check` verifies the expectations against the fixture tree: every fixture +has an entry, every named file exists in its fixture, and every named scenario's directory exists. It exits 1 +on any drift, and `run` refuses to start while there is drift. + +## Running it + +From the dokimi project directory, with the dependencies installed (`make dev`), and a harness CLI on `PATH`: + +```sh +python scripts/reach_conformance.py check +python scripts/reach_conformance.py run --harness codex= --out reach-out +python scripts/reach_conformance.py merge reach-out/*.matrix.json --out reach-matrix.json --md reach-matrix.md +``` + +`--harness NAME[=MODEL]` is repeatable: `claude-code`, `codex` and `copilot` are known. A harness without a model +is skipped, as is one without its credential or its CLI (`unavailable` in `dokimi/repos/conformance.py` +names which). `run` writes `.matrix.json` to `--out`, and the per-fixture receipts under +`-runs/`. Each scenario is capped by `--budget` (USD, default 0.10) and `--timeout` (seconds per +scenario, default 300). `run` exits 1 only when a run is `failed`. + +Credentials are the names the harness profile forwards (`dokimi/repos/profiles/*.yaml`, `env_allow`): +`ANTHROPIC_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK` for Claude Code, `CODEX_API_KEY` for Codex (`OPENAI_API_KEY` is +not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. + +`merge` joins matrices from several harnesses or runs. It refuses a second run of the same harness version on the +same fixture, so a merge cannot double-count. + +The offline tests (`tests/test_reach_conformance.py`) cover the agreement rules, scenario ids, alias handling, +skipped and failed runs, merge, rendering, the drift checks, and the CLI, with the harness runs faked. No test +calls a model. + +## Limits + +- **Prompt capture, not behaviour.** `observed` is what the model reported seeing in its prompt, through marker + tokens (see §7). A model can under-report, and a `loaded` cell does not show the instruction was followed. +- **Links.** A symbolic link is reported as an alias of its target. The receipt cannot show a harness ignoring + the link's name, so an `alias` cell is not evidence that a harness reads or skips it. +- **Unknown is not false.** `unknown` in the documented column means the docs are silent, not that the file is + not loaded. +- **One version per run.** The version is the one the harness reports; a matrix shows the versions that ran. +- **Cost.** Each scenario makes at least one model call, capped at `--budget`. The nightly run covers every + scenario of every fixture for each harness that has a model, a credential and a CLI. + +## Nightly workflow + +`.github/workflows/reach-matrix.yml` runs on a schedule (daily, 04:37 UTC) and on `workflow_dispatch`. It is not a +pull-request check, and it has no `pull_request` trigger. + +- One job per harness. Each checks its credential, installs its CLI only when the credential is present (pinned + versions: `@anthropic-ai/claude-code`, `@openai/codex`, `@github/copilot`), and runs `run` with the model from a + repository variable (`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, `REACH_MODEL_COPILOT`). With no variable, + the harness is recorded as skipped. +- Each job gets only its own secret in its environment. +- The `matrix` job merges the three artifacts, appends the page to the run summary, and uploads + `reach-matrix.json` and `reach-matrix.md` as the `reach-matrix` artifact (90 days). Each harness's matrix is a + 30-day artifact of its own. +- `permissions: {}` at the top; each job has `contents: read`. + +To enable a harness, set its model variable and its secret. Nothing is committed: no key, token or model string +is in the repository. + +Secrets: `ANTHROPIC_API_KEY` (or `AWS_BEARER_TOKEN_BEDROCK`), `CODEX_API_KEY`, `COPILOT_GITHUB_TOKEN`. Variables: +`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, `REACH_MODEL_COPILOT`. Runners: `intel-ubuntu-latest` on the Intel +organisation, `ubuntu-latest` elsewhere; the jobs need `npm` on the runner for the CLI installs. + +## Adding a case or a harness + +A case is a directory under `data/reach-conformance/` with a root guidance arrangement, plus an entry in +`expectations.yaml` for every harness whose documentation says something about it. `check` must pass. + +A harness is a `Harness` in `HARNESSES` (`dokimi/repos/conformance.py`), with its CLI and the credential +names its profile forwards, a profile under `dokimi/repos/profiles/`, its npm pin in the workflow, and a +job in the workflow. Its documented states go in `expectations.yaml` with the vendor page as `why`. + +Both schemas are versioned (`reach_conformance.v1`, `reach_matrix.v1`). A change to a field bumps the version and +the artifact registry (`dokimi/artifacts.py`). diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 550c2ec10..822e5959b 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -12912,6 +12912,7 @@ "'plan'", "'plugin_delta'", "'questions'", + "'reach_matrix'", "'repo_budget'", "'repo_environment'", "'repo_readiness'", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 856b4616f..ffb7d747c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1113,6 +1113,10 @@ Each run lists its own `unknowns` (no usable answer, a source with one token reported, prose above the limit, unseeded tokens). The receipt-level `unknowns` repeat the limits above. +The same receipt, run over the fixtures in `data/reach-conformance/` for several +harnesses and versions, is the reach conformance matrix; see +[reach-matrix.md](reach-matrix.md). + ## 8. L0 scan (`repo_readiness.v1`) `repo scan ` checks every mechanically checkable claim in the guidance diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 893731d14..2475db4d1 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -14,7 +14,7 @@ ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts into an agent's context), ``effective_instructions`` (the effective-instructions/v0 receipt of a reach probe) and ``task_inspect`` (which eval-task format each -path under a tree is). This module stamps each with a +path under a tree is) and ``reach_matrix`` (the reach conformance matrix). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay forward-compatible and self-describing. @@ -60,6 +60,7 @@ "effective_instructions": "effective_instructions.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", + "reach_matrix": "reach_matrix.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py new file mode 100644 index 000000000..df8cf0a00 --- /dev/null +++ b/dokimi/dokimi/repos/conformance.py @@ -0,0 +1,294 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The reach conformance suite (docs/reach-matrix.md): fixture repositories, what each harness's vendor +documentation says they load, and the matrix of what each harness run observed. + +A run is one harness at one version and model over one fixture: ``repo reach --format receipt`` writes its +effective-instructions receipt, and each source's observed state is compared with the documented one. A +harness that cannot run here (no model, credential or CLI) gets a ``skipped`` run and no cells. +""" + +from __future__ import annotations + +import shutil +import sys +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Mapping, Sequence + +import yaml + +from dokimi import process +from dokimi.artifacts import load_artifact, save_artifact, schema_errors + +SCHEMA_EXPECTATIONS = "reach_conformance.v1" +RECEIPT_NAME = "effective_instructions.json" +UNKNOWN = "unknown" +RUN_STATUSES = ("ran", "errors", "failed", "skipped") +AGREEMENTS = ("agrees", "disagrees", "inconclusive", "no_claim", "alias") +DEFAULT_TIMEOUT_S = 300 +DEFAULT_BUDGET_USD = 0.10 + + +class ConformanceError(RuntimeError): + """An expectations file, a matrix or a merge that cannot be used.""" + + +@dataclass(frozen=True) +class Harness: + """A harness the suite knows: its CLI, and the variables one of which must be set for a run.""" + + name: str + bin: str + credentials: tuple[str, ...] + + +# Credentials are the names the profile forwards (repos/profiles/*.yaml env_allow), so a key set here is one the run gets. +HARNESSES: dict[str, Harness] = {h.name: h for h in ( + Harness("claude-code", "claude", ("ANTHROPIC_API_KEY", "AWS_BEARER_TOKEN_BEDROCK")), + Harness("codex", "codex", ("CODEX_API_KEY",)), + Harness("copilot", "copilot", ("COPILOT_GITHUB_TOKEN",)), +)} + + +def unavailable(harness: Harness, model: str, env: Mapping[str, str], + which: Callable[[str], str | None] = shutil.which) -> str | None: + """Why ``harness`` cannot run here, or None when it can.""" + if not model: + return "no model configured for this harness" + if not any(env.get(name) for name in harness.credentials): + return f"none of {', '.join(harness.credentials)} is set" + if which(harness.bin) is None: + return f"{harness.bin} is not on PATH" + return None + + +def cases(fixtures: Path) -> list[str]: + """The fixture repositories: the directories under ``fixtures``, by name.""" + return sorted(p.name for p in fixtures.iterdir() if p.is_dir()) + + +def load_expectations(path: Path) -> dict[str, Any]: + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, ValueError, yaml.YAMLError) as exc: + raise ConformanceError(f"{path}: {exc}") from exc + errors = schema_errors(SCHEMA_EXPECTATIONS, data) + if errors: + raise ConformanceError(f"{path}: " + "; ".join(errors)) + return data + + +def check_expectations(expect: Mapping[str, Any], fixtures: Path) -> list[str]: + """What makes the expectations disagree with the fixture tree; empty when they agree.""" + declared, found = set(expect["cases"]), set(cases(fixtures)) + problems = [f"no expectations for fixture {c}" for c in sorted(found - declared)] + problems += [f"expectations name no fixture {c}" for c in sorted(declared - found)] + for case in sorted(declared & found): + problems += _case_problems(case, expect["cases"][case], fixtures / case) + return problems + + +def _case_problems(case: str, block: Mapping[str, Any], root: Path) -> list[str]: + out = [] + for harness, body in block.items(): + if harness == "summary": + continue + for path, states in body["files"].items(): + if not (root / path).is_file() and not (root / path).is_symlink(): + out.append(f"{case}: {harness} names {path}, which the fixture does not hold") + for scenario in states: + kind, _, where = scenario.partition("=") + if kind not in ("cwd", "read") or not (root / where).is_dir(): + out.append(f"{case}: {harness} names scenario {scenario}, which the fixture does not have") + return out + + +def documented(expect: Mapping[str, Any], case: str, harness: str, path: str, scenario: str) -> str: + """The state the harness's vendor documentation gives this file in this scenario; unknown when it says nothing.""" + block = expect["cases"].get(case, {}).get(harness) + if not block: + return UNKNOWN + return block["files"].get(path, {}).get(scenario, UNKNOWN) + + +def scenario_id(run: Mapping[str, Any]) -> str: + """The reach scenario id of one receipt run: ``read=`` for a run told to read a file, else ``cwd=``.""" + if run.get("target"): + return "read=" + str(PurePosixPath(run["target"]).parent) + return "cwd=" + (run.get("cwd") or ".") + + +def agreement(observed: str, documented_state: str, alias: bool) -> str: + if alias: + return "alias" + if documented_state == UNKNOWN: + return "no_claim" + if observed in (UNKNOWN, "fetched"): + return "inconclusive" + return "agrees" if observed == documented_state else "disagrees" + + +def cells_of(receipt: Mapping[str, Any], case: str, expect: Mapping[str, Any]) -> list[dict[str, Any]]: + """One cell per source per scenario of one receipt (an effective-instructions receipt of one fixture).""" + harness, version = receipt["runtime"]["name"], receipt["runtime"]["version"] + out = [] + for run in receipt["receipts"]: + scenario = scenario_id(run) + for src in run["sources"]: + alias = src.get("alias_of") + state = documented(expect, case, harness, src["path"], scenario) + cell = {"harness": harness, "version": version, "case": case, "scenario": scenario, + "path": src["path"], "observed": src["observed"], "documented": state, + "agreement": agreement(src["observed"], state, alias is not None)} + if alias: + cell["alias_of"] = alias + out.append(cell) + return out + + +def run_record(harness: str, case: str, status: str, reason: str | None, version: str, model: str, + scenarios: int) -> dict[str, Any]: + return {"harness": harness, "version": version, "model": model, "case": case, "status": status, + "reason": reason, "scenarios": scenarios} + + +def summary(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + counts = Counter(c["agreement"] for c in cells) + status = Counter(r["status"] for r in runs) + return {**{a: counts.get(a, 0) for a in AGREEMENTS}, "runs": {s: status.get(s, 0) for s in RUN_STATUSES}} + + +def build_matrix(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]], + generated_at: str | None = None) -> dict[str, Any]: + """The ``reach_matrix.v1`` payload: runs and cells in a stable order, with the counts.""" + return { + "schema_version": "reach_matrix.v1", + "generated_at": generated_at or datetime.now(timezone.utc).isoformat(), + "expectations": SCHEMA_EXPECTATIONS, + "runs": sorted((dict(r) for r in runs), key=lambda r: (r["case"], r["harness"], r["version"])), + "cells": sorted((dict(c) for c in cells), + key=lambda c: (c["case"], c["harness"], c["version"], c["path"], c["scenario"])), + "summary": summary(runs, cells), + } + + +def _reason(proc: process.Completed) -> str: + if proc.timed_out: + return "timed out" + tail = proc.stderr_bytes.decode("utf-8", "replace").strip()[-300:] + return f"exit {proc.returncode}" + (f": {tail}" if tail else "") + + +def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: Path, + expect: Mapping[str, Any], *, timeout: int, budget: float) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Run ``repo reach`` over one fixture; its run record and cells. The receipt is read from ``out``.""" + case = fixture.name + shutil.rmtree(out, ignore_errors=True) # a receipt left by an earlier run is not this run's + argv = [sys.executable, "-m", "dokimi", "repo", "reach", str(fixture), "--manifest-file", str(guidance), + "--harness", harness.name, "--model", model, "--out", str(out), "--format", "receipt", + "--timeout", str(timeout), "--budget", f"{budget:g}"] + proc = process.run(argv, timeout=timeout * 16 + 300, env=process.Env.inherit()) + receipt_path = out / RECEIPT_NAME + if proc.timed_out or not receipt_path.is_file(): + return run_record(harness.name, case, "failed", _reason(proc), UNKNOWN, model, 0), [] + receipt = load_artifact("effective_instructions", receipt_path) + status = "ran" if proc.returncode == 0 else "errors" + reason = None if status == "ran" else _reason(proc) + record = run_record(harness.name, case, status, reason, receipt["runtime"]["version"], model, + len(receipt["receipts"])) + return record, cells_of(receipt, case, expect) + + +def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[str, Any], out_dir: Path, *, + timeout: int = DEFAULT_TIMEOUT_S, budget: float = DEFAULT_BUDGET_USD, + env: Mapping[str, str], which: Callable[[str], str | None] = shutil.which) -> dict[str, Any]: + """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture.""" + reason = unavailable(harness, model, env, which) + guidance = fixtures / "guidance.yaml" + runs: list[dict[str, Any]] = [] + cells: list[dict[str, Any]] = [] + for case in cases(fixtures): + if reason: + runs.append(run_record(harness.name, case, "skipped", reason, UNKNOWN, model, 0)) + continue + record, found = run_case(harness, model, fixtures / case, guidance, out_dir / case, expect, + timeout=timeout, budget=budget) + runs.append(record) + cells += found + return build_matrix(runs, cells) + + +def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """One matrix from several: the runs and cells of each, and no run of the same harness version twice.""" + if not matrices: + raise ConformanceError("nothing to merge") + runs: list[dict[str, Any]] = [] + cells: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for matrix in matrices: + for run in matrix["runs"]: + key = (run["harness"], run["version"], run["case"]) + if key in seen: + raise ConformanceError(f"two matrices hold {run['harness']} {run['version']} on {run['case']}") + seen.add(key) + runs += matrix["runs"] + cells += matrix["cells"] + return build_matrix(runs, cells, generated_at=max(m["generated_at"] for m in matrices)) + + +def write_matrix(matrix: Mapping[str, Any], path: Path) -> None: + save_artifact("reach_matrix", dict(matrix), path) + + +def _cell_text(cell: Mapping[str, Any] | None) -> str: + if cell is None: + return "-" + text = cell["observed"] + if cell["agreement"] == "alias": + text = f"link, {text}" + elif cell["agreement"] == "disagrees": + text += f" (docs: {cell['documented']})" + return text.replace("|", "\\|") + + +def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | None = None) -> str: + """The matrix as a markdown page: the runs, then one table per fixture with a column per harness version.""" + summaries = summaries or {} + counts = matrix["summary"] + columns = sorted({(r["harness"], r["version"]) for r in matrix["runs"] if r["status"] != "skipped"}) + lines = [ + "# Reach matrix", "", + f"Generated {matrix['generated_at']}. Observed states are what each run's model reported seeing " + "(`repo reach`); documented states are the vendor documentation's, in `data/reach-conformance/expectations.yaml`.", + "", + f"Cells: {counts['agrees']} agree with the documentation, {counts['disagrees']} disagree, " + f"{counts['inconclusive']} inconclusive, {counts['no_claim']} with no documented claim, " + f"{counts['alias']} links. Runs: {counts['runs']['ran']} ran, {counts['runs']['errors']} with errors, " + f"{counts['runs']['failed']} failed, {counts['runs']['skipped']} skipped.", "", + "## Runs", "", "| fixture | harness | version | model | status | reason |", "|---|---|---|---|---|---|", + ] + lines += [f"| {r['case']} | {r['harness']} | {r['version']} | {r['model'] or '-'} | {r['status']} | " + f"{(r['reason'] or '-').replace('|', chr(92) + '|')} |" for r in matrix["runs"]] + lines += ["", "Cell legend: `loaded` or `not_loaded` as observed; `fetched` is read with a tool first; " + "`unknown` is no usable answer. `(docs: X)` marks a disagreement with the documentation; a " + "cell with no `(docs: ...)` agrees, or the documentation makes no claim.", ""] + for case in sorted({c["case"] for c in matrix["cells"]}): + lines += _case_section(matrix, case, columns, summaries.get(case, "")) + return "\n".join(lines).rstrip() + "\n" + + +def _case_section(matrix: Mapping[str, Any], case: str, columns: list[tuple[str, str]], summary_text: str) -> list[str]: + cells = [c for c in matrix["cells"] if c["case"] == case] + index = {(c["harness"], c["version"], c["path"], c["scenario"]): c for c in cells} + rows = sorted({(c["path"], c["scenario"]) for c in cells}) + header = ["file", "scenario"] + [f"{h} {v}" for h, v in columns] + out = [f"## {case}", ""] + ([summary_text, ""] if summary_text else []) + out += ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)] + for path, scenario in rows: + texts = [_cell_text(index.get((h, v, path, scenario))) for h, v in columns] + out.append("| " + " | ".join([path, scenario, *texts]) + " |") + return out + [""] diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index b9b3d2cb1..a8504e86e 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -67,6 +67,7 @@ class KindFields: counts=("receipts",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "reach_matrix": KindFields(time=("generated_at",), counts=("runs", "cells")), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/reach_conformance.v1.json b/dokimi/dokimi/schemas/reach_conformance.v1.json new file mode 100644 index 000000000..4cb7ad1c5 --- /dev/null +++ b/dokimi/dokimi/schemas/reach_conformance.v1.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/reach_conformance.v1.json", + "title": "Reach conformance expectations v1", + "description": "What each harness's vendor documentation says each fixture repository loads, per reach scenario (data/reach-conformance/expectations.yaml, docs/reach-matrix.md). A file or scenario that is not listed is unknown: the documentation does not say.", + "type": "object", + "required": ["schema", "cases"], + "properties": { + "schema": {"const": "reach_conformance.v1"}, + "cases": { + "type": "object", + "description": "Keyed by the fixture repository's directory name.", + "additionalProperties": { + "type": "object", + "required": ["summary"], + "properties": {"summary": {"type": "string"}}, + "additionalProperties": {"$ref": "#/$defs/harness"} + } + } + }, + "$defs": { + "harness": { + "type": "object", + "required": ["why", "files"], + "additionalProperties": false, + "properties": { + "why": {"type": "string", "description": "The vendor documentation the states rest on, in short."}, + "files": { + "type": "object", + "description": "Keyed by the guidance file's path inside the fixture.", + "additionalProperties": { + "type": "object", + "description": "Keyed by reach scenario id: cwd=., cwd= or read=.", + "additionalProperties": {"enum": ["loaded", "not_loaded", "unknown"]} + } + } + } + } + } +} diff --git a/dokimi/dokimi/schemas/reach_matrix.v1.json b/dokimi/dokimi/schemas/reach_matrix.v1.json new file mode 100644 index 000000000..1cfc8459b --- /dev/null +++ b/dokimi/dokimi/schemas/reach_matrix.v1.json @@ -0,0 +1,68 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/reach_matrix.v1.json", + "title": "Reach conformance matrix v1", + "description": "The matrix of the reach conformance suite (docs/reach-matrix.md): for each harness run over each fixture repository, the state each guidance file was observed in, per reach scenario, next to what the harness's vendor documentation says (reach_conformance.v1). One matrix is written per harness and several are merged into one. A `skipped` run has no cells; a cell whose documented state is `unknown` makes no claim.", + "type": "object", + "required": ["schema_version", "generated_at", "expectations", "runs", "cells", "summary"], + "properties": { + "schema_version": {"const": "reach_matrix.v1"}, + "generated_at": {"type": "string"}, + "expectations": {"const": "reach_conformance.v1", "description": "The schema of the expectations file the documented states come from."}, + "runs": { + "type": "array", + "items": { + "type": "object", + "required": ["harness", "version", "model", "case", "status", "reason", "scenarios"], + "properties": { + "harness": {"type": "string"}, + "version": {"type": "string", "description": "The harness's own version as reach reports it; `unknown` when no receipt came back."}, + "model": {"type": "string", "description": "The model string passed to the harness; empty when none was configured."}, + "case": {"type": "string", "description": "The fixture repository's directory name."}, + "status": {"enum": ["ran", "errors", "failed", "skipped"], "description": "ran: every scenario answered; errors: a receipt came back with at least one scenario that gave no usable answer; failed: no receipt; skipped: the harness could not run here."}, + "reason": {"type": ["string", "null"]}, + "scenarios": {"type": "integer", "minimum": 0} + } + } + }, + "cells": { + "type": "array", + "items": { + "type": "object", + "required": ["harness", "version", "case", "scenario", "path", "observed", "documented", "agreement"], + "properties": { + "harness": {"type": "string"}, + "version": {"type": "string"}, + "case": {"type": "string"}, + "scenario": {"type": "string", "pattern": "^(cwd=|read=)"}, + "path": {"type": "string"}, + "alias_of": {"type": "string", "description": "Set when the path is a link; its observed state is the target's."}, + "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, + "documented": {"enum": ["loaded", "not_loaded", "unknown"]}, + "agreement": {"enum": ["agrees", "disagrees", "inconclusive", "no_claim", "alias"], "description": "agrees and disagrees compare observed with documented; inconclusive: observed is unknown or fetched; no_claim: the docs say nothing; alias: a link, not compared."} + } + } + }, + "summary": { + "type": "object", + "required": ["agrees", "disagrees", "inconclusive", "no_claim", "alias", "runs"], + "properties": { + "agrees": {"type": "integer", "minimum": 0}, + "disagrees": {"type": "integer", "minimum": 0}, + "inconclusive": {"type": "integer", "minimum": 0}, + "no_claim": {"type": "integer", "minimum": 0}, + "alias": {"type": "integer", "minimum": 0}, + "runs": { + "type": "object", + "required": ["ran", "errors", "failed", "skipped"], + "properties": { + "ran": {"type": "integer", "minimum": 0}, + "errors": {"type": "integer", "minimum": 0}, + "failed": {"type": "integer", "minimum": 0}, + "skipped": {"type": "integer", "minimum": 0} + } + } + } + } + } +} diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 0f2d53b27..33c29ea68 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -129,6 +129,7 @@ nav: - repo-evaluation-contract.md - repo-evaluation-mergeability.md - repo-evaluation-agent-modes.md + - reach-matrix.md - preparing-a-repository.md - preparing-a-repository-evidence.md - Extending Dokimi: diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py new file mode 100644 index 000000000..35995c080 --- /dev/null +++ b/dokimi/scripts/reach_conformance.py @@ -0,0 +1,108 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Drive the reach conformance suite (docs/reach-matrix.md). + + python scripts/reach_conformance.py check + python scripts/reach_conformance.py run --harness codex= --out DIR + python scripts/reach_conformance.py merge DIR/*.matrix.json --out DIR/reach-matrix.json --md DIR/reach-matrix.md + +``run`` makes paid model calls for every harness that has a model, a credential and a CLI here; the rest are +written as skipped runs. Run it from the dokimi project directory (``python -m dokimi`` must resolve). +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from dokimi.artifacts import load_artifact +from dokimi.repos import conformance as cf + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_FIXTURES = ROOT / "data" / "reach-conformance" +DEFAULT_EXPECTATIONS = DEFAULT_FIXTURES / "expectations.yaml" + + +def _expectations(path: Path) -> dict: + return cf.load_expectations(path) + + +def cmd_check(args: argparse.Namespace) -> int: + problems = cf.check_expectations(_expectations(args.expectations), args.fixtures) + for problem in problems: + print(problem, file=sys.stderr) + if not problems: + print(f"ok: {len(cf.cases(args.fixtures))} fixtures, expectations agree") + return 1 if problems else 0 + + +def _harness_spec(text: str) -> tuple[cf.Harness, str]: + name, _, model = text.partition("=") + if name not in cf.HARNESSES: + raise argparse.ArgumentTypeError(f"unknown harness {name!r}; known: {', '.join(cf.HARNESSES)}") + return cf.HARNESSES[name], model + + +def cmd_run(args: argparse.Namespace) -> int: + expect = _expectations(args.expectations) + problems = cf.check_expectations(expect, args.fixtures) + if problems: + print("\n".join(problems), file=sys.stderr) + return 1 + failed = False + args.out.mkdir(parents=True, exist_ok=True) + for harness, model in args.harness: + matrix = cf.run_harness(harness, model, args.fixtures, expect, args.out / f"{harness.name}-runs", + timeout=args.timeout, budget=args.budget, env=os.environ) + cf.write_matrix(matrix, args.out / f"{harness.name}.matrix.json") + counts = matrix["summary"]["runs"] + print(f"{harness.name}: ran {counts['ran']}, errors {counts['errors']}, " + f"failed {counts['failed']}, skipped {counts['skipped']}") + failed = failed or counts["failed"] > 0 + return 1 if failed else 0 + + +def cmd_merge(args: argparse.Namespace) -> int: + expect = _expectations(args.expectations) + matrices = [load_artifact("reach_matrix", path) for path in args.inputs] + merged = cf.merge(matrices) + cf.write_matrix(merged, args.out) + summaries = {case: block["summary"] for case, block in expect["cases"].items()} + args.md.write_text(cf.render_markdown(merged, summaries), encoding="utf-8") + print(f"merged {len(matrices)} matrices: {len(merged['runs'])} runs, {len(merged['cells'])} cells") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--fixtures", type=Path, default=DEFAULT_FIXTURES) + parser.add_argument("--expectations", type=Path, default=DEFAULT_EXPECTATIONS) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check", help="verify the expectations against the fixture tree") + run = sub.add_parser("run", help="run reach over the fixtures for each harness") + run.add_argument("--harness", action="append", type=_harness_spec, required=True, + metavar="NAME[=MODEL]", help="repeatable; no model makes the harness skipped") + run.add_argument("--out", type=Path, required=True, help="directory for .matrix.json") + run.add_argument("--timeout", type=int, default=cf.DEFAULT_TIMEOUT_S, help="per-scenario seconds") + run.add_argument("--budget", type=float, default=cf.DEFAULT_BUDGET_USD, help="per-scenario USD cap") + merge = sub.add_parser("merge", help="merge matrices into one JSON and one markdown page") + merge.add_argument("inputs", nargs="+", type=Path) + merge.add_argument("--out", type=Path, required=True) + merge.add_argument("--md", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "run": + return cmd_run(args) + if args.command == "merge": + return cmd_merge(args) + return cmd_check(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dokimi/tests/fixtures/artifacts/reach_conformance.v1.json b/dokimi/tests/fixtures/artifacts/reach_conformance.v1.json new file mode 100644 index 000000000..5b88bb037 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/reach_conformance.v1.json @@ -0,0 +1,22 @@ +{ + "schema": "reach_conformance.v1", + "cases": { + "nested-agents": { + "summary": "A root AGENTS.md and a packages/app/AGENTS.md below it.", + "codex": { + "why": "Codex walks from the git root down to cwd, one file per directory.", + "files": { + "AGENTS.md": {"cwd=.": "loaded", "cwd=packages/app": "loaded"}, + "packages/app/AGENTS.md": {"cwd=.": "not_loaded", "cwd=packages/app": "loaded"} + } + } + }, + "large-file": { + "summary": "A root AGENTS.md above the size cap.", + "claude-code": { + "why": "No size limit is documented.", + "files": {} + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json new file mode 100644 index 000000000..0b728a016 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json @@ -0,0 +1,66 @@ +{ + "schema_version": "reach_matrix.v1", + "generated_at": "2026-10-31T04:37:00+00:00", + "expectations": "reach_conformance.v1", + "runs": [ + { + "harness": "codex", + "version": "0.166.0", + "model": "example-model", + "case": "nested-agents", + "status": "ran", + "reason": null, + "scenarios": 2 + }, + { + "harness": "codex", + "version": "unknown", + "model": "", + "case": "nested-agents", + "status": "skipped", + "reason": "none of CODEX_API_KEY is set", + "scenarios": 0 + } + ], + "cells": [ + { + "harness": "codex", + "version": "0.166.0", + "case": "nested-agents", + "scenario": "cwd=packages/app", + "path": "packages/app/AGENTS.md", + "observed": "loaded", + "documented": "loaded", + "agreement": "agrees" + }, + { + "harness": "codex", + "version": "0.166.0", + "case": "nested-agents", + "scenario": "cwd=.", + "path": "AGENTS.md", + "observed": "not_loaded", + "documented": "loaded", + "agreement": "disagrees" + }, + { + "harness": "codex", + "version": "0.166.0", + "case": "nested-agents", + "scenario": "cwd=.", + "path": "LINK.md", + "alias_of": "AGENTS.md", + "observed": "loaded", + "documented": "unknown", + "agreement": "alias" + } + ], + "summary": { + "agrees": 1, + "disagrees": 1, + "inconclusive": 0, + "no_claim": 0, + "alias": 1, + "runs": {"ran": 1, "errors": 0, "failed": 0, "skipped": 1} + } +} diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py new file mode 100644 index 000000000..abcb9af1a --- /dev/null +++ b/dokimi/tests/test_reach_conformance.py @@ -0,0 +1,302 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The reach conformance suite, offline: which harnesses can run, how observed and documented states agree, +the matrix and its markdown page, and the fixtures against their expectations. Harness runs are faked.""" + +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path + +import pytest + +from dokimi import process +from dokimi.artifacts import load_artifact, save_artifact, schema_errors +from dokimi.repos import conformance as cf +from dokimi.repos.receipt import build_receipt +from tests.test_repo_receipt import BUDGET, ROOT_ROW, _record, _reach, _tree + +PROJECT = Path(__file__).resolve().parents[1] +FIXTURES = PROJECT / "data" / "reach-conformance" +SCRIPT = PROJECT / "scripts" / "reach_conformance.py" +CODEX = cf.HARNESSES["codex"] +CLAUDE = cf.HARNESSES["claude-code"] +CASE_BLOCK = {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}, "CLAUDE.md": {"cwd=.": "not_loaded"}}} + + +def _expect(**blocks: dict) -> dict: + return {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "toy case", **blocks}}} + + +def _found(name: str) -> str: + return f"/usr/bin/{name}" + + +def _missing(name: str) -> None: + return None + + +def _load_cli(): + spec = importlib.util.spec_from_file_location("reach_conformance_cli", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +cli = _load_cli() + + +def _run(**fields) -> dict: + base = {"harness": "codex", "version": "0.166.0", "model": "m", "case": "toy", "status": "ran", + "reason": None, "scenarios": 1} + return {**base, **fields} + + +def _cell(**fields) -> dict: + base = {"harness": "codex", "version": "0.166.0", "case": "toy", "scenario": "cwd=.", "path": "AGENTS.md", + "observed": "loaded", "documented": "loaded", "agreement": "agrees"} + return {**base, **fields} + + +def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00") -> dict: + return cf.build_matrix(runs or [_run()], cells or [_cell()], generated_at=generated_at) + + +def test_unavailable_names_the_missing_piece(): + env = {"CODEX_API_KEY": "k"} + assert cf.unavailable(CODEX, "gpt", env, which=_found) is None + assert "no model" in cf.unavailable(CODEX, "", env, which=_found) + assert "CODEX_API_KEY" in cf.unavailable(CODEX, "gpt", {}, which=_found) + assert "not on PATH" in cf.unavailable(CODEX, "gpt", env, which=_missing) + + +def test_unavailable_forwards_only_the_credentials_the_profile_forwards(): + # codex's profile forwards CODEX_API_KEY alone, so OPENAI_API_KEY would not reach the run + assert cf.unavailable(CODEX, "gpt", {"OPENAI_API_KEY": "k"}, which=_found) is not None + assert cf.unavailable(CLAUDE, "m", {"AWS_BEARER_TOKEN_BEDROCK": "t"}, which=_found) is None + + +@pytest.mark.parametrize(("observed", "documented", "alias", "expected"), [ + ("loaded", "loaded", False, "agrees"), + ("not_loaded", "loaded", False, "disagrees"), + ("loaded", "not_loaded", False, "disagrees"), + ("loaded", "unknown", False, "no_claim"), + ("unknown", "unknown", False, "no_claim"), + ("unknown", "loaded", False, "inconclusive"), + ("fetched", "loaded", False, "inconclusive"), + ("loaded", "loaded", True, "alias"), +]) +def test_agreement_rules(observed, documented, alias, expected): + assert cf.agreement(observed, documented, alias) == expected + + +@pytest.mark.parametrize(("run", "scenario"), [ + ({"cwd": ".", "target": None}, "cwd=."), + ({"cwd": "packages/app", "target": None}, "cwd=packages/app"), + ({"cwd": ".", "target": "packages/app/REACH_PROBE.txt"}, "read=packages/app"), +]) +def test_scenario_id_names_the_cwd_or_the_read_directory(run, scenario): + assert cf.scenario_id(run) == scenario + + +def test_documented_is_unknown_unless_the_docs_name_the_file_and_scenario(): + expect = _expect(codex=CASE_BLOCK) + assert cf.documented(expect, "toy", "codex", "CLAUDE.md", "cwd=.") == "not_loaded" + assert cf.documented(expect, "toy", "codex", "CLAUDE.md", "cwd=sub") == "unknown" + assert cf.documented(expect, "toy", "claude-code", "AGENTS.md", "cwd=.") == "unknown" + assert cf.documented(expect, "other", "codex", "AGENTS.md", "cwd=.") == "unknown" + + +def test_cells_compare_each_source_with_the_documentation(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + cells = cf.cells_of(receipt, "toy", _expect(**{"claude-code": { + "why": "docs", "files": {"AGENTS.md": {"cwd=.": "not_loaded"}}}})) + agents = next(c for c in cells if c["path"] == "AGENTS.md") + assert agents == {"harness": "claude-code", "version": "2.1.0", "case": "toy", "scenario": "cwd=.", + "path": "AGENTS.md", "observed": "loaded", "documented": "not_loaded", + "agreement": "disagrees"} + + +def test_a_symbolic_link_is_an_alias_of_its_target(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + os.symlink("AGENTS.md", tmp_path / "CLAUDE.md") + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002", "aliases": ["CLAUDE.md"]}] + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW], files=files), BUDGET, + tree) + link = next(c for c in cf.cells_of(receipt, "toy", _expect()) if c["path"] == "CLAUDE.md") + assert (link["agreement"], link["alias_of"], link["observed"]) == ("alias", "AGENTS.md", "loaded") + + +def test_build_matrix_counts_and_validates_against_the_schema(): + runs = [_run(), _run(harness="copilot", status="skipped", reason="no key", version="unknown")] + cells = [_cell(), _cell(agreement="disagrees", observed="not_loaded", documented="loaded", path="CLAUDE.md"), + _cell(agreement="alias", path="LINK.md", alias_of="AGENTS.md")] + matrix = cf.build_matrix(runs, cells, generated_at="2026-10-31T00:00:00+00:00") + assert matrix["summary"] == {"agrees": 1, "disagrees": 1, "inconclusive": 0, "no_claim": 0, "alias": 1, + "runs": {"ran": 1, "errors": 0, "failed": 0, "skipped": 1}} + assert schema_errors("reach_matrix.v1", matrix) == [] + + +def _fake_reach(monkeypatch, receipt: dict, *, returncode: int = 0, timed_out: bool = False, + write: bool = True) -> list[list[str]]: + calls: list[list[str]] = [] + + def fake_run(argv, *, timeout, env, **_): + calls.append(list(argv)) + out = Path(argv[argv.index("--out") + 1]) + if write: + out.mkdir(parents=True, exist_ok=True) + save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + return process.Completed(argv=tuple(argv), returncode=returncode, timed_out=timed_out, + stderr_bytes=b"boom") + + monkeypatch.setattr(cf.process, "run", fake_run) + return calls + + +def _toy_fixture(tmp_path: Path) -> tuple[Path, dict]: + fixture = tmp_path / "toy" + tree = _tree(fixture, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + return fixture, receipt + + +def test_run_case_reads_the_receipt_the_run_wrote(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + calls = _fake_reach(monkeypatch, receipt) + expect = _expect(**{"claude-code": {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}}}}) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "guidance.yaml", tmp_path / "out", expect, + timeout=30, budget=0.5) + assert record == _run(harness="claude-code", version="2.1.0", model="m", status="ran", + scenarios=len(receipt["receipts"])) + assert any(c["path"] == "AGENTS.md" and c["agreement"] == "agrees" for c in cells) + argv = calls[0] + assert argv[argv.index("--harness") + 1] == "claude-code" and argv[argv.index("--format") + 1] == "receipt" + + +def test_run_case_marks_a_run_with_a_failing_scenario_as_errors(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, returncode=1) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5) + assert (record["status"], record["reason"]) == ("errors", "exit 1: boom") + assert cells + + +@pytest.mark.parametrize(("kwargs", "reason"), [ + ({"write": False}, "exit 0"), + ({"timed_out": True, "returncode": process.TIMEOUT_RC}, "timed out"), +]) +def test_run_case_without_a_receipt_is_failed(tmp_path, monkeypatch, kwargs, reason): + fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, **kwargs) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5) + assert record["status"] == "failed" and reason in record["reason"] + assert cells == [] + + +def test_run_harness_skips_every_fixture_when_the_harness_cannot_run(tmp_path, monkeypatch): + def must_not_run(*_args, **_kwargs): + raise AssertionError("an unavailable harness must not be run") + + monkeypatch.setattr(cf.process, "run", must_not_run) + matrix = cf.run_harness(CODEX, "gpt", FIXTURES, cf.load_expectations(FIXTURES / "expectations.yaml"), + tmp_path, env={}, which=_found) + assert {r["case"] for r in matrix["runs"]} == set(cf.cases(FIXTURES)) + assert all(r["status"] == "skipped" and "CODEX_API_KEY" in r["reason"] for r in matrix["runs"]) + assert matrix["cells"] == [] and matrix["summary"]["runs"]["skipped"] == len(matrix["runs"]) + + +def test_merge_joins_matrices_and_keeps_the_latest_time(): + first = _matrix(generated_at="2026-10-30T00:00:00+00:00") + second = _matrix(runs=[_run(harness="claude-code", version="2.1.0")], + cells=[_cell(harness="claude-code", version="2.1.0")], + generated_at="2026-10-31T00:00:00+00:00") + merged = cf.merge([first, second]) + assert len(merged["runs"]) == 2 and len(merged["cells"]) == 2 + assert merged["generated_at"] == "2026-10-31T00:00:00+00:00" + assert schema_errors("reach_matrix.v1", merged) == [] + + +def test_merge_refuses_a_harness_version_on_one_fixture_twice(): + with pytest.raises(cf.ConformanceError, match="codex 0.166.0 on toy"): + cf.merge([_matrix(), _matrix()]) + + +def test_merge_refuses_nothing(): + with pytest.raises(cf.ConformanceError, match="nothing to merge"): + cf.merge([]) + + +def test_render_markdown_shows_observed_documented_and_links(): + matrix = _matrix( + runs=[_run(), _run(harness="copilot", version="1.0.105", status="skipped", reason="a|b")], + cells=[_cell(), _cell(path="CLAUDE.md", observed="not_loaded", documented="loaded", agreement="disagrees"), + _cell(path="LINK.md", observed="loaded", documented="unknown", agreement="alias", alias_of="AGENTS.md")], + ) + md = cf.render_markdown(matrix, {"toy": "toy case"}) + assert md.isascii() + assert "## toy" in md and "toy case" in md + assert "| file | scenario | codex 0.166.0 |" in md + assert "| AGENTS.md | cwd=. | loaded |" in md + assert "| CLAUDE.md | cwd=. | not_loaded (docs: loaded) |" in md + assert "link, loaded" in md + assert "a\\|b" in md + assert "copilot 1.0.105" not in md.split("## toy")[1] + + +def test_expectations_match_the_fixture_tree(): + expect = cf.load_expectations(FIXTURES / "expectations.yaml") + assert cf.check_expectations(expect, FIXTURES) == [] + assert {"large-file", "symlinked-file", "subtree-claude-shadow"} <= set(cf.cases(FIXTURES)) + + +def test_check_expectations_reports_each_kind_of_drift(tmp_path): + fixtures = tmp_path / "fixtures" + _tree(fixtures / "toy", {"AGENTS.md": "a\n"}) + (fixtures / "extra").mkdir() + expect = {"schema": "reach_conformance.v1", "cases": { + "toy": {"summary": "s", "codex": {"why": "w", "files": { + "CLAUDE.md": {"cwd=.": "loaded"}, "AGENTS.md": {"cwd=nope": "loaded"}}}}, + "gone": {"summary": "s"}, + }} + problems = "\n".join(cf.check_expectations(expect, fixtures)) + assert "no expectations for fixture extra" in problems + assert "expectations name no fixture gone" in problems + assert "codex names CLAUDE.md, which the fixture does not hold" in problems + assert "codex names scenario cwd=nope" in problems + + +def test_load_expectations_rejects_another_schema(tmp_path): + path = tmp_path / "expectations.yaml" + path.write_text("schema: other.v1\ncases: {}\n", encoding="utf-8") + with pytest.raises(cf.ConformanceError): + cf.load_expectations(path) + + +def test_cli_check_passes_on_the_fixtures_and_fails_on_drift(tmp_path, capsys): + assert cli.main(["check"]) == 0 + (tmp_path / "undeclared").mkdir() + assert cli.main(["--fixtures", str(tmp_path), "--expectations", str(FIXTURES / "expectations.yaml"), + "check"]) == 1 + assert "no expectations for fixture undeclared" in capsys.readouterr().err + + +def test_cli_run_rejects_an_unknown_harness(tmp_path): + with pytest.raises(SystemExit): + cli.main(["run", "--harness", "nope=x", "--out", str(tmp_path)]) + + +def test_cli_merge_writes_json_and_markdown(tmp_path): + left, right = tmp_path / "left.json", tmp_path / "right.json" + cf.write_matrix(_matrix(), left) + cf.write_matrix(_matrix(runs=[_run(harness="claude-code", version="2.1.0")], + cells=[_cell(harness="claude-code", version="2.1.0")]), right) + out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" + assert cli.main(["merge", str(left), str(right), "--out", str(out), "--md", str(md)]) == 0 + assert load_artifact("reach_matrix", out)["summary"]["agrees"] == 2 + assert md.read_text(encoding="utf-8").startswith("# Reach matrix") diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 3a24afd19..7fd02e024 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -49,6 +49,7 @@ "effective_instructions.v1": ({}, "toy", "m", "2026-10-31T00:00:00+00:00"), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "reach_matrix.v1": ({}, None, None, "2026-10-31T04:37:00+00:00"), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 170c1138b..14d577355 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -33,6 +33,7 @@ "questions.v2": {"questions": 3}, "repo_budget.v1": {"entries": 9}, "effective_instructions.v1": {"receipts": 1}, + "reach_matrix.v1": {"runs": 2, "cells": 3}, "repo_environment.v1": {"instructions": 4, "findings": 13}, "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6}, From 92ad6ea1f8cb1340ae40f20e93370359be3039ce Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:32:59 -0700 Subject: [PATCH 04/29] fix(repo): a malformed reach receipt fails its run, not the whole harness load_artifact raised on a receipt that did not validate, which aborted the remaining fixtures and harnesses of the nightly run. Such a run is now failed with the validation error as its reason. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/conformance.py | 7 +++++-- dokimi/tests/test_reach_conformance.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index df8cf0a00..232df2d84 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -22,7 +22,7 @@ import yaml from dokimi import process -from dokimi.artifacts import load_artifact, save_artifact, schema_errors +from dokimi.artifacts import ArtifactValidationError, load_artifact, save_artifact, schema_errors SCHEMA_EXPECTATIONS = "reach_conformance.v1" RECEIPT_NAME = "effective_instructions.json" @@ -195,7 +195,10 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P receipt_path = out / RECEIPT_NAME if proc.timed_out or not receipt_path.is_file(): return run_record(harness.name, case, "failed", _reason(proc), UNKNOWN, model, 0), [] - receipt = load_artifact("effective_instructions", receipt_path) + try: + receipt = load_artifact("effective_instructions", receipt_path) + except ArtifactValidationError as exc: # one bad receipt fails its run, not the whole harness + return run_record(harness.name, case, "failed", f"invalid receipt: {exc}", UNKNOWN, model, 0), [] status = "ran" if proc.returncode == 0 else "errors" reason = None if status == "ran" else _reason(proc) record = run_record(harness.name, case, status, reason, receipt["runtime"]["version"], model, diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index abcb9af1a..257df2a88 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -199,6 +199,22 @@ def test_run_case_without_a_receipt_is_failed(tmp_path, monkeypatch, kwargs, rea assert cells == [] +def test_run_case_with_a_malformed_receipt_is_failed_not_fatal(tmp_path, monkeypatch): + fixture, _receipt = _toy_fixture(tmp_path) + + def fake_run(argv, *, timeout, env, **_): + out = Path(argv[argv.index("--out") + 1]) + out.mkdir(parents=True, exist_ok=True) + (out / cf.RECEIPT_NAME).write_text('{"schema_version": "effective_instructions.v1"}', encoding="utf-8") + return process.Completed(argv=tuple(argv), returncode=0) + + monkeypatch.setattr(cf.process, "run", fake_run) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5) + assert record["status"] == "failed" and record["reason"].startswith("invalid receipt") + assert cells == [] + + def test_run_harness_skips_every_fixture_when_the_harness_cannot_run(tmp_path, monkeypatch): def must_not_run(*_args, **_kwargs): raise AssertionError("an unavailable harness must not be run") From 41ef2c0acc6c45b9f3ccab9437924056b671c5be Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:38:59 -0700 Subject: [PATCH 05/29] docs(repo): keep the reach matrix comment blocks to two lines The static comment-block check fails a block over two lines. The sources and the design move to docs/reach-matrix.md, which both comments point to. Signed-off-by: Nikolay Petrov --- .github/workflows/reach-matrix.yml | 6 ++---- dokimi/data/reach-conformance/expectations.yaml | 7 ++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reach-matrix.yml b/.github/workflows/reach-matrix.yml index c954587c4..2ae214eeb 100644 --- a/.github/workflows/reach-matrix.yml +++ b/.github/workflows/reach-matrix.yml @@ -3,10 +3,8 @@ name: reach-matrix -# The nightly reach conformance matrix (dokimi/docs/reach-matrix.md): each harness runs `repo reach` over the -# fixtures in dokimi/data/reach-conformance, and the three matrices merge into one page. This makes paid model -# calls, so it is not a check on pull requests. A harness with no model variable, no credential or no CLI on -# the runner is recorded as skipped; it does not fail the run. +# Paid model calls, so never a PR check; a harness without a model, credential or CLI is skipped, not failed. +# Design and secrets: dokimi/docs/reach-matrix.md. on: schedule: - cron: '37 4 * * *' diff --git a/dokimi/data/reach-conformance/expectations.yaml b/dokimi/data/reach-conformance/expectations.yaml index d3692af42..bd9e11cac 100644 --- a/dokimi/data/reach-conformance/expectations.yaml +++ b/dokimi/data/reach-conformance/expectations.yaml @@ -1,11 +1,8 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# What each harness's vendor documentation says each fixture loads, per reach scenario -# (cwd=. is the root, cwd= starts there, read= starts at the root and reads a file in ). -# A file or scenario not listed is `unknown`: the docs do not say, so the matrix reports the -# observation without a claim. Sources: code.claude.com/docs/en/memory (Claude Code, "AGENTS.md" -# and "How CLAUDE.md files load") and learn.chatgpt.com/docs/agent-configuration/agents-md (Codex). +# Vendor-documented state per file and reach scenario; unlisted means unknown (no claim). +# Sources and scenario meanings: docs/reach-matrix.md. schema: reach_conformance.v1 cases: root-agents: From 84a7bc1c6958032403fe534705d22e0cfceec398 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:43:57 -0700 Subject: [PATCH 06/29] fix(repo): every unreadable reach receipt fails its run The catch named only the schema error, so a receipt that is not JSON or not UTF-8 still aborted the harness. Catch OSError and ValueError, which covers both, and test a truncated and a non-object receipt. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/conformance.py | 4 ++-- dokimi/tests/test_reach_conformance.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 232df2d84..5adcb6505 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -22,7 +22,7 @@ import yaml from dokimi import process -from dokimi.artifacts import ArtifactValidationError, load_artifact, save_artifact, schema_errors +from dokimi.artifacts import load_artifact, save_artifact, schema_errors SCHEMA_EXPECTATIONS = "reach_conformance.v1" RECEIPT_NAME = "effective_instructions.json" @@ -197,7 +197,7 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P return run_record(harness.name, case, "failed", _reason(proc), UNKNOWN, model, 0), [] try: receipt = load_artifact("effective_instructions", receipt_path) - except ArtifactValidationError as exc: # one bad receipt fails its run, not the whole harness + except (OSError, ValueError) as exc: # one bad receipt fails its run, not the whole harness return run_record(harness.name, case, "failed", f"invalid receipt: {exc}", UNKNOWN, model, 0), [] status = "ran" if proc.returncode == 0 else "errors" reason = None if status == "ran" else _reason(proc) diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 257df2a88..f4ab3582d 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -199,13 +199,18 @@ def test_run_case_without_a_receipt_is_failed(tmp_path, monkeypatch, kwargs, rea assert cells == [] -def test_run_case_with_a_malformed_receipt_is_failed_not_fatal(tmp_path, monkeypatch): +@pytest.mark.parametrize("text", [ + '{"schema_version": "effective_instructions.v1"}', # fails the schema + '{"schema_version": "effective_instructions.v1", "receipts": [', # truncated JSON + "[1, 2]", # not an object +]) +def test_run_case_with_a_malformed_receipt_is_failed_not_fatal(tmp_path, monkeypatch, text): fixture, _receipt = _toy_fixture(tmp_path) def fake_run(argv, *, timeout, env, **_): out = Path(argv[argv.index("--out") + 1]) out.mkdir(parents=True, exist_ok=True) - (out / cf.RECEIPT_NAME).write_text('{"schema_version": "effective_instructions.v1"}', encoding="utf-8") + (out / cf.RECEIPT_NAME).write_text(text, encoding="utf-8") return process.Completed(argv=tuple(argv), returncode=0) monkeypatch.setattr(cf.process, "run", fake_run) From 49026970bc157c73fb9357545432b0e97efc13b1 Mon Sep 17 00:00:00 2001 From: "Petrov, Nikolay A" Date: Sun, 1 Nov 2026 21:03:40 -0800 Subject: [PATCH 07/29] fix(repo): a reach matrix names its expectations revision; merge refuses a mix reach_matrix.v1 gains expectations_digest, the sha256 of the expectations as canonical JSON. merge refuses matrices built from different revisions, and the merge command refuses matrices whose digest is not that of the expectations file it renders against. Failed and skipped runs, which name no version, no longer count as a repeated harness version on merge. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nikolay Petrov --- dokimi/docs/reach-matrix.md | 2 +- dokimi/dokimi/repos/conformance.py | 25 +++++++++-- dokimi/dokimi/schemas/reach_matrix.v1.json | 3 +- dokimi/scripts/reach_conformance.py | 3 ++ .../fixtures/artifacts/reach_matrix.v1.json | 1 + dokimi/tests/test_reach_conformance.py | 44 ++++++++++++++++--- 6 files changed, 67 insertions(+), 11 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index a9aa71d54..45a094a12 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -21,7 +21,7 @@ The matrix has two kinds of record: receipt came back with a scenario that gave no usable answer; `failed` means no receipt (a timeout, or a crash before the receipt was written); `skipped` means the harness could not run here, with the reason. - `cells`: one per source file per scenario, with the observed state, the documented state and an agreement: - `agrees`, `disagrees`, `inconclusive` (observed unknown or only fetched), `no_claim` (the docs say nothing), + `agrees`, `disagrees`, `inconclusive` (observed unknown, or only `fetched`: reach saw the file only in a `read=` scenario, after the agent read it with a tool, so its loading rule is not shown), `no_claim` (the docs say nothing), or `alias` (a link, whose state is its target's and is not compared). The schema is `reach_matrix.v1` (`dokimi/schemas/reach_matrix.v1.json`). diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 5adcb6505..a3b9a8a6f 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -11,6 +11,8 @@ from __future__ import annotations +import hashlib +import json import shutil import sys from collections import Counter @@ -82,6 +84,12 @@ def load_expectations(path: Path) -> dict[str, Any]: return data +def expectations_digest(expect: Mapping[str, Any]) -> str: + """The sha256 of the expectations as canonical JSON: matrices built from different documented states differ in it.""" + text = json.dumps(expect, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + def check_expectations(expect: Mapping[str, Any], fixtures: Path) -> list[str]: """What makes the expectations disagree with the fixture tree; empty when they agree.""" declared, found = set(expect["cases"]), set(cases(fixtures)) @@ -162,13 +170,14 @@ def summary(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any] return {**{a: counts.get(a, 0) for a in AGREEMENTS}, "runs": {s: status.get(s, 0) for s in RUN_STATUSES}} -def build_matrix(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]], +def build_matrix(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]], digest: str, generated_at: str | None = None) -> dict[str, Any]: """The ``reach_matrix.v1`` payload: runs and cells in a stable order, with the counts.""" return { "schema_version": "reach_matrix.v1", "generated_at": generated_at or datetime.now(timezone.utc).isoformat(), "expectations": SCHEMA_EXPECTATIONS, + "expectations_digest": digest, "runs": sorted((dict(r) for r in runs), key=lambda r: (r["case"], r["harness"], r["version"])), "cells": sorted((dict(c) for c in cells), key=lambda c: (c["case"], c["harness"], c["version"], c["path"], c["scenario"])), @@ -222,25 +231,33 @@ def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[st timeout=timeout, budget=budget) runs.append(record) cells += found - return build_matrix(runs, cells) + return build_matrix(runs, cells, expectations_digest(expect)) def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: - """One matrix from several: the runs and cells of each, and no run of the same harness version twice.""" + """One matrix from several: the runs and cells of each, and no run of the same known harness version twice. + + The matrices must come from one revision of the expectations, else their documented states differ.""" if not matrices: raise ConformanceError("nothing to merge") + digests = sorted({m["expectations_digest"] for m in matrices}) + if len(digests) > 1: + raise ConformanceError(f"the matrices come from {len(digests)} revisions of the expectations: " + + ", ".join(d[:12] for d in digests)) runs: list[dict[str, Any]] = [] cells: list[dict[str, Any]] = [] seen: set[tuple[str, str, str]] = set() for matrix in matrices: for run in matrix["runs"]: + if run["version"] == UNKNOWN: # a failed or skipped run names no version, so a rerun of one is not a repeat + continue key = (run["harness"], run["version"], run["case"]) if key in seen: raise ConformanceError(f"two matrices hold {run['harness']} {run['version']} on {run['case']}") seen.add(key) runs += matrix["runs"] cells += matrix["cells"] - return build_matrix(runs, cells, generated_at=max(m["generated_at"] for m in matrices)) + return build_matrix(runs, cells, digests[0], generated_at=max(m["generated_at"] for m in matrices)) def write_matrix(matrix: Mapping[str, Any], path: Path) -> None: diff --git a/dokimi/dokimi/schemas/reach_matrix.v1.json b/dokimi/dokimi/schemas/reach_matrix.v1.json index 1cfc8459b..14b5d4709 100644 --- a/dokimi/dokimi/schemas/reach_matrix.v1.json +++ b/dokimi/dokimi/schemas/reach_matrix.v1.json @@ -4,11 +4,12 @@ "title": "Reach conformance matrix v1", "description": "The matrix of the reach conformance suite (docs/reach-matrix.md): for each harness run over each fixture repository, the state each guidance file was observed in, per reach scenario, next to what the harness's vendor documentation says (reach_conformance.v1). One matrix is written per harness and several are merged into one. A `skipped` run has no cells; a cell whose documented state is `unknown` makes no claim.", "type": "object", - "required": ["schema_version", "generated_at", "expectations", "runs", "cells", "summary"], + "required": ["schema_version", "generated_at", "expectations", "expectations_digest", "runs", "cells", "summary"], "properties": { "schema_version": {"const": "reach_matrix.v1"}, "generated_at": {"type": "string"}, "expectations": {"const": "reach_conformance.v1", "description": "The schema of the expectations file the documented states come from."}, + "expectations_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The sha256 of the expectations file's content as canonical JSON (sorted keys, no spaces); matrices with different digests are not merged."}, "runs": { "type": "array", "items": { diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index 35995c080..7783b2b8f 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -69,6 +69,9 @@ def cmd_merge(args: argparse.Namespace) -> int: expect = _expectations(args.expectations) matrices = [load_artifact("reach_matrix", path) for path in args.inputs] merged = cf.merge(matrices) + if merged["expectations_digest"] != cf.expectations_digest(expect): + print(f"✗ the matrices were built from another revision of {args.expectations}", file=sys.stderr) + return 1 cf.write_matrix(merged, args.out) summaries = {case: block["summary"] for case, block in expect["cases"].items()} args.md.write_text(cf.render_markdown(merged, summaries), encoding="utf-8") diff --git a/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json index 0b728a016..fbcc5fcc2 100644 --- a/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json +++ b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json @@ -2,6 +2,7 @@ "schema_version": "reach_matrix.v1", "generated_at": "2026-10-31T04:37:00+00:00", "expectations": "reach_conformance.v1", + "expectations_digest": "0000000000000000000000000000000000000000000000000000000000000000", "runs": [ { "harness": "codex", diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index f4ab3582d..5d130a94e 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -60,8 +60,11 @@ def _cell(**fields) -> dict: return {**base, **fields} -def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00") -> dict: - return cf.build_matrix(runs or [_run()], cells or [_cell()], generated_at=generated_at) +DIGEST = "a" * 64 + + +def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00", digest=DIGEST) -> dict: + return cf.build_matrix(runs or [_run()], cells or [_cell()], digest, generated_at=generated_at) def test_unavailable_names_the_missing_piece(): @@ -134,7 +137,7 @@ def test_build_matrix_counts_and_validates_against_the_schema(): runs = [_run(), _run(harness="copilot", status="skipped", reason="no key", version="unknown")] cells = [_cell(), _cell(agreement="disagrees", observed="not_loaded", documented="loaded", path="CLAUDE.md"), _cell(agreement="alias", path="LINK.md", alias_of="AGENTS.md")] - matrix = cf.build_matrix(runs, cells, generated_at="2026-10-31T00:00:00+00:00") + matrix = cf.build_matrix(runs, cells, DIGEST, generated_at="2026-10-31T00:00:00+00:00") assert matrix["summary"] == {"agrees": 1, "disagrees": 1, "inconclusive": 0, "no_claim": 0, "alias": 1, "runs": {"ran": 1, "errors": 0, "failed": 0, "skipped": 1}} assert schema_errors("reach_matrix.v1", matrix) == [] @@ -248,6 +251,27 @@ def test_merge_refuses_a_harness_version_on_one_fixture_twice(): cf.merge([_matrix(), _matrix()]) +def test_merge_refuses_matrices_from_two_revisions_of_the_expectations(): + second = _matrix(runs=[_run(harness="claude-code")], cells=[_cell(harness="claude-code")], digest="b" * 64) + with pytest.raises(cf.ConformanceError, match="2 revisions of the expectations"): + cf.merge([_matrix(), second]) + + +def test_expectations_digest_follows_the_content_not_the_key_order(): + expect = cf.load_expectations(FIXTURES / "expectations.yaml") + reordered = dict(reversed(list(expect.items()))) + assert cf.expectations_digest(reordered) == cf.expectations_digest(expect) + assert cf.expectations_digest({**expect, "note": "edited"}) != cf.expectations_digest(expect) + matrix = cf.run_harness(CODEX, "", FIXTURES, expect, Path("unused"), env={}, which=_found) + assert matrix["expectations_digest"] == cf.expectations_digest(expect) + + +def test_merge_keeps_two_failed_runs_of_one_harness(): + failed = _run(status="failed", reason="timed out", version="unknown", scenarios=0) + merged = cf.merge([_matrix(runs=[failed], cells=[]), _matrix(runs=[failed], cells=[])]) + assert [r["status"] for r in merged["runs"]] == ["failed", "failed"] + + def test_merge_refuses_nothing(): with pytest.raises(cf.ConformanceError, match="nothing to merge"): cf.merge([]) @@ -313,11 +337,21 @@ def test_cli_run_rejects_an_unknown_harness(tmp_path): def test_cli_merge_writes_json_and_markdown(tmp_path): + digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) left, right = tmp_path / "left.json", tmp_path / "right.json" - cf.write_matrix(_matrix(), left) + cf.write_matrix(_matrix(digest=digest), left) cf.write_matrix(_matrix(runs=[_run(harness="claude-code", version="2.1.0")], - cells=[_cell(harness="claude-code", version="2.1.0")]), right) + cells=[_cell(harness="claude-code", version="2.1.0")], digest=digest), right) out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" assert cli.main(["merge", str(left), str(right), "--out", str(out), "--md", str(md)]) == 0 assert load_artifact("reach_matrix", out)["summary"]["agrees"] == 2 assert md.read_text(encoding="utf-8").startswith("# Reach matrix") + + +def test_cli_merge_refuses_matrices_from_another_revision_of_the_expectations(tmp_path, capsys): + left = tmp_path / "left.json" + cf.write_matrix(_matrix(), left) + out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" + assert cli.main(["merge", str(left), "--out", str(out), "--md", str(md)]) == 1 + assert "another revision" in capsys.readouterr().err + assert not out.exists() and not md.exists() From 51c8d6d99ceb9106c6dd5054b87dbd27de52c0dd Mon Sep 17 00:00:00 2001 From: "Petrov, Nikolay A" Date: Sun, 1 Nov 2026 21:21:53 -0800 Subject: [PATCH 08/29] fix(repo): skip only failed and skipped runs in the merge repeat check; document the digest Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nikolay Petrov --- dokimi/docs/reach-matrix.md | 4 +++- dokimi/dokimi/repos/conformance.py | 2 +- dokimi/tests/test_reach_conformance.py | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 45a094a12..e8ecc6488 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -87,7 +87,9 @@ Credentials are the names the harness profile forwards (`dokimi/repos/profiles/* not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. `merge` joins matrices from several harnesses or runs. It refuses a second run of the same harness version on the -same fixture, so a merge cannot double-count. +same fixture, so a merge cannot double-count (failed and skipped runs hold no cells and are kept as they are). Each +matrix carries `expectations_digest`, the sha256 of the expectations it was scored against; `merge` refuses matrices +with different digests, and the `merge` command refuses ones whose digest is not that of its `--expectations` file. The offline tests (`tests/test_reach_conformance.py`) cover the agreement rules, scenario ids, alias handling, skipped and failed runs, merge, rendering, the drift checks, and the CLI, with the harness runs faked. No test diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index a3b9a8a6f..3a9bb7567 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -249,7 +249,7 @@ def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: seen: set[tuple[str, str, str]] = set() for matrix in matrices: for run in matrix["runs"]: - if run["version"] == UNKNOWN: # a failed or skipped run names no version, so a rerun of one is not a repeat + if run["status"] in ("failed", "skipped"): # no receipt, so no cells to count twice and no version to key on continue key = (run["harness"], run["version"], run["case"]) if key in seen: diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 5d130a94e..2d8de5db5 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -272,6 +272,12 @@ def test_merge_keeps_two_failed_runs_of_one_harness(): assert [r["status"] for r in merged["runs"]] == ["failed", "failed"] +def test_merge_refuses_two_ran_runs_even_when_their_version_is_unknown(): + ran = _run(version="unknown") + with pytest.raises(cf.ConformanceError, match="codex unknown on toy"): + cf.merge([_matrix(runs=[ran]), _matrix(runs=[ran])]) + + def test_merge_refuses_nothing(): with pytest.raises(cf.ConformanceError, match="nothing to merge"): cf.merge([]) From 455d2e9da005531d473a98d4c0b1c58b075bf2ad Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 10:39:41 +0000 Subject: [PATCH 09/29] Create the markdown output directory before merge writes it Signed-off-by: Anton Volkov Signed-off-by: Nikolay Petrov --- dokimi/scripts/reach_conformance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index 7783b2b8f..3443cfd42 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -74,6 +74,7 @@ def cmd_merge(args: argparse.Namespace) -> int: return 1 cf.write_matrix(merged, args.out) summaries = {case: block["summary"] for case, block in expect["cases"].items()} + args.md.parent.mkdir(parents=True, exist_ok=True) args.md.write_text(cf.render_markdown(merged, summaries), encoding="utf-8") print(f"merged {len(matrices)} matrices: {len(merged['runs'])} runs, {len(merged['cells'])} cells") return 0 From 06b589330110bb504eb0acd15fc0a975761b50df Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 10:39:51 +0000 Subject: [PATCH 10/29] Document that run_harness env only decides availability Signed-off-by: Anton Volkov Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/conformance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 3a9bb7567..f4556d412 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -218,7 +218,9 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[str, Any], out_dir: Path, *, timeout: int = DEFAULT_TIMEOUT_S, budget: float = DEFAULT_BUDGET_USD, env: Mapping[str, str], which: Callable[[str], str | None] = shutil.which) -> dict[str, Any]: - """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture.""" + """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture. + + ``env`` only decides availability; each run inherits the process environment.""" reason = unavailable(harness, model, env, which) guidance = fixtures / "guidance.yaml" runs: list[dict[str, Any]] = [] From 1afd32a1d4a744c2cbb9da7c1a44788f559cbf4d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 10:39:59 +0000 Subject: [PATCH 11/29] Reject a scenario key with no directory in the expectations check Signed-off-by: Anton Volkov Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/conformance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index f4556d412..6090c5b36 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -110,7 +110,7 @@ def _case_problems(case: str, block: Mapping[str, Any], root: Path) -> list[str] out.append(f"{case}: {harness} names {path}, which the fixture does not hold") for scenario in states: kind, _, where = scenario.partition("=") - if kind not in ("cwd", "read") or not (root / where).is_dir(): + if kind not in ("cwd", "read") or not where or not (root / where).is_dir(): out.append(f"{case}: {harness} names scenario {scenario}, which the fixture does not have") return out From c3295e1652c283ef49279a498b9f08da93b7665f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 10:40:08 +0000 Subject: [PATCH 12/29] Reject a repeated harness name in the run command Signed-off-by: Anton Volkov Signed-off-by: Nikolay Petrov --- dokimi/scripts/reach_conformance.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index 3443cfd42..b394c9e1c 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -52,6 +52,10 @@ def cmd_run(args: argparse.Namespace) -> int: if problems: print("\n".join(problems), file=sys.stderr) return 1 + names = [harness.name for harness, _ in args.harness] + if len(set(names)) != len(names): + print("✗ each --harness may be given once: a second run would overwrite the first's matrix", file=sys.stderr) + return 1 failed = False args.out.mkdir(parents=True, exist_ok=True) for harness, model in args.harness: From 48979fc460bee59bfe33c3388a23fef73b3a7533 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 10:40:17 +0000 Subject: [PATCH 13/29] Let the test matrix helper take empty runs and cells Signed-off-by: Anton Volkov Signed-off-by: Nikolay Petrov --- dokimi/tests/test_reach_conformance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 2d8de5db5..675a4d663 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -64,7 +64,8 @@ def _cell(**fields) -> dict: def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00", digest=DIGEST) -> dict: - return cf.build_matrix(runs or [_run()], cells or [_cell()], digest, generated_at=generated_at) + return cf.build_matrix([_run()] if runs is None else runs, [_cell()] if cells is None else cells, digest, + generated_at=generated_at) def test_unavailable_names_the_missing_piece(): @@ -270,6 +271,7 @@ def test_merge_keeps_two_failed_runs_of_one_harness(): failed = _run(status="failed", reason="timed out", version="unknown", scenarios=0) merged = cf.merge([_matrix(runs=[failed], cells=[]), _matrix(runs=[failed], cells=[])]) assert [r["status"] for r in merged["runs"]] == ["failed", "failed"] + assert merged["cells"] == [] def test_merge_refuses_two_ran_runs_even_when_their_version_is_unknown(): From 5580b5df31224f2d541ca376fcf450e9f73f8195 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 10:40:25 +0000 Subject: [PATCH 14/29] Leave failed runs out of the markdown version columns Signed-off-by: Anton Volkov Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/conformance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 6090c5b36..37924f6ea 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -281,7 +281,7 @@ def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | No """The matrix as a markdown page: the runs, then one table per fixture with a column per harness version.""" summaries = summaries or {} counts = matrix["summary"] - columns = sorted({(r["harness"], r["version"]) for r in matrix["runs"] if r["status"] != "skipped"}) + columns = sorted({(r["harness"], r["version"]) for r in matrix["runs"] if r["status"] not in ("skipped", "failed")}) lines = [ "# Reach matrix", "", f"Generated {matrix['generated_at']}. Observed states are what each run's model reported seeing " From 21452d2ff39e63cc78994f60032ced61ee6d84d9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 08:15:43 -0800 Subject: [PATCH 15/29] fix(repo): answer the reach matrix review - CI: exempt the fixtures' CLAUDE.md and AGENTS.override.md from the license header check, add the header to docs/reach-matrix.md, and regenerate the CLI snapshot (reach_matrix sorts before redteam). - Nightly workflow: default bash shell; pip cache; Node.js set up for the CLI installs; a CLI is installed only with its model variable; Claude Code uses Bedrock (CLAUDE_CODE_USE_BEDROCK, AWS_REGION) only when the Anthropic key is absent; the merge fails when a harness wrote no matrix. - Markdown: a multi-line stderr reason stays on one table row. - CLI: an unusable expectations file, matrix or merge input prints one line and exits 1, no traceback. - Docs: --budget is enforced by Claude Code only; "marker self-report" label; versioning rule matches the persisted-format policy. - Tests for the duplicate harness, the markdown directory, the failed-run column, the malformed scenario key, the receipt cleanup, the run exit code and the merge repeat rules. Signed-off-by: Nikolay Petrov --- .github/.licenserc.yaml | 4 + .github/workflows/reach-matrix.yml | 43 ++++++++++- dokimi/docs/reach-matrix.md | 48 +++++++----- dokimi/docs/reference/cli-interface.json | 2 +- dokimi/dokimi/repos/conformance.py | 9 ++- dokimi/scripts/reach_conformance.py | 11 +-- dokimi/tests/test_reach_conformance.py | 93 +++++++++++++++++++++++- 7 files changed, 179 insertions(+), 31 deletions(-) diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index 5efd79c1d..dbaf2144e 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -51,6 +51,10 @@ header: - 'AGENTS.md' - 'dokimi/data/agent_profiles/**' - 'dokimi/data/skills/**' + # Reach conformance fixtures: the CLAUDE.md and AGENTS.override.md files a harness loads verbatim (AGENTS.md is + # ignored above, at any depth). + - 'dokimi/data/reach-conformance/**/CLAUDE.md' + - 'dokimi/data/reach-conformance/**/AGENTS.override.md' # Eval oracles: patches applied to the workspace and reference answers # written verbatim as the agent's answer (.agent-eval/run.py). - '.agent-eval/tasks/**/*.patch' diff --git a/.github/workflows/reach-matrix.yml b/.github/workflows/reach-matrix.yml index 2ae214eeb..38a7dd9e1 100644 --- a/.github/workflows/reach-matrix.yml +++ b/.github/workflows/reach-matrix.yml @@ -12,6 +12,10 @@ on: permissions: {} +defaults: + run: + shell: bash + concurrency: group: ${{ github.workflow }} cancel-in-progress: false @@ -29,18 +33,27 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' - name: Check the credential id: creds env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} run: | - if [ -n "$ANTHROPIC_API_KEY$AWS_BEARER_TOKEN_BEDROCK" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi + if [ -n "$ANTHROPIC_API_KEY" ]; then backend=anthropic + elif [ -n "$AWS_BEARER_TOKEN_BEDROCK" ]; then backend=bedrock + else backend=none; fi + echo "backend=$backend" >> "$GITHUB_OUTPUT" + if [ "$backend" = none ]; then echo "present=false" >> "$GITHUB_OUTPUT"; else echo "present=true" >> "$GITHUB_OUTPUT"; fi - name: Install the CLI - if: steps.creds.outputs.present == 'true' + if: steps.creds.outputs.present == 'true' && vars.REACH_MODEL_CLAUDE_CODE != '' run: npm install -g @anthropic-ai/claude-code@2.1.316 - name: Run reach over the fixtures working-directory: dokimi @@ -49,6 +62,8 @@ jobs: REACH_MODEL: ${{ vars.REACH_MODEL_CLAUDE_CODE }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + CLAUDE_CODE_USE_BEDROCK: ${{ steps.creds.outputs.backend == 'bedrock' && '1' || '' }} + AWS_REGION: ${{ vars.AWS_REGION }} run: python scripts/reach_conformance.py run --harness "claude-code=$REACH_MODEL" --out reach-out - name: Upload the matrix if: always() @@ -71,9 +86,14 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' - name: Check the credential id: creds env: @@ -81,7 +101,7 @@ jobs: run: | if [ -n "$CODEX_API_KEY" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi - name: Install the CLI - if: steps.creds.outputs.present == 'true' + if: steps.creds.outputs.present == 'true' && vars.REACH_MODEL_CODEX != '' run: npm install -g @openai/codex@0.166.0 - name: Run reach over the fixtures working-directory: dokimi @@ -111,9 +131,14 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' - name: Check the credential id: creds env: @@ -121,7 +146,7 @@ jobs: run: | if [ -n "$COPILOT_GITHUB_TOKEN" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi - name: Install the CLI - if: steps.creds.outputs.present == 'true' + if: steps.creds.outputs.present == 'true' && vars.REACH_MODEL_COPILOT != '' run: npm install -g @github/copilot@1.0.105 - name: Run reach over the fixtures working-directory: dokimi @@ -153,6 +178,8 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi run: pip install --require-hashes -r requirements.lock.txt @@ -166,6 +193,14 @@ jobs: env: PYTHONPATH: . run: | + missing=0 + for harness in claude-code codex copilot; do + if [ ! -f "reach-in/$harness.matrix.json" ]; then + echo "::error::no $harness matrix: its job failed before writing one" + missing=1 + fi + done + [ "$missing" -eq 0 ] || exit 1 python scripts/reach_conformance.py merge reach-in/*.matrix.json \ --out reach-out/reach-matrix.json --md reach-out/reach-matrix.md cat reach-out/reach-matrix.md >> "$GITHUB_STEP_SUMMARY" diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index e8ecc6488..ea3067764 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -1,3 +1,8 @@ + + # Reach conformance matrix The reach conformance suite asks, for each agent harness and version, which guidance files it loads in a set of @@ -79,12 +84,14 @@ python scripts/reach_conformance.py merge reach-out/*.matrix.json --out reach-ma `--harness NAME[=MODEL]` is repeatable: `claude-code`, `codex` and `copilot` are known. A harness without a model is skipped, as is one without its credential or its CLI (`unavailable` in `dokimi/repos/conformance.py` names which). `run` writes `.matrix.json` to `--out`, and the per-fixture receipts under -`-runs/`. Each scenario is capped by `--budget` (USD, default 0.10) and `--timeout` (seconds per -scenario, default 300). `run` exits 1 only when a run is `failed`. +`-runs/`. Each scenario is capped by `--timeout` (seconds per scenario, default 300). `--budget` (USD, +default 0.10) is passed to Claude Code as `--max-budget-usd` only: Codex and Copilot have no budget flag, so for +them the timeout is the only cap. `run` exits 1 only when a run is `failed`. Credentials are the names the harness profile forwards (`dokimi/repos/profiles/*.yaml`, `env_allow`): `ANTHROPIC_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK` for Claude Code, `CODEX_API_KEY` for Codex (`OPENAI_API_KEY` is -not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. +not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and +`AWS_REGION`: Claude Code does not select Bedrock from the token alone. `merge` joins matrices from several harnesses or runs. It refuses a second run of the same harness version on the same fixture, so a merge cannot double-count (failed and skipped runs hold no cells and are kept as they are). Each @@ -97,37 +104,41 @@ calls a model. ## Limits -- **Prompt capture, not behaviour.** `observed` is what the model reported seeing in its prompt, through marker +- **Marker self-report, not behaviour.** `observed` is what the model reported seeing in its prompt, through marker tokens (see §7). A model can under-report, and a `loaded` cell does not show the instruction was followed. - **Links.** A symbolic link is reported as an alias of its target. The receipt cannot show a harness ignoring the link's name, so an `alias` cell is not evidence that a harness reads or skips it. - **Unknown is not false.** `unknown` in the documented column means the docs are silent, not that the file is not loaded. - **One version per run.** The version is the one the harness reports; a matrix shows the versions that ran. -- **Cost.** Each scenario makes at least one model call, capped at `--budget`. The nightly run covers every - scenario of every fixture for each harness that has a model, a credential and a CLI. +- **Cost.** Each scenario makes at least one model call. Claude Code runs are capped at `--budget`; Codex and + Copilot runs are bounded only by `--timeout`. The nightly run covers every scenario of every fixture for each + harness that has a model, a credential and a CLI. ## Nightly workflow `.github/workflows/reach-matrix.yml` runs on a schedule (daily, 04:37 UTC) and on `workflow_dispatch`. It is not a pull-request check, and it has no `pull_request` trigger. -- One job per harness. Each checks its credential, installs its CLI only when the credential is present (pinned - versions: `@anthropic-ai/claude-code`, `@openai/codex`, `@github/copilot`), and runs `run` with the model from a - repository variable (`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, `REACH_MODEL_COPILOT`). With no variable, - the harness is recorded as skipped. -- Each job gets only its own secret in its environment. -- The `matrix` job merges the three artifacts, appends the page to the run summary, and uploads - `reach-matrix.json` and `reach-matrix.md` as the `reach-matrix` artifact (90 days). Each harness's matrix is a - 30-day artifact of its own. +- One job per harness. Each checks its credential, installs its CLI only when the credential and the model + variable are both present (pinned versions: `@anthropic-ai/claude-code`, `@openai/codex`, `@github/copilot`), and + runs `run` with the model from a repository variable (`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, + `REACH_MODEL_COPILOT`). With no variable, the harness is recorded as skipped. +- Each job gets only its own secret in its environment. The Claude Code job uses the Anthropic key when it is set; + otherwise the Bedrock token, with `CLAUDE_CODE_USE_BEDROCK=1` and `AWS_REGION` from the variable of that name. +- The `matrix` job checks that all three matrices arrived and fails, publishing nothing, when one is missing (its + harness job died before writing it). It merges the three artifacts, appends the page to the run summary, and + uploads `reach-matrix.json` and `reach-matrix.md` as the `reach-matrix` artifact (90 days). Each harness's matrix + is a 30-day artifact of its own. - `permissions: {}` at the top; each job has `contents: read`. To enable a harness, set its model variable and its secret. Nothing is committed: no key, token or model string is in the repository. Secrets: `ANTHROPIC_API_KEY` (or `AWS_BEARER_TOKEN_BEDROCK`), `CODEX_API_KEY`, `COPILOT_GITHUB_TOKEN`. Variables: -`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, `REACH_MODEL_COPILOT`. Runners: `intel-ubuntu-latest` on the Intel -organisation, `ubuntu-latest` elsewhere; the jobs need `npm` on the runner for the CLI installs. +`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, `REACH_MODEL_COPILOT`, and `AWS_REGION` for the Bedrock path. +Runners: `intel-ubuntu-latest` on the Intel organisation, `ubuntu-latest` elsewhere; each job sets up Node.js for +its CLI install. ## Adding a case or a harness @@ -138,5 +149,6 @@ A harness is a `Harness` in `HARNESSES` (`dokimi/repos/conformance.py`), with it names its profile forwards, a profile under `dokimi/repos/profiles/`, its npm pin in the workflow, and a job in the workflow. Its documented states go in `expectations.yaml` with the vendor page as `why`. -Both schemas are versioned (`reach_conformance.v1`, `reach_matrix.v1`). A change to a field bumps the version and -the artifact registry (`dokimi/artifacts.py`). +Both schemas are versioned (`reach_conformance.v1`, `reach_matrix.v1`). Compatible widening may edit the existing +version; breaking changes require a new version, updating `SCHEMA_VERSIONS` and retaining the old version in +`SUPERSEDED_VERSIONS` (`dokimi/artifacts.py`). diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a643be1b8..625b59709 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -13154,8 +13154,8 @@ "'plan'", "'plugin_delta'", "'questions'", - "'redteam'", "'reach_matrix'", + "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_readiness'", diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 37924f6ea..3d5ea4505 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -266,6 +266,11 @@ def write_matrix(matrix: Mapping[str, Any], path: Path) -> None: save_artifact("reach_matrix", dict(matrix), path) +def _table_text(text: str) -> str: + """One table cell: a stderr reason can run over several lines, and a pipe would end the cell early.""" + return " ".join(text.split()).replace("|", "\\|") + + def _cell_text(cell: Mapping[str, Any] | None) -> str: if cell is None: return "-" @@ -274,7 +279,7 @@ def _cell_text(cell: Mapping[str, Any] | None) -> str: text = f"link, {text}" elif cell["agreement"] == "disagrees": text += f" (docs: {cell['documented']})" - return text.replace("|", "\\|") + return _table_text(text) def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | None = None) -> str: @@ -294,7 +299,7 @@ def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | No "## Runs", "", "| fixture | harness | version | model | status | reason |", "|---|---|---|---|---|---|", ] lines += [f"| {r['case']} | {r['harness']} | {r['version']} | {r['model'] or '-'} | {r['status']} | " - f"{(r['reason'] or '-').replace('|', chr(92) + '|')} |" for r in matrix["runs"]] + f"{_table_text(r['reason'] or '-')} |" for r in matrix["runs"]] lines += ["", "Cell legend: `loaded` or `not_loaded` as observed; `fetched` is read with a tool first; " "`unknown` is no usable answer. `(docs: X)` marks a disagreement with the documentation; a " "cell with no `(docs: ...)` agrees, or the documentation makes no claim.", ""] diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index b394c9e1c..c14f7cd33 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -105,11 +105,12 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) - if args.command == "run": - return cmd_run(args) - if args.command == "merge": - return cmd_merge(args) - return cmd_check(args) + commands = {"run": cmd_run, "merge": cmd_merge, "check": cmd_check} + try: + return commands[args.command](args) + except (cf.ConformanceError, OSError, ValueError) as exc: # an input the suite cannot use: say so, no traceback + print(f"✗ {exc}", file=sys.stderr) + return 1 if __name__ == "__main__": diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 675a4d663..9a287d105 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -314,7 +314,7 @@ def test_check_expectations_reports_each_kind_of_drift(tmp_path): (fixtures / "extra").mkdir() expect = {"schema": "reach_conformance.v1", "cases": { "toy": {"summary": "s", "codex": {"why": "w", "files": { - "CLAUDE.md": {"cwd=.": "loaded"}, "AGENTS.md": {"cwd=nope": "loaded"}}}}, + "CLAUDE.md": {"cwd=.": "loaded"}, "AGENTS.md": {"cwd=nope": "loaded", "cwd": "loaded"}}}}, "gone": {"summary": "s"}, }} problems = "\n".join(cf.check_expectations(expect, fixtures)) @@ -322,6 +322,7 @@ def test_check_expectations_reports_each_kind_of_drift(tmp_path): assert "expectations name no fixture gone" in problems assert "codex names CLAUDE.md, which the fixture does not hold" in problems assert "codex names scenario cwd=nope" in problems + assert "codex names scenario cwd, which" in problems # no "=": the empty directory is the fixture root, not a scenario def test_load_expectations_rejects_another_schema(tmp_path): @@ -363,3 +364,93 @@ def test_cli_merge_refuses_matrices_from_another_revision_of_the_expectations(tm assert cli.main(["merge", str(left), "--out", str(out), "--md", str(md)]) == 1 assert "another revision" in capsys.readouterr().err assert not out.exists() and not md.exists() + + +def test_merge_accepts_one_harness_version_on_two_fixtures(): + merged = cf.merge([_matrix(), _matrix(runs=[_run(case="other")], cells=[_cell(case="other")])]) + assert sorted(r["case"] for r in merged["runs"]) == ["other", "toy"] + + +def test_merge_accepts_two_versions_of_one_harness_on_one_fixture(): + merged = cf.merge([_matrix(), _matrix(runs=[_run(version="0.167.0")], cells=[_cell(version="0.167.0")])]) + assert sorted(r["version"] for r in merged["runs"]) == ["0.166.0", "0.167.0"] + + +def test_merge_refuses_two_errors_runs_of_one_harness_version(): + errors = _run(status="errors", reason="exit 1") + with pytest.raises(cf.ConformanceError, match="codex 0.166.0 on toy"): + cf.merge([_matrix(runs=[errors]), _matrix(runs=[errors])]) + + +def test_merge_accepts_a_skipped_run_beside_a_ran_run_of_one_version(): + skipped = _run(status="skipped", reason="no model configured for this harness", scenarios=0) + merged = cf.merge([_matrix(runs=[skipped], cells=[]), _matrix(runs=[_run()], cells=[_cell()])]) + assert sorted(r["status"] for r in merged["runs"]) == ["ran", "skipped"] + + +def test_render_markdown_keeps_a_multiline_reason_on_one_row(): + matrix = _matrix(runs=[_run(status="errors", reason="exit 1: a\nb | c")], cells=[]) + lines = cf.render_markdown(matrix).splitlines() + assert "| toy | codex | 0.166.0 | m | errors | exit 1: a b \\| c |" in lines + + +def test_render_markdown_adds_no_column_for_a_failed_run(): + matrix = _matrix(runs=[_run(), _run(status="failed", version="unknown", reason="timed out", scenarios=0)]) + header = [line for line in cf.render_markdown(matrix).splitlines() if line.startswith("| file | scenario")] + assert header == ["| file | scenario | codex 0.166.0 |"] + + +def test_run_case_does_not_read_a_receipt_an_earlier_run_left(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + out = tmp_path / "out" + save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + _fake_reach(monkeypatch, receipt, write=False) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", out, _expect(), + timeout=30, budget=0.5) + assert record["status"] == "failed" and cells == [] + + +def test_cli_run_refuses_a_harness_given_twice_before_any_call(tmp_path, monkeypatch, capsys): + def no_call(*args, **kwargs): + raise AssertionError("run_harness must not be called") + + monkeypatch.setattr(cli.cf, "run_harness", no_call) + out = tmp_path / "out" + assert cli.main(["run", "--harness", "codex=a", "--harness", "codex=b", "--out", str(out)]) == 1 + assert "may be given once" in capsys.readouterr().err + assert not out.exists() + + +@pytest.mark.parametrize(("status", "exit_code"), [("failed", 1), ("errors", 0)]) +def test_cli_run_exits_1_only_when_a_run_failed(tmp_path, monkeypatch, status, exit_code): + run = _run(status=status, reason="exit 1", version="unknown" if status == "failed" else "0.166.0", + scenarios=0 if status == "failed" else 1) + monkeypatch.setattr(cli.cf, "run_harness", lambda *args, **kwargs: cf.build_matrix([run], [], DIGEST)) + assert cli.main(["run", "--harness", "codex=m", "--out", str(tmp_path)]) == exit_code + assert (tmp_path / "codex.matrix.json").is_file() + + +def test_cli_merge_creates_both_output_directories_when_missing(tmp_path): + digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) + left = tmp_path / "left.json" + cf.write_matrix(_matrix(digest=digest), left) + out, md = tmp_path / "a" / "b" / "reach-matrix.json", tmp_path / "c" / "reach-matrix.md" + assert cli.main(["merge", str(left), "--out", str(out), "--md", str(md)]) == 0 + assert out.is_file() and md.is_file() + + +def test_cli_reports_unusable_inputs_without_a_traceback(tmp_path, capsys): + digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) + left, bad = tmp_path / "left.json", tmp_path / "bad.json" + cf.write_matrix(_matrix(digest=digest), left) + bad.write_text("{}", encoding="utf-8") + out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" + cases = [ + (["--expectations", str(tmp_path / "missing.yaml"), "check"], "missing.yaml"), + (["merge", str(left), str(left), "--out", str(out), "--md", str(md)], "two matrices hold codex"), + (["merge", str(bad), "--out", str(out), "--md", str(md)], "reach_matrix"), + ] + for argv, text in cases: + assert cli.main(argv) == 1 + err = capsys.readouterr().err + assert text in err and "Traceback" not in err From d32307ed602d7ec9eff9cd8af8fa6cdb79cd12d0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 09:40:22 -0800 Subject: [PATCH 16/29] fix(repo): answer the reach matrix review - run_harness: each repo reach run gets exactly the environment it is given, not the inherited one - codex: the suite runs data/reach-conformance/codex.profile.yaml, whose provider reads CODEX_API_KEY (codex exec sends no bearer for the built-in profile) - run_case: a receipt that cannot be cleared fails the run instead of being read as this run's - merge: generated_at is compared as an instant; a timestamp that is not timezone-aware is refused - markdown: every dynamic table field goes through _table_text, not only the reason - docs: PYTHONPATH=. for the scripts; --budget is per Claude Code scenario Signed-off-by: Nikolay Petrov --- .../data/reach-conformance/codex.profile.yaml | 77 +++++++++++++++++ dokimi/docs/reach-matrix.md | 25 ++++-- dokimi/dokimi/repos/conformance.py | 58 ++++++++++--- dokimi/scripts/reach_conformance.py | 3 +- dokimi/tests/test_reach_conformance.py | 85 +++++++++++++++++-- 5 files changed, 219 insertions(+), 29 deletions(-) create mode 100644 dokimi/data/reach-conformance/codex.profile.yaml diff --git a/dokimi/data/reach-conformance/codex.profile.yaml b/dokimi/data/reach-conformance/codex.profile.yaml new file mode 100644 index 000000000..75e428330 --- /dev/null +++ b/dokimi/data/reach-conformance/codex.profile.yaml @@ -0,0 +1,77 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# The codex profile the reach conformance suite runs (dokimi/repos/conformance.py). It is profiles/codex.yaml plus +# one provider in config_files: `codex exec` sends no bearer for the built-in profile (401 Missing bearer), so the +# suite selects a provider that reads its key from CODEX_API_KEY, which the profile forwards. Keep the two files in +# step; tests/test_reach_conformance.py reads this one. The provider is the vendor endpoint, so OPENAI_BASE_URL does +# not change it, and the egress list below is the same host. +schema: harness_profile.v1 +name: codex +bin: codex # npm's codex is a node launcher; mount the native one (.../vendor//bin/codex) with --bin +version_cmd: ["{bin}", "--version"] +# Codex 0.160 runs every tool call through codex-code-mode-host, which it starts from +# beside its own binary; without it in the container no command can run. +bin_companions: [codex-code-mode-host] +# `codex exec` is non-interactive; the prompt follows `--`, so a leading "-" is not a flag. No turn or spend cap: +# {max_turns}/{budget} are not passed and the timeout bounds the run. CODEX_HOME is fresh per run (no --ignore-user-config). +invoke: + - "{bin}" + - "exec" + - "--model" + - "{model}" + - "--ephemeral" + - "--skip-git-repo-check" + - "--json" + - "--color" + - "never" + - "{tools}" + - "--" + - "{prompt}" +# Codex has no tool-off switch: `none` and `read` both get its bwrap sandbox, which needs user namespaces. +# Where the host denies them every command fails; `all` relies on the framework's container for isolation. +tools: + none: ["--sandbox", "read-only"] + read: ["--sandbox", "read-only"] + all: ["--dangerously-bypass-approvals-and-sandbox"] +config_dir_env: CODEX_HOME +env_allow: + - PATH + - HOME + - LANG + - SYSTEMROOT + - CODEX_API_KEY +# OPENAI_API_KEY is not forwarded: `all` mode's shell reads every forwarded variable, and `codex exec` reads neither key. + - OPENAI_BASE_URL + - http_proxy + - https_proxy + - no_proxy + - HTTP_PROXY + - HTTPS_PROXY + - NO_PROXY +# The provider `model_provider` selects; its env_key names the variable that holds the bearer. +config_files: + config.toml: | + model_provider = "conformance" + + [model_providers.conformance] + name = "OpenAI" + base_url = "https://api.openai.com/v1" + wire_api = "responses" + env_key = "CODEX_API_KEY" +egress: + - api.openai.com +endpoint_env: OPENAI_BASE_URL +trace: codex-json +guidance: + names: [AGENTS.md, AGENTS.override.md] + resolution: codex-home + git-root-to-cwd + lazy-by-shell-read + imports: null + notes: >- + Documented for codex-cli 0.160: one AGENTS.override.md or AGENTS.md per + directory from the git root down to the working directory, joined root + first; an AGENTS.override.md replaces the AGENTS.md beside it. CLAUDE.md, + .github/copilot-instructions.md and .github/instructions are not read. + Measured on 0.160 with gpt-6-luna (oneDAL review task): root AGENTS.md + applied, subtree AGENTS.md applied only after the agent printed them with a + shell command; AGENTS.override.md hid the root AGENTS.md. diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index ea3067764..7244d60d9 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -73,24 +73,30 @@ on any drift, and `run` refuses to start while there is drift. ## Running it -From the dokimi project directory, with the dependencies installed (`make dev`), and a harness CLI on `PATH`: +From the dokimi project directory, with the dependencies installed (`make dev`), and a harness CLI on `PATH`. Set +`PYTHONPATH=.`: `make dev` installs the dependencies, not the package, and a script's own directory is what Python +puts on the import path. ```sh -python scripts/reach_conformance.py check -python scripts/reach_conformance.py run --harness codex= --out reach-out -python scripts/reach_conformance.py merge reach-out/*.matrix.json --out reach-matrix.json --md reach-matrix.md +PYTHONPATH=. python scripts/reach_conformance.py check +PYTHONPATH=. python scripts/reach_conformance.py run --harness codex= --out reach-out +PYTHONPATH=. python scripts/reach_conformance.py merge reach-out/*.matrix.json --out reach-matrix.json --md reach-matrix.md ``` `--harness NAME[=MODEL]` is repeatable: `claude-code`, `codex` and `copilot` are known. A harness without a model is skipped, as is one without its credential or its CLI (`unavailable` in `dokimi/repos/conformance.py` names which). `run` writes `.matrix.json` to `--out`, and the per-fixture receipts under `-runs/`. Each scenario is capped by `--timeout` (seconds per scenario, default 300). `--budget` (USD, -default 0.10) is passed to Claude Code as `--max-budget-usd` only: Codex and Copilot have no budget flag, so for -them the timeout is the only cap. `run` exits 1 only when a run is `failed`. +per scenario, default 0.10) is passed to Claude Code as `--max-budget-usd` only: Codex and Copilot have no budget +flag, so for them the timeout is the only cap. `run` exits 1 only when a run is `failed`. `run` gives each +`repo reach` process exactly the environment it was started with, and each harness gets only what its profile +forwards from that. Credentials are the names the harness profile forwards (`dokimi/repos/profiles/*.yaml`, `env_allow`): `ANTHROPIC_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK` for Claude Code, `CODEX_API_KEY` for Codex (`OPENAI_API_KEY` is -not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and +not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. Codex runs with `data/reach-conformance/codex.profile.yaml`, +which defines a provider that reads `CODEX_API_KEY` (`codex exec` sends no bearer without one), so the key is what +authenticates the run. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and `AWS_REGION`: Claude Code does not select Bedrock from the token alone. `merge` joins matrices from several harnesses or runs. It refuses a second run of the same harness version on the @@ -111,8 +117,9 @@ calls a model. - **Unknown is not false.** `unknown` in the documented column means the docs are silent, not that the file is not loaded. - **One version per run.** The version is the one the harness reports; a matrix shows the versions that ran. -- **Cost.** Each scenario makes at least one model call. Claude Code runs are capped at `--budget`; Codex and - Copilot runs are bounded only by `--timeout`. The nightly run covers every scenario of every fixture for each +- **Cost.** Each scenario makes at least one model call. Each Claude Code scenario is capped at `--budget`, not + the whole fixture or suite (a fixture with three scenarios can spend three times the cap); Codex and Copilot + scenarios are bounded only by `--timeout`. The nightly run covers every scenario of every fixture for each harness that has a model, a credential and a CLI. ## Nightly workflow diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 3d5ea4505..b2ca859c6 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -41,17 +41,26 @@ class ConformanceError(RuntimeError): @dataclass(frozen=True) class Harness: - """A harness the suite knows: its CLI, and the variables one of which must be set for a run.""" + """A harness the suite knows: its CLI, the variables one of which must be set for a run, and the profile + ``repo reach`` runs it with (a built-in name, or a profile YAML path; the harness name when ``profile`` is empty).""" name: str bin: str credentials: tuple[str, ...] + profile: str = "" + @property + def profile_ref(self) -> str: + return self.profile or self.name + + +# The codex profile with a provider that reads CODEX_API_KEY: the built-in profile sends no bearer (see the file). +CODEX_PROFILE = Path(__file__).resolve().parents[2] / "data" / "reach-conformance" / "codex.profile.yaml" # Credentials are the names the profile forwards (repos/profiles/*.yaml env_allow), so a key set here is one the run gets. HARNESSES: dict[str, Harness] = {h.name: h for h in ( Harness("claude-code", "claude", ("ANTHROPIC_API_KEY", "AWS_BEARER_TOKEN_BEDROCK")), - Harness("codex", "codex", ("CODEX_API_KEY",)), + Harness("codex", "codex", ("CODEX_API_KEY",), profile=str(CODEX_PROFILE)), Harness("copilot", "copilot", ("COPILOT_GITHUB_TOKEN",)), )} @@ -193,14 +202,21 @@ def _reason(proc: process.Completed) -> str: def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: Path, - expect: Mapping[str, Any], *, timeout: int, budget: float) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """Run ``repo reach`` over one fixture; its run record and cells. The receipt is read from ``out``.""" + expect: Mapping[str, Any], *, timeout: int, budget: float, + env: Mapping[str, str]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Run ``repo reach`` over one fixture with exactly ``env`` as its environment; its run record and cells. + The receipt is read from ``out``.""" case = fixture.name - shutil.rmtree(out, ignore_errors=True) # a receipt left by an earlier run is not this run's + try: + shutil.rmtree(out) # a receipt left by an earlier run is not this run's + except FileNotFoundError: + pass + except OSError as exc: # a receipt that stays would be read as this run's: fail the run instead + return run_record(harness.name, case, "failed", f"cannot clear {out}: {exc}", UNKNOWN, model, 0), [] argv = [sys.executable, "-m", "dokimi", "repo", "reach", str(fixture), "--manifest-file", str(guidance), - "--harness", harness.name, "--model", model, "--out", str(out), "--format", "receipt", + "--harness", harness.profile_ref, "--model", model, "--out", str(out), "--format", "receipt", "--timeout", str(timeout), "--budget", f"{budget:g}"] - proc = process.run(argv, timeout=timeout * 16 + 300, env=process.Env.inherit()) + proc = process.run(argv, timeout=timeout * 16 + 300, env=process.Env.only((), env)) receipt_path = out / RECEIPT_NAME if proc.timed_out or not receipt_path.is_file(): return run_record(harness.name, case, "failed", _reason(proc), UNKNOWN, model, 0), [] @@ -220,7 +236,7 @@ def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[st env: Mapping[str, str], which: Callable[[str], str | None] = shutil.which) -> dict[str, Any]: """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture. - ``env`` only decides availability; each run inherits the process environment.""" + ``env`` decides availability and is the whole environment each run gets (nothing else is inherited).""" reason = unavailable(harness, model, env, which) guidance = fixtures / "guidance.yaml" runs: list[dict[str, Any]] = [] @@ -230,7 +246,7 @@ def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[st runs.append(run_record(harness.name, case, "skipped", reason, UNKNOWN, model, 0)) continue record, found = run_case(harness, model, fixtures / case, guidance, out_dir / case, expect, - timeout=timeout, budget=budget) + timeout=timeout, budget=budget, env=env) runs.append(record) cells += found return build_matrix(runs, cells, expectations_digest(expect)) @@ -259,7 +275,19 @@ def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: seen.add(key) runs += matrix["runs"] cells += matrix["cells"] - return build_matrix(runs, cells, digests[0], generated_at=max(m["generated_at"] for m in matrices)) + latest = max((m["generated_at"] for m in matrices), key=_instant) + return build_matrix(runs, cells, digests[0], generated_at=latest) + + +def _instant(text: str) -> datetime: + """The moment a ``generated_at`` names: offsets differ between matrices, so text does not order them.""" + try: + when = datetime.fromisoformat(text) + except ValueError as exc: + raise ConformanceError(f"generated_at {text!r} is not an ISO 8601 timestamp") from exc + if when.tzinfo is None: + raise ConformanceError(f"generated_at {text!r} has no time zone") + return when def write_matrix(matrix: Mapping[str, Any], path: Path) -> None: @@ -287,6 +315,7 @@ def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | No summaries = summaries or {} counts = matrix["summary"] columns = sorted({(r["harness"], r["version"]) for r in matrix["runs"] if r["status"] not in ("skipped", "failed")}) + # every dynamic table field goes through _table_text: a pipe or newline in any of them breaks the row lines = [ "# Reach matrix", "", f"Generated {matrix['generated_at']}. Observed states are what each run's model reported seeing " @@ -298,8 +327,9 @@ def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | No f"{counts['runs']['failed']} failed, {counts['runs']['skipped']} skipped.", "", "## Runs", "", "| fixture | harness | version | model | status | reason |", "|---|---|---|---|---|---|", ] - lines += [f"| {r['case']} | {r['harness']} | {r['version']} | {r['model'] or '-'} | {r['status']} | " - f"{_table_text(r['reason'] or '-')} |" for r in matrix["runs"]] + lines += [f"| {_table_text(r['case'])} | {_table_text(r['harness'])} | {_table_text(r['version'])} | " + f"{_table_text(r['model'] or '-')} | {_table_text(r['status'])} | {_table_text(r['reason'] or '-')} |" + for r in matrix["runs"]] lines += ["", "Cell legend: `loaded` or `not_loaded` as observed; `fetched` is read with a tool first; " "`unknown` is no usable answer. `(docs: X)` marks a disagreement with the documentation; a " "cell with no `(docs: ...)` agrees, or the documentation makes no claim.", ""] @@ -312,10 +342,10 @@ def _case_section(matrix: Mapping[str, Any], case: str, columns: list[tuple[str, cells = [c for c in matrix["cells"] if c["case"] == case] index = {(c["harness"], c["version"], c["path"], c["scenario"]): c for c in cells} rows = sorted({(c["path"], c["scenario"]) for c in cells}) - header = ["file", "scenario"] + [f"{h} {v}" for h, v in columns] + header = ["file", "scenario"] + [_table_text(f"{h} {v}") for h, v in columns] out = [f"## {case}", ""] + ([summary_text, ""] if summary_text else []) out += ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)] for path, scenario in rows: texts = [_cell_text(index.get((h, v, path, scenario))) for h, v in columns] - out.append("| " + " | ".join([path, scenario, *texts]) + " |") + out.append("| " + " | ".join([_table_text(path), _table_text(scenario), *texts]) + " |") return out + [""] diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index c14f7cd33..6c3b50dd5 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -8,7 +8,8 @@ python scripts/reach_conformance.py merge DIR/*.matrix.json --out DIR/reach-matrix.json --md DIR/reach-matrix.md ``run`` makes paid model calls for every harness that has a model, a credential and a CLI here; the rest are -written as skipped runs. Run it from the dokimi project directory (``python -m dokimi`` must resolve). +written as skipped runs. Run it from the dokimi project directory with ``PYTHONPATH=.``: a script's own directory, +not the project, is on the import path, and ``make dev`` does not install the package. """ from __future__ import annotations diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 9a287d105..e6b30c827 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -8,6 +8,7 @@ import importlib.util import os +import tomllib from pathlib import Path import pytest @@ -15,6 +16,7 @@ from dokimi import process from dokimi.artifacts import load_artifact, save_artifact, schema_errors from dokimi.repos import conformance as cf +from dokimi.repos.harness_profile import load_profile from dokimi.repos.receipt import build_receipt from tests.test_repo_receipt import BUDGET, ROOT_ROW, _record, _reach, _tree @@ -173,7 +175,7 @@ def test_run_case_reads_the_receipt_the_run_wrote(tmp_path, monkeypatch): calls = _fake_reach(monkeypatch, receipt) expect = _expect(**{"claude-code": {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}}}}) record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "guidance.yaml", tmp_path / "out", expect, - timeout=30, budget=0.5) + timeout=30, budget=0.5, env={}) assert record == _run(harness="claude-code", version="2.1.0", model="m", status="ran", scenarios=len(receipt["receipts"])) assert any(c["path"] == "AGENTS.md" and c["agreement"] == "agrees" for c in cells) @@ -185,7 +187,7 @@ def test_run_case_marks_a_run_with_a_failing_scenario_as_errors(tmp_path, monkey fixture, receipt = _toy_fixture(tmp_path) _fake_reach(monkeypatch, receipt, returncode=1) record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), - timeout=30, budget=0.5) + timeout=30, budget=0.5, env={}) assert (record["status"], record["reason"]) == ("errors", "exit 1: boom") assert cells @@ -198,7 +200,7 @@ def test_run_case_without_a_receipt_is_failed(tmp_path, monkeypatch, kwargs, rea fixture, receipt = _toy_fixture(tmp_path) _fake_reach(monkeypatch, receipt, **kwargs) record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), - timeout=30, budget=0.5) + timeout=30, budget=0.5, env={}) assert record["status"] == "failed" and reason in record["reason"] assert cells == [] @@ -219,7 +221,7 @@ def fake_run(argv, *, timeout, env, **_): monkeypatch.setattr(cf.process, "run", fake_run) record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), - timeout=30, budget=0.5) + timeout=30, budget=0.5, env={}) assert record["status"] == "failed" and record["reason"].startswith("invalid receipt") assert cells == [] @@ -406,7 +408,7 @@ def test_run_case_does_not_read_a_receipt_an_earlier_run_left(tmp_path, monkeypa save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) _fake_reach(monkeypatch, receipt, write=False) record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", out, _expect(), - timeout=30, budget=0.5) + timeout=30, budget=0.5, env={}) assert record["status"] == "failed" and cells == [] @@ -454,3 +456,76 @@ def test_cli_reports_unusable_inputs_without_a_traceback(tmp_path, capsys): assert cli.main(argv) == 1 err = capsys.readouterr().err assert text in err and "Traceback" not in err + + +def test_run_case_hands_codex_the_suite_profile_with_its_provider(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + calls = _fake_reach(monkeypatch, receipt) + cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), timeout=30, budget=0.5, env={}) + argv = calls[0] + assert argv[argv.index("--harness") + 1] == str(cf.CODEX_PROFILE) + + +def test_the_codex_suite_profile_authenticates_with_the_key_it_forwards(tmp_path): + suite = load_profile(CODEX.profile_ref) + builtin = load_profile("codex") + assert "CODEX_API_KEY" in suite.env_allow + assert (suite.invoke, suite.env_allow, suite.egress) == (builtin.invoke, builtin.env_allow, builtin.egress) + suite.write_config(tmp_path, {"config_dir": tmp_path.as_posix(), "support_dir": tmp_path.as_posix()}) + config = tomllib.loads((tmp_path / "config.toml").read_text(encoding="utf-8")) + provider = config["model_providers"][config["model_provider"]] # the provider the run selects + assert provider["env_key"] == "CODEX_API_KEY" + assert provider["base_url"] == "https://api.openai.com/v1" + + +def test_run_harness_gives_each_run_exactly_the_environment_it_is_given(tmp_path, monkeypatch): + fixture, _receipt = _toy_fixture(tmp_path) + seen: list[dict[str, str]] = [] + + def fake_run(argv, *, timeout, env, **_): + seen.append(env.resolve()) + return process.Completed(argv=tuple(argv), returncode=1) + + monkeypatch.setattr(cf.process, "run", fake_run) + given = {"CODEX_API_KEY": "k", "PATH": "/usr/bin"} + matrix = cf.run_harness(CODEX, "gpt", tmp_path, _expect(), tmp_path / "runs", env=given, which=_found) + assert seen == [given] + assert [r["status"] for r in matrix["runs"]] == ["failed"] + + +def test_run_case_fails_when_an_earlier_receipt_cannot_be_cleared(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + out = tmp_path / "out" + save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + calls = _fake_reach(monkeypatch, receipt) + + def refuse(path, *args, **kwargs): + raise PermissionError(13, "Permission denied", str(path)) + + monkeypatch.setattr(cf.shutil, "rmtree", refuse) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", out, _expect(), + timeout=30, budget=0.5, env={}) + assert record["status"] == "failed" and "cannot clear" in record["reason"] + assert cells == [] and calls == [] # the old receipt is never read, and no probe runs + + +def test_merge_orders_matrices_by_the_instant_they_name_not_their_text(): + # 10:00 at +02:00 is 08:00 UTC, an hour before 09:00 UTC, though its text sorts higher + earlier = _matrix(generated_at="2026-11-02T10:00:00+02:00") + later = _matrix(runs=[_run(harness="claude-code", version="2.1.0")], + cells=[_cell(harness="claude-code", version="2.1.0")], generated_at="2026-11-02T09:00:00+00:00") + assert cf.merge([earlier, later])["generated_at"] == "2026-11-02T09:00:00+00:00" + + +@pytest.mark.parametrize("stamp", ["2026-11-02 10:00", "not a time"]) +def test_merge_refuses_a_timestamp_it_cannot_order(stamp): + with pytest.raises(cf.ConformanceError, match="generated_at"): + cf.merge([_matrix(generated_at=stamp)]) + + +def test_render_markdown_escapes_every_table_field_not_only_the_reason(): + matrix = _matrix(runs=[_run(model="provider|model")], cells=[_cell(path="A|B.md", scenario="cwd=x|y")]) + lines = cf.render_markdown(matrix).splitlines() + assert "| toy | codex | 0.166.0 | provider\\|model | ran | - |" in lines + assert "| file | scenario | codex 0.166.0 |" in lines + assert "| A\\|B.md | cwd=x\\|y | loaded |" in lines From 0784ce27c0c313016663e5db45b696dc5b6f4080 Mon Sep 17 00:00:00 2001 From: "Petrov, Nikolay A" Date: Mon, 2 Nov 2026 10:16:45 -0800 Subject: [PATCH 17/29] fix(repo): shorten the codex profile comment; list the reach matrix in runs.md Co-Authored-By: Claude Opus 5.5 Signed-off-by: Petrov, Nikolay A --- dokimi/data/reach-conformance/codex.profile.yaml | 7 ++----- dokimi/docs/runs.md | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/dokimi/data/reach-conformance/codex.profile.yaml b/dokimi/data/reach-conformance/codex.profile.yaml index 75e428330..c37f6af77 100644 --- a/dokimi/data/reach-conformance/codex.profile.yaml +++ b/dokimi/data/reach-conformance/codex.profile.yaml @@ -1,11 +1,8 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# The codex profile the reach conformance suite runs (dokimi/repos/conformance.py). It is profiles/codex.yaml plus -# one provider in config_files: `codex exec` sends no bearer for the built-in profile (401 Missing bearer), so the -# suite selects a provider that reads its key from CODEX_API_KEY, which the profile forwards. Keep the two files in -# step; tests/test_reach_conformance.py reads this one. The provider is the vendor endpoint, so OPENAI_BASE_URL does -# not change it, and the egress list below is the same host. +# profiles/codex.yaml plus a provider that reads CODEX_API_KEY: `codex exec` sends no bearer for the built-in one +# (401). Keep both files in step; the provider is the vendor endpoint, so OPENAI_BASE_URL does not change it. schema: harness_profile.v1 name: codex bin: codex # npm's codex is a node launcher; mount the native one (.../vendor//bin/codex) with --bin diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index 140f941dd..5244bdaf4 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -44,8 +44,8 @@ skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_vers fields depends on the kind: `product` of a questions set, `library_name` of an arms run, `artifact.ref` of a novelty run, `repo.id` of a reach run, `judge_model` and `evaluated_at` of an eval. An artifact that records no - subject shows `-` (an eval, a task run, a questions set from `questions - generate`, which records the model but not the product); nothing is guessed + subject shows `-` (an eval, a task run, a reach matrix, a questions set from + `questions generate`, which records the model but not the product); nothing is guessed from the path. Rows are newest first. When an artifact records no time, its file's mtime is used for the order and shown with `*`. - **Nothing is dropped silently.** The summary counts the files skipped, by From abfd72476f71c7be6acf135715bd0aa8c88b602e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 10:06:40 -0800 Subject: [PATCH 18/29] docs(repo): shorten the codex suite profile's comment block to two lines Signed-off-by: Nikolay Petrov --- dokimi/data/reach-conformance/codex.profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/data/reach-conformance/codex.profile.yaml b/dokimi/data/reach-conformance/codex.profile.yaml index c37f6af77..92ea4e13c 100644 --- a/dokimi/data/reach-conformance/codex.profile.yaml +++ b/dokimi/data/reach-conformance/codex.profile.yaml @@ -1,8 +1,8 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# profiles/codex.yaml plus a provider that reads CODEX_API_KEY: `codex exec` sends no bearer for the built-in one -# (401). Keep both files in step; the provider is the vendor endpoint, so OPENAI_BASE_URL does not change it. +# The codex profile of the reach suite: profiles/codex.yaml plus a provider that reads CODEX_API_KEY, because the +# built-in one sends no bearer (401 Missing bearer). Keep the two in step; the rest is in intel/dokimi#364. schema: harness_profile.v1 name: codex bin: codex # npm's codex is a node launcher; mount the native one (.../vendor//bin/codex) with --bin From a6e03777fa096efa6794567fc9c29d237c809849 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 10:09:01 -0800 Subject: [PATCH 19/29] fix(repo): ship the codex suite profile in the package; find its CLI on the run's PATH The reach suite's codex profile lived under data/, outside the package, so an installed wheel could not find it. It moves to repos/conformance_profiles/ and pyproject's package-data ships it; a test checks the wheel list and loads it through importlib.resources. Availability looked for the codex CLI on this process's PATH, but the run gets only the env it is given. It now searches the PATH of that env, so a CLI on the parent's PATH alone is not available and one only on the run's PATH is. Signed-off-by: Nikolay Petrov --- dokimi/docs/reach-matrix.md | 2 +- dokimi/dokimi/repos/conformance.py | 17 +++++++---- .../repos/conformance_profiles/codex.yaml} | 0 dokimi/pyproject.toml | 1 + dokimi/tests/test_packaging.py | 2 ++ dokimi/tests/test_reach_conformance.py | 28 +++++++++++++++++-- 6 files changed, 42 insertions(+), 8 deletions(-) rename dokimi/{data/reach-conformance/codex.profile.yaml => dokimi/repos/conformance_profiles/codex.yaml} (100%) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 7244d60d9..634b60d71 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -94,7 +94,7 @@ forwards from that. Credentials are the names the harness profile forwards (`dokimi/repos/profiles/*.yaml`, `env_allow`): `ANTHROPIC_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK` for Claude Code, `CODEX_API_KEY` for Codex (`OPENAI_API_KEY` is -not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. Codex runs with `data/reach-conformance/codex.profile.yaml`, +not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. Codex runs with `dokimi/repos/conformance_profiles/codex.yaml`, which defines a provider that reads `CODEX_API_KEY` (`codex exec` sends no bearer without one), so the key is what authenticates the run. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and `AWS_REGION`: Claude Code does not select Bedrock from the token alone. diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index b2ca859c6..c3a808084 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -13,6 +13,7 @@ import hashlib import json +import os import shutil import sys from collections import Counter @@ -55,7 +56,8 @@ def profile_ref(self) -> str: # The codex profile with a provider that reads CODEX_API_KEY: the built-in profile sends no bearer (see the file). -CODEX_PROFILE = Path(__file__).resolve().parents[2] / "data" / "reach-conformance" / "codex.profile.yaml" +# Package data, so an installed wheel finds it next to this module. +CODEX_PROFILE = Path(__file__).resolve().parent / "conformance_profiles" / "codex.yaml" # Credentials are the names the profile forwards (repos/profiles/*.yaml env_allow), so a key set here is one the run gets. HARNESSES: dict[str, Harness] = {h.name: h for h in ( @@ -65,14 +67,19 @@ def profile_ref(self) -> str: )} +def _which_on(cmd: str, search: str) -> str | None: + """``cmd`` on ``search``, a PATH value: the search a run's child process makes, not this process's.""" + return shutil.which(cmd, path=search) + + def unavailable(harness: Harness, model: str, env: Mapping[str, str], - which: Callable[[str], str | None] = shutil.which) -> str | None: - """Why ``harness`` cannot run here, or None when it can.""" + which: Callable[[str, str], str | None] = _which_on) -> str | None: + """Why ``harness`` cannot run here, or None when it can. The CLI is looked up on the PATH of ``env``.""" if not model: return "no model configured for this harness" if not any(env.get(name) for name in harness.credentials): return f"none of {', '.join(harness.credentials)} is set" - if which(harness.bin) is None: + if which(harness.bin, env.get("PATH", os.defpath)) is None: return f"{harness.bin} is not on PATH" return None @@ -233,7 +240,7 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[str, Any], out_dir: Path, *, timeout: int = DEFAULT_TIMEOUT_S, budget: float = DEFAULT_BUDGET_USD, - env: Mapping[str, str], which: Callable[[str], str | None] = shutil.which) -> dict[str, Any]: + env: Mapping[str, str], which: Callable[[str, str], str | None] = _which_on) -> dict[str, Any]: """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture. ``env`` decides availability and is the whole environment each run gets (nothing else is inherited).""" diff --git a/dokimi/data/reach-conformance/codex.profile.yaml b/dokimi/dokimi/repos/conformance_profiles/codex.yaml similarity index 100% rename from dokimi/data/reach-conformance/codex.profile.yaml rename to dokimi/dokimi/repos/conformance_profiles/codex.yaml diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index c645cedb0..c97a10b0a 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -102,6 +102,7 @@ dokimi = [ "runner/spec.schema.json", "schemas/*.json", "repos/profiles/*.yaml", + "repos/conformance_profiles/*.yaml", "repos/support/*.py", "harnesses/swebench_specs.json", ] diff --git a/dokimi/tests/test_packaging.py b/dokimi/tests/test_packaging.py index 1eac17184..cb5df4436 100644 --- a/dokimi/tests/test_packaging.py +++ b/dokimi/tests/test_packaging.py @@ -18,6 +18,7 @@ import dokimi from dokimi import _version, config, intents, registry from dokimi.cli import build_parser +from dokimi.repos import conformance from dokimi.runner import spec PROJECT = Path(__file__).resolve().parents[1] @@ -100,6 +101,7 @@ def _stray(package: Path) -> list[str]: intents._DEFAULT_INTENTS, config.DEFAULT_PRODUCTS_PATH, spec.DEFAULT_SCHEMA_PATH, + conformance.CODEX_PROFILE, ], ids=lambda p: p.name) def test_default_data_files_ship_in_the_wheel(path): assert path.is_file() diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index e6b30c827..c5175f313 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -6,6 +6,7 @@ from __future__ import annotations +import importlib.resources import importlib.util import os import tomllib @@ -32,11 +33,11 @@ def _expect(**blocks: dict) -> dict: return {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "toy case", **blocks}}} -def _found(name: str) -> str: +def _found(name: str, search: str) -> str: return f"/usr/bin/{name}" -def _missing(name: str) -> None: +def _missing(name: str, search: str) -> None: return None @@ -78,6 +79,21 @@ def test_unavailable_names_the_missing_piece(): assert "not on PATH" in cf.unavailable(CODEX, "gpt", env, which=_missing) +def test_unavailable_looks_for_the_cli_on_the_path_the_run_gets(tmp_path, monkeypatch): + run_bin, parent_bin = tmp_path / "run", tmp_path / "parent" + for folder in (run_bin, parent_bin): + folder.mkdir() + (folder / "codex").write_text("#!/bin/sh\n", encoding="utf-8") + (folder / "codex").chmod(0o755) + empty = tmp_path / "empty" + empty.mkdir() + monkeypatch.setenv("PATH", str(parent_bin)) # this process's PATH has a codex; the run's does not + assert "not on PATH" in cf.unavailable(CODEX, "gpt", {"CODEX_API_KEY": "k", "PATH": str(empty)}) + assert cf.unavailable(CODEX, "gpt", {"CODEX_API_KEY": "k", "PATH": str(run_bin)}) is None + monkeypatch.setenv("PATH", str(empty)) # the other way round: the run's PATH has the codex, this process's has none + assert cf.unavailable(CODEX, "gpt", {"CODEX_API_KEY": "k", "PATH": str(run_bin)}) is None + + def test_unavailable_forwards_only_the_credentials_the_profile_forwards(): # codex's profile forwards CODEX_API_KEY alone, so OPENAI_API_KEY would not reach the run assert cf.unavailable(CODEX, "gpt", {"OPENAI_API_KEY": "k"}, which=_found) is not None @@ -478,6 +494,14 @@ def test_the_codex_suite_profile_authenticates_with_the_key_it_forwards(tmp_path assert provider["base_url"] == "https://api.openai.com/v1" +def test_the_codex_suite_profile_loads_through_the_package_resources(): + # The installed-wheel case: the file is found through the package, not through the source tree. + shipped = importlib.resources.files("dokimi.repos") / "conformance_profiles" / "codex.yaml" + with importlib.resources.as_file(shipped) as path: + assert load_profile(path).name == "codex" + assert path.read_bytes() == cf.CODEX_PROFILE.read_bytes() + + def test_run_harness_gives_each_run_exactly_the_environment_it_is_given(tmp_path, monkeypatch): fixture, _receipt = _toy_fixture(tmp_path) seen: list[dict[str, str]] = [] From 0b7a8bf3a46a6a3d9454d9a4374af00f3ab93003 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 10:32:21 -0800 Subject: [PATCH 20/29] docs(repo): say which preflight errors make run exit 1 run also exits 1 before any fixture runs: expectations drift, a harness given twice, an input that cannot be loaded. Errors-only and skipped runs still exit 0. Signed-off-by: Nikolay Petrov --- dokimi/docs/reach-matrix.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 634b60d71..549ca0d1b 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -88,7 +88,9 @@ is skipped, as is one without its credential or its CLI (`unavailable` in `dokim names which). `run` writes `.matrix.json` to `--out`, and the per-fixture receipts under `-runs/`. Each scenario is capped by `--timeout` (seconds per scenario, default 300). `--budget` (USD, per scenario, default 0.10) is passed to Claude Code as `--max-budget-usd` only: Codex and Copilot have no budget -flag, so for them the timeout is the only cap. `run` exits 1 only when a run is `failed`. `run` gives each +flag, so for them the timeout is the only cap. `run` exits 1 on a preflight or input error (expectations drift, a +harness given twice, an input that cannot be loaded, before any fixture runs) or when a run is `failed`; errors-only +and skipped runs exit 0. `run` gives each `repo reach` process exactly the environment it was started with, and each harness gets only what its profile forwards from that. From b4cb403580e9db8051e7c6e13e484630d8b26b7d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 16:17:14 -0800 Subject: [PATCH 21/29] fix(repo): check harness keys in the expectations; say what inconclusive and link cells mean The expectations check now names a harness key that no harness has, so a typo cannot silently leave its cells as no_claim. The matrix legend explains the inconclusive and link cells, and the run's --budget help says it caps Claude Code only. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/conformance.py | 7 +++++-- dokimi/scripts/reach_conformance.py | 3 ++- dokimi/tests/test_reach_conformance.py | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index c3a808084..2696e7feb 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -121,6 +121,8 @@ def _case_problems(case: str, block: Mapping[str, Any], root: Path) -> list[str] for harness, body in block.items(): if harness == "summary": continue + if harness not in HARNESSES: + out.append(f"{case}: {harness} is not a harness") for path, states in body["files"].items(): if not (root / path).is_file() and not (root / path).is_symlink(): out.append(f"{case}: {harness} names {path}, which the fixture does not hold") @@ -338,8 +340,9 @@ def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | No f"{_table_text(r['model'] or '-')} | {_table_text(r['status'])} | {_table_text(r['reason'] or '-')} |" for r in matrix["runs"]] lines += ["", "Cell legend: `loaded` or `not_loaded` as observed; `fetched` is read with a tool first; " - "`unknown` is no usable answer. `(docs: X)` marks a disagreement with the documentation; a " - "cell with no `(docs: ...)` agrees, or the documentation makes no claim.", ""] + "`unknown` is no usable answer. `(docs: X)` marks a disagreement with the documentation; `link, X` " + "marks a symlink; `inconclusive` is an observed `unknown` or `fetched` where the documentation names a " + "state; a cell with none of these agrees, or the documentation makes no claim.", ""] for case in sorted({c["case"] for c in matrix["cells"]}): lines += _case_section(matrix, case, columns, summaries.get(case, "")) return "\n".join(lines).rstrip() + "\n" diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index 6c3b50dd5..a1955f56d 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -96,7 +96,8 @@ def build_parser() -> argparse.ArgumentParser: metavar="NAME[=MODEL]", help="repeatable; no model makes the harness skipped") run.add_argument("--out", type=Path, required=True, help="directory for .matrix.json") run.add_argument("--timeout", type=int, default=cf.DEFAULT_TIMEOUT_S, help="per-scenario seconds") - run.add_argument("--budget", type=float, default=cf.DEFAULT_BUDGET_USD, help="per-scenario USD cap") + run.add_argument("--budget", type=float, default=cf.DEFAULT_BUDGET_USD, + help="per-scenario USD cap for Claude Code only; Codex and Copilot have no budget flag") merge = sub.add_parser("merge", help="merge matrices into one JSON and one markdown page") merge.add_argument("inputs", nargs="+", type=Path) merge.add_argument("--out", type=Path, required=True) diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index c5175f313..9992dab05 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -343,6 +343,14 @@ def test_check_expectations_reports_each_kind_of_drift(tmp_path): assert "codex names scenario cwd, which" in problems # no "=": the empty directory is the fixture root, not a scenario +def test_check_expectations_names_a_harness_key_that_no_harness_has(tmp_path): + fixtures = tmp_path / "fixtures" + _tree(fixtures / "toy", {"AGENTS.md": "a\n"}) + expect = {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "s", "claud": { + "why": "w", "files": {"AGENTS.md": {"cwd=.": "loaded"}}}}}} + assert "toy: claud is not a harness" in cf.check_expectations(expect, fixtures) + + def test_load_expectations_rejects_another_schema(tmp_path): path = tmp_path / "expectations.yaml" path.write_text("schema: other.v1\ncases: {}\n", encoding="utf-8") @@ -363,6 +371,18 @@ def test_cli_run_rejects_an_unknown_harness(tmp_path): cli.main(["run", "--harness", "nope=x", "--out", str(tmp_path)]) +def test_cli_run_budget_help_says_it_caps_claude_code_only(capsys): + with pytest.raises(SystemExit): + cli.main(["run", "--help"]) + assert "Claude Code only" in " ".join(capsys.readouterr().out.split()) + + +def test_render_markdown_legend_names_the_inconclusive_and_link_cells(): + matrix = _matrix(runs=[_run()], cells=[_cell(observed="unknown", documented="loaded", agreement="inconclusive")]) + legend = cf.render_markdown(matrix).split("Cell legend", 1)[1].split("\n\n", 1)[0] + assert "inconclusive" in legend and "link" in legend + + def test_cli_merge_writes_json_and_markdown(tmp_path): digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) left, right = tmp_path / "left.json", tmp_path / "right.json" From 87c79cbce7ae41781e0b6596795e00e445f06ac3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 16:17:14 -0800 Subject: [PATCH 22/29] docs(repo): write reach outputs under results/ and reports/; reword the codex profile note The example run and merge write to the gitignored results/ and reports/ directories. A file cut at the size cap reads as unknown. The codex profile comment no longer names a PR number, which would go stale after merge. Signed-off-by: Nikolay Petrov --- dokimi/docs/reach-matrix.md | 6 +++--- dokimi/dokimi/repos/conformance_profiles/codex.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 549ca0d1b..b624078b0 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -44,7 +44,7 @@ arrangement that exercises one loading rule. | `claude-imports-agents` | `CLAUDE.md` containing `@AGENTS.md` | a Claude Code `@` import is followed; Codex does not read `CLAUDE.md` | | `override-file` | `AGENTS.md` beside `AGENTS.override.md` | Codex uses the override in place of `AGENTS.md` | | `subtree-claude-shadow` | `sub/AGENTS.md` beside `sub/CLAUDE.md` | a directory's `CLAUDE.md` shadows its `AGENTS.md` in Claude Code | -| `large-file` | a root `AGENTS.md` of about 40 KB | the size cap; the docs give no cut-off for Claude Code, and 32 KiB for Codex | +| `large-file` | a root `AGENTS.md` of about 40 KB | the size cap; the docs give no cut-off for Claude Code, and 32 KiB for Codex. A file cut at the cap reads as `unknown` | | `symlinked-file` | `CLAUDE.md` as a symbolic link to `AGENTS.md` | how a link is named and read; the docs do not say | The fixtures are plain directories. `guidance.yaml` next to them is a `repo_manifest.v1` that names every @@ -79,8 +79,8 @@ puts on the import path. ```sh PYTHONPATH=. python scripts/reach_conformance.py check -PYTHONPATH=. python scripts/reach_conformance.py run --harness codex= --out reach-out -PYTHONPATH=. python scripts/reach_conformance.py merge reach-out/*.matrix.json --out reach-matrix.json --md reach-matrix.md +PYTHONPATH=. python scripts/reach_conformance.py run --harness codex= --out results/reach-out +PYTHONPATH=. python scripts/reach_conformance.py merge results/reach-out/*.matrix.json --out reports/reach-matrix.json --md reports/reach-matrix.md ``` `--harness NAME[=MODEL]` is repeatable: `claude-code`, `codex` and `copilot` are known. A harness without a model diff --git a/dokimi/dokimi/repos/conformance_profiles/codex.yaml b/dokimi/dokimi/repos/conformance_profiles/codex.yaml index 92ea4e13c..f4d3d4f68 100644 --- a/dokimi/dokimi/repos/conformance_profiles/codex.yaml +++ b/dokimi/dokimi/repos/conformance_profiles/codex.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # The codex profile of the reach suite: profiles/codex.yaml plus a provider that reads CODEX_API_KEY, because the -# built-in one sends no bearer (401 Missing bearer). Keep the two in step; the rest is in intel/dokimi#364. +# built-in one sends no bearer (401 Missing bearer). Keep the two in step. schema: harness_profile.v1 name: codex bin: codex # npm's codex is a node launcher; mount the native one (.../vendor//bin/codex) with --bin From 07166bd1756be9c916a1740b9768b634dc84172b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 22:26:28 -0800 Subject: [PATCH 23/29] fix(repo): refuse an out directory that holds the fixtures; check only the scenarios reach runs A run clears / before it probes, so an out directory that holds the fixtures (or sits in them) deleted fixtures. run_harness and the run command now refuse it before anything is created or cleared. check also accepted scenario ids reach never runs (read=. and a cwd= directory with no guidance file); it now accepts only the ids reach produces. The receipt-failure doc now names invalid and unreadable receipts. Signed-off-by: Nikolay Petrov --- dokimi/docs/reach-matrix.md | 14 ++++--- dokimi/dokimi/repos/conformance.py | 22 ++++++++++- dokimi/scripts/reach_conformance.py | 1 + dokimi/tests/test_reach_conformance.py | 54 +++++++++++++++++++++++++- 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index b624078b0..40489f3fa 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -23,8 +23,9 @@ documented one. A harness that cannot run here gets a `skipped` run and no cells The matrix has two kinds of record: - `runs`: one per harness and fixture, with a status. `ran` means every scenario answered; `errors` means a - receipt came back with a scenario that gave no usable answer; `failed` means no receipt (a timeout, or a - crash before the receipt was written); `skipped` means the harness could not run here, with the reason. + receipt came back with a scenario that gave no usable answer; `failed` means no usable receipt (a timeout, a + crash before it was written, or an invalid or unreadable receipt); `skipped` means the harness could not run + here, with the reason. - `cells`: one per source file per scenario, with the observed state, the documented state and an agreement: `agrees`, `disagrees`, `inconclusive` (observed unknown, or only `fetched`: reach saw the file only in a `read=` scenario, after the agent read it with a tool, so its loading rule is not shown), `no_claim` (the docs say nothing), or `alias` (a link, whose state is its target's and is not compared). @@ -33,8 +34,8 @@ The schema is `reach_matrix.v1` (`dokimi/schemas/reach_matrix.v1.json`). ## The fixtures -Each directory under `data/reach-conformance/` is one case. Each has a root `AGENTS.md` or `CLAUDE.md` -arrangement that exercises one loading rule. +Each directory under `data/reach-conformance/` is one case. Each holds the guidance files of one loading rule, +at the root or in subdirectories. | case | what it holds | the rule it tests | |---|---|---| @@ -65,7 +66,8 @@ observation is reported, and nothing is claimed about it. Copilot has no documen CLI, so every Copilot cell is `no_claim`; a Copilot run still records what it observed. Scenario ids are `cwd=.` (start in the root), `cwd=` (start in a subdirectory) and `read=` (start in -the root and have the model read a file in ``). +the root and have the model read a file in ``). `cwd=` and `read=` exist only for a directory +below the root that holds a guidance file; `check` rejects any other id. `python scripts/reach_conformance.py check` verifies the expectations against the fixture tree: every fixture has an entry, every named file exists in its fixture, and every named scenario's directory exists. It exits 1 @@ -151,7 +153,7 @@ its CLI install. ## Adding a case or a harness -A case is a directory under `data/reach-conformance/` with a root guidance arrangement, plus an entry in +A case is a directory under `data/reach-conformance/` with the guidance files it exercises, plus an entry in `expectations.yaml` for every harness whose documentation says something about it. `check` must pass. A harness is a `Harness` in `HARNESSES` (`dokimi/repos/conformance.py`), with its CLI and the credential diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 2696e7feb..1823d9350 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -34,6 +34,8 @@ AGREEMENTS = ("agrees", "disagrees", "inconclusive", "no_claim", "alias") DEFAULT_TIMEOUT_S = 300 DEFAULT_BUDGET_USD = 0.10 +# The guidance names data/reach-conformance/guidance.yaml globs, so every harness probes them in each fixture. +MANIFEST_NAMES = ("AGENTS.md", "AGENTS.override.md", "CLAUDE.md", "CLAUDE.local.md") class ConformanceError(RuntimeError): @@ -116,8 +118,17 @@ def check_expectations(expect: Mapping[str, Any], fixtures: Path) -> list[str]: return problems +def scenarios_of(root: Path) -> set[str]: + """The scenario ids reach runs over one fixture: ``cwd=.``, and ``cwd=`` and ``read=`` for each + directory below the root that holds a guidance file the conformance manifest globs.""" + dirs = {p.relative_to(root).parent.as_posix() for p in root.rglob("*") if p.name in MANIFEST_NAMES} + dirs.discard(".") + return {"cwd=."} | {f"cwd={d}" for d in dirs} | {f"read={d}" for d in dirs} + + def _case_problems(case: str, block: Mapping[str, Any], root: Path) -> list[str]: out = [] + produced = scenarios_of(root) for harness, body in block.items(): if harness == "summary": continue @@ -127,8 +138,7 @@ def _case_problems(case: str, block: Mapping[str, Any], root: Path) -> list[str] if not (root / path).is_file() and not (root / path).is_symlink(): out.append(f"{case}: {harness} names {path}, which the fixture does not hold") for scenario in states: - kind, _, where = scenario.partition("=") - if kind not in ("cwd", "read") or not where or not (root / where).is_dir(): + if scenario not in produced: out.append(f"{case}: {harness} names scenario {scenario}, which the fixture does not have") return out @@ -240,12 +250,20 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P return record, cells_of(receipt, case, expect) +def check_out_dir(out: Path, fixtures: Path) -> None: + """Refuse an output directory that is, holds or sits in the fixtures: a run clears ``/`` first.""" + a, b = out.resolve(), fixtures.resolve() + if a == b or a.is_relative_to(b) or b.is_relative_to(a): + raise ConformanceError(f"{out} and the fixtures {fixtures} overlap: a run would clear a fixture's directory") + + def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[str, Any], out_dir: Path, *, timeout: int = DEFAULT_TIMEOUT_S, budget: float = DEFAULT_BUDGET_USD, env: Mapping[str, str], which: Callable[[str, str], str | None] = _which_on) -> dict[str, Any]: """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture. ``env`` decides availability and is the whole environment each run gets (nothing else is inherited).""" + check_out_dir(out_dir, fixtures) reason = unavailable(harness, model, env, which) guidance = fixtures / "guidance.yaml" runs: list[dict[str, Any]] = [] diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py index a1955f56d..764a2b9f5 100644 --- a/dokimi/scripts/reach_conformance.py +++ b/dokimi/scripts/reach_conformance.py @@ -57,6 +57,7 @@ def cmd_run(args: argparse.Namespace) -> int: if len(set(names)) != len(names): print("✗ each --harness may be given once: a second run would overwrite the first's matrix", file=sys.stderr) return 1 + cf.check_out_dir(args.out, args.fixtures) # before anything is created or cleared failed = False args.out.mkdir(parents=True, exist_ok=True) for harness, model in args.harness: diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 9992dab05..517ced0a4 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -13,6 +13,7 @@ from pathlib import Path import pytest +import yaml from dokimi import process from dokimi.artifacts import load_artifact, save_artifact, schema_errors @@ -351,6 +352,23 @@ def test_check_expectations_names_a_harness_key_that_no_harness_has(tmp_path): assert "toy: claud is not a harness" in cf.check_expectations(expect, fixtures) +def test_check_expectations_names_only_the_scenarios_reach_runs(tmp_path): + fixtures = tmp_path / "fixtures" + _tree(fixtures / "toy", {"AGENTS.md": "a\n", "sub/AGENTS.md": "b\n", "packages/app.py": "x\n"}) + expect = {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "s", "codex": {"why": "w", "files": { + "AGENTS.md": {"cwd=.": "loaded", "cwd=sub": "loaded", "read=sub": "loaded", "read=.": "loaded"}, + "sub/AGENTS.md": {"cwd=packages": "loaded"}}}}}} + problems = "\n".join(cf.check_expectations(expect, fixtures)) + assert "toy: codex names scenario read=., which" in problems # the root is never a read= directory + assert "toy: codex names scenario cwd=packages, which" in problems # packages holds no guidance file + assert "cwd=sub" not in problems and "read=sub" not in problems + + +def test_manifest_names_are_the_guidance_globs_of_the_conformance_manifest(): + manifest = yaml.safe_load((FIXTURES / "guidance.yaml").read_text(encoding="utf-8")) + assert sorted(glob.removeprefix("**/") for glob in manifest["guidance"]["globs"]) == sorted(cf.MANIFEST_NAMES) + + def test_load_expectations_rejects_another_schema(tmp_path): path = tmp_path / "expectations.yaml" path.write_text("schema: other.v1\ncases: {}\n", encoding="utf-8") @@ -448,6 +466,37 @@ def test_run_case_does_not_read_a_receipt_an_earlier_run_left(tmp_path, monkeypa assert record["status"] == "failed" and cells == [] +@pytest.mark.parametrize("layout", ["same", "parent", "inside", "through-link"]) +def test_run_harness_refuses_an_out_directory_that_holds_the_fixtures(tmp_path, monkeypatch, layout): + suite = tmp_path / "suite" + fixtures = suite / "codex-runs" # the out directory of a run is out/-runs: this is out_dir = fixtures + _tree(fixtures / "toy", {"AGENTS.md": "a\n"}) + (fixtures / "guidance.yaml").write_text("guidance: {}\n", encoding="utf-8") + out = {"same": fixtures, "parent": suite, "inside": fixtures / "toy", "through-link": tmp_path / "link"}[layout] + if layout == "through-link": + out.symlink_to(suite, target_is_directory=True) + + def no_run(*args, **kwargs): + raise AssertionError("a run must not start") + + monkeypatch.setattr(cf, "run_case", no_run) + with pytest.raises(cf.ConformanceError, match="overlap"): + cf.run_harness(CODEX, "m", fixtures, _expect(), out, env={"CODEX_API_KEY": "k", "PATH": "/usr/bin"}, + which=_found) + assert (fixtures / "toy" / "AGENTS.md").read_text(encoding="utf-8") == "a\n" + assert (fixtures / "guidance.yaml").is_file() + + +def test_cli_run_refuses_an_out_directory_that_holds_the_fixtures(monkeypatch, capsys): + def no_call(*args, **kwargs): + raise AssertionError("run_harness must not be called") + + monkeypatch.setattr(cli.cf, "run_harness", no_call) + assert cli.main(["run", "--harness", "codex=m", "--out", str(FIXTURES.parent)]) == 1 # out holds the fixtures + assert "overlap" in capsys.readouterr().err + assert not (FIXTURES.parent / "codex-runs").exists() + + def test_cli_run_refuses_a_harness_given_twice_before_any_call(tmp_path, monkeypatch, capsys): def no_call(*args, **kwargs): raise AssertionError("run_harness must not be called") @@ -523,7 +572,8 @@ def test_the_codex_suite_profile_loads_through_the_package_resources(): def test_run_harness_gives_each_run_exactly_the_environment_it_is_given(tmp_path, monkeypatch): - fixture, _receipt = _toy_fixture(tmp_path) + fixtures = tmp_path / "fixtures" + _toy_fixture(fixtures) seen: list[dict[str, str]] = [] def fake_run(argv, *, timeout, env, **_): @@ -532,7 +582,7 @@ def fake_run(argv, *, timeout, env, **_): monkeypatch.setattr(cf.process, "run", fake_run) given = {"CODEX_API_KEY": "k", "PATH": "/usr/bin"} - matrix = cf.run_harness(CODEX, "gpt", tmp_path, _expect(), tmp_path / "runs", env=given, which=_found) + matrix = cf.run_harness(CODEX, "gpt", fixtures, _expect(), tmp_path / "runs", env=given, which=_found) assert seen == [given] assert [r["status"] for r in matrix["runs"]] == ["failed"] From f5f92d7b1926a78abe6817e0b4c5e19a3878ceec Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 02:56:40 -0800 Subject: [PATCH 24/29] fix(repo): fail a reach run in which no scenario answered A rejected key or a hung CLI left every scenario without an answer, yet the run was recorded as errors, so `run` exited 0 and the nightly published a matrix with no observation. Such a run is now failed, and an errors or failed run's reason is the first scenario error from reach.json, not reach's stderr. Signed-off-by: Alexander Rybkin --- dokimi/docs/reach-matrix.md | 3 +- dokimi/dokimi/repos/conformance.py | 15 +++++++++- dokimi/dokimi/schemas/reach_matrix.v1.json | 2 +- dokimi/tests/test_reach_conformance.py | 35 +++++++++++++++++++++- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 40489f3fa..4e83ec633 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -24,7 +24,8 @@ The matrix has two kinds of record: - `runs`: one per harness and fixture, with a status. `ran` means every scenario answered; `errors` means a receipt came back with a scenario that gave no usable answer; `failed` means no usable receipt (a timeout, a - crash before it was written, or an invalid or unreadable receipt); `skipped` means the harness could not run + crash before it was written, an invalid or unreadable receipt, or one in which no scenario answered, as when the + credential is rejected; the reason is the first scenario's error); `skipped` means the harness could not run here, with the reason. - `cells`: one per source file per scenario, with the observed state, the documented state and an agreement: `agrees`, `disagrees`, `inconclusive` (observed unknown, or only `fetched`: reach saw the file only in a `read=` scenario, after the agent read it with a tool, so its loading rule is not shown), `no_claim` (the docs say nothing), diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 1823d9350..cf4569124 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -220,6 +220,15 @@ def _reason(proc: process.Completed) -> str: return f"exit {proc.returncode}" + (f": {tail}" if tail else "") +def _scenario_rows(out: Path) -> list[dict[str, Any]]: + """The scenario rows of the run's ``reach.json``; none when it is missing or unreadable.""" + try: + rows = json.loads((out / "reach.json").read_text(encoding="utf-8"))["scenarios"] + except (OSError, ValueError, KeyError, TypeError): + return [] + return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else [] + + def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: Path, expect: Mapping[str, Any], *, timeout: int, budget: float, env: Mapping[str, str]) -> tuple[dict[str, Any], list[dict[str, Any]]]: @@ -243,8 +252,12 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P receipt = load_artifact("effective_instructions", receipt_path) except (OSError, ValueError) as exc: # one bad receipt fails its run, not the whole harness return run_record(harness.name, case, "failed", f"invalid receipt: {exc}", UNKNOWN, model, 0), [] + rows = _scenario_rows(out) + errors = [str(r["error"]) for r in rows if r.get("error")] + if rows and len(errors) == len(rows): # no scenario answered (a revoked key, a hung CLI): nothing was observed + return run_record(harness.name, case, "failed", errors[0], receipt["runtime"]["version"], model, 0), [] status = "ran" if proc.returncode == 0 else "errors" - reason = None if status == "ran" else _reason(proc) + reason = None if status == "ran" else (errors[0] if errors else _reason(proc)) record = run_record(harness.name, case, status, reason, receipt["runtime"]["version"], model, len(receipt["receipts"])) return record, cells_of(receipt, case, expect) diff --git a/dokimi/dokimi/schemas/reach_matrix.v1.json b/dokimi/dokimi/schemas/reach_matrix.v1.json index 14b5d4709..5691e2abe 100644 --- a/dokimi/dokimi/schemas/reach_matrix.v1.json +++ b/dokimi/dokimi/schemas/reach_matrix.v1.json @@ -20,7 +20,7 @@ "version": {"type": "string", "description": "The harness's own version as reach reports it; `unknown` when no receipt came back."}, "model": {"type": "string", "description": "The model string passed to the harness; empty when none was configured."}, "case": {"type": "string", "description": "The fixture repository's directory name."}, - "status": {"enum": ["ran", "errors", "failed", "skipped"], "description": "ran: every scenario answered; errors: a receipt came back with at least one scenario that gave no usable answer; failed: no receipt; skipped: the harness could not run here."}, + "status": {"enum": ["ran", "errors", "failed", "skipped"], "description": "ran: every scenario answered; errors: a receipt came back with at least one scenario that gave no usable answer; failed: no usable receipt, or no scenario answered; skipped: the harness could not run here."}, "reason": {"type": ["string", "null"]}, "scenarios": {"type": "integer", "minimum": 0} } diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 517ced0a4..e6b825641 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -8,6 +8,7 @@ import importlib.resources import importlib.util +import json import os import tomllib from pathlib import Path @@ -164,7 +165,7 @@ def test_build_matrix_counts_and_validates_against_the_schema(): def _fake_reach(monkeypatch, receipt: dict, *, returncode: int = 0, timed_out: bool = False, - write: bool = True) -> list[list[str]]: + write: bool = True, rows: list[dict] | None = None) -> list[list[str]]: calls: list[list[str]] = [] def fake_run(argv, *, timeout, env, **_): @@ -173,6 +174,8 @@ def fake_run(argv, *, timeout, env, **_): if write: out.mkdir(parents=True, exist_ok=True) save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + if rows is not None: + (out / "reach.json").write_text(json.dumps({"scenarios": rows}), encoding="utf-8") return process.Completed(argv=tuple(argv), returncode=returncode, timed_out=timed_out, stderr_bytes=b"boom") @@ -209,6 +212,36 @@ def test_run_case_marks_a_run_with_a_failing_scenario_as_errors(tmp_path, monkey assert cells +AUTH_ERROR = "exit 1 (no answer): API Error: 403 Authentication failed" + + +def test_run_case_with_no_scenario_answered_is_failed_and_names_the_scenario_error(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, returncode=1, rows=[{**ROOT_ROW, "error": AUTH_ERROR}]) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert (record["status"], record["reason"], cells) == ("failed", AUTH_ERROR, []) + + +def test_run_case_with_one_scenario_unanswered_is_errors_and_names_its_error(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + rows = [ROOT_ROW, {**ROOT_ROW, "id": "read=sub", "error": AUTH_ERROR}] + _fake_reach(monkeypatch, receipt, returncode=1, rows=rows) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert (record["status"], record["reason"]) == ("errors", AUTH_ERROR) + assert cells + + +def test_cli_run_exits_1_when_no_scenario_of_a_fixture_answered(tmp_path, monkeypatch, capsys): + _fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, returncode=1, rows=[{**ROOT_ROW, "error": AUTH_ERROR}]) + monkeypatch.setattr(cf.shutil, "which", lambda cmd, path=None: f"/usr/bin/{cmd}") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake") + assert cli.main(["run", "--harness", "claude-code=m", "--out", str(tmp_path / "o")]) == 1 + assert "errors 0, failed 8" in capsys.readouterr().out + + @pytest.mark.parametrize(("kwargs", "reason"), [ ({"write": False}, "exit 0"), ({"timed_out": True, "returncode": process.TIMEOUT_RC}, "timed out"), From fabc6f86a90d8f658b09a5dbca21e3510fbb4dae Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 02:57:24 -0800 Subject: [PATCH 25/29] fix(repo): score no not_loaded from a reach scenario that answered in prose A model that refuses the probe answers in prose with no tokens; the receipt flags its not_loaded states as possible under-reports, but the matrix scored them against the docs and published false disagreements. Such a not_loaded is now observed unknown, so its cell is inconclusive. Signed-off-by: Alexander Rybkin --- dokimi/docs/reach-matrix.md | 2 +- dokimi/dokimi/repos/conformance.py | 17 +++++++++++------ dokimi/tests/test_reach_conformance.py | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 4e83ec633..50cef1eb0 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -28,7 +28,7 @@ The matrix has two kinds of record: credential is rejected; the reason is the first scenario's error); `skipped` means the harness could not run here, with the reason. - `cells`: one per source file per scenario, with the observed state, the documented state and an agreement: - `agrees`, `disagrees`, `inconclusive` (observed unknown, or only `fetched`: reach saw the file only in a `read=` scenario, after the agent read it with a tool, so its loading rule is not shown), `no_claim` (the docs say nothing), + `agrees`, `disagrees`, `inconclusive` (observed unknown, which includes a `not_loaded` from a scenario that answered in prose and so may under-report; or only `fetched`: reach saw the file only in a `read=` scenario, after the agent read it with a tool, so its loading rule is not shown), `no_claim` (the docs say nothing), or `alias` (a link, whose state is its target's and is not compared). The schema is `reach_matrix.v1` (`dokimi/schemas/reach_matrix.v1.json`). diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index cf4569124..f4648e6d8 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -20,12 +20,13 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from typing import Any, Callable, Mapping, Sequence +from typing import Any, Callable, Collection, Mapping, Sequence import yaml from dokimi import process from dokimi.artifacts import load_artifact, save_artifact, schema_errors +from dokimi.repos.reach import PROSE_LIMIT SCHEMA_EXPECTATIONS = "reach_conformance.v1" RECEIPT_NAME = "effective_instructions.json" @@ -168,8 +169,10 @@ def agreement(observed: str, documented_state: str, alias: bool) -> str: return "agrees" if observed == documented_state else "disagrees" -def cells_of(receipt: Mapping[str, Any], case: str, expect: Mapping[str, Any]) -> list[dict[str, Any]]: - """One cell per source per scenario of one receipt (an effective-instructions receipt of one fixture).""" +def cells_of(receipt: Mapping[str, Any], case: str, expect: Mapping[str, Any], + prose: Collection[str] = ()) -> list[dict[str, Any]]: + """One cell per source per scenario of one receipt (an effective-instructions receipt of one fixture). + A ``not_loaded`` in a scenario of ``prose`` (it answered in prose, so it may under-report) is observed unknown.""" harness, version = receipt["runtime"]["name"], receipt["runtime"]["version"] out = [] for run in receipt["receipts"]: @@ -177,9 +180,10 @@ def cells_of(receipt: Mapping[str, Any], case: str, expect: Mapping[str, Any]) - for src in run["sources"]: alias = src.get("alias_of") state = documented(expect, case, harness, src["path"], scenario) + observed = UNKNOWN if scenario in prose and src["observed"] == "not_loaded" else src["observed"] cell = {"harness": harness, "version": version, "case": case, "scenario": scenario, - "path": src["path"], "observed": src["observed"], "documented": state, - "agreement": agreement(src["observed"], state, alias is not None)} + "path": src["path"], "observed": observed, "documented": state, + "agreement": agreement(observed, state, alias is not None)} if alias: cell["alias_of"] = alias out.append(cell) @@ -260,7 +264,8 @@ def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: P reason = None if status == "ran" else (errors[0] if errors else _reason(proc)) record = run_record(harness.name, case, status, reason, receipt["runtime"]["version"], model, len(receipt["receipts"])) - return record, cells_of(receipt, case, expect) + prose = {str(r.get("id")) for r in rows if (r.get("prose") or 0) > PROSE_LIMIT} + return record, cells_of(receipt, case, expect, prose) def check_out_dir(out: Path, fixtures: Path) -> None: diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index e6b825641..3a970aaf4 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -233,6 +233,20 @@ def test_run_case_with_one_scenario_unanswered_is_errors_and_names_its_error(tmp assert cells +def test_run_case_scores_no_not_loaded_of_a_scenario_that_answered_in_prose(tmp_path, monkeypatch): + fixture = tmp_path / "toy" + tree = _tree(fixture, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + row = {**ROOT_ROW, "prose": 14} # a refusal: words, no tokens + records = [_record("AGENTS.md", "cwd=.", "not_loaded"), _record("CLAUDE.md", "cwd=.", "loaded")] + _fake_reach(monkeypatch, build_receipt(_reach(records, [row]), BUDGET, tree), rows=[row]) + expect = _expect(**{"claude-code": {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}, + "CLAUDE.md": {"cwd=.": "loaded"}}}}) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", expect, + timeout=30, budget=0.5, env={}) + found = {c["path"]: (c["observed"], c["agreement"]) for c in cells if c["path"] in ("AGENTS.md", "CLAUDE.md")} + assert found == {"AGENTS.md": ("unknown", "inconclusive"), "CLAUDE.md": ("loaded", "agrees")} + assert record["status"] == "ran" + def test_cli_run_exits_1_when_no_scenario_of_a_fixture_answered(tmp_path, monkeypatch, capsys): _fixture, receipt = _toy_fixture(tmp_path) _fake_reach(monkeypatch, receipt, returncode=1, rows=[{**ROOT_ROW, "error": AUTH_ERROR}]) From f723fd3ceb65ffeeaa4d016e63edce80c9d9ecd0 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 02:58:25 -0800 Subject: [PATCH 26/29] fix(repo): stamp the fixture tree in a reach matrix and refuse to merge two expectations_digest covers the documented states only. A matrix observed over an edited fixture tree merged with one over the original, and the edit read as a change between harness versions. reach_matrix.v1 now carries fixtures_digest, and merge refuses matrices with different ones. Signed-off-by: Alexander Rybkin --- dokimi/docs/reach-matrix.md | 4 ++- dokimi/dokimi/repos/conformance.py | 24 ++++++++++--- dokimi/dokimi/schemas/reach_matrix.v1.json | 3 +- .../fixtures/artifacts/reach_matrix.v1.json | 1 + dokimi/tests/test_reach_conformance.py | 35 ++++++++++++++++--- 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 50cef1eb0..270b593f8 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -107,7 +107,9 @@ authenticates the run. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and `merge` joins matrices from several harnesses or runs. It refuses a second run of the same harness version on the same fixture, so a merge cannot double-count (failed and skipped runs hold no cells and are kept as they are). Each matrix carries `expectations_digest`, the sha256 of the expectations it was scored against; `merge` refuses matrices -with different digests, and the `merge` command refuses ones whose digest is not that of its `--expectations` file. +with different digests, and the `merge` command refuses ones whose digest is not that of its `--expectations` file. Each also carries +`fixtures_digest`, the sha256 of the fixture tree its runs observed, and `merge` refuses matrices with different +ones: an edited fixture would otherwise read as a change between harness versions. The offline tests (`tests/test_reach_conformance.py`) cover the agreement rules, scenario ids, alias handling, skipped and failed runs, merge, rendering, the drift checks, and the CLI, with the harness runs faked. No test diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index f4648e6d8..9f078afbb 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -109,6 +109,18 @@ def expectations_digest(expect: Mapping[str, Any]) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() +def fixtures_digest(fixtures: Path) -> str: + """The sha256 of the fixture tree (each file's path and bytes, a link's target): what the observed states come from.""" + h = hashlib.sha256() + for p in sorted(fixtures.rglob("*"), key=lambda p: p.relative_to(fixtures).as_posix()): + if not (p.is_symlink() or p.is_file()): + continue + data = b"L" + os.readlink(p).encode() if p.is_symlink() else b"F" + p.read_bytes() + for part in (p.relative_to(fixtures).as_posix().encode(), data): + h.update(len(part).to_bytes(8, "big") + part) + return h.hexdigest() + + def check_expectations(expect: Mapping[str, Any], fixtures: Path) -> list[str]: """What makes the expectations disagree with the fixture tree; empty when they agree.""" declared, found = set(expect["cases"]), set(cases(fixtures)) @@ -203,13 +215,14 @@ def summary(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any] def build_matrix(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]], digest: str, - generated_at: str | None = None) -> dict[str, Any]: + fixtures: str, generated_at: str | None = None) -> dict[str, Any]: """The ``reach_matrix.v1`` payload: runs and cells in a stable order, with the counts.""" return { "schema_version": "reach_matrix.v1", "generated_at": generated_at or datetime.now(timezone.utc).isoformat(), "expectations": SCHEMA_EXPECTATIONS, "expectations_digest": digest, + "fixtures_digest": fixtures, "runs": sorted((dict(r) for r in runs), key=lambda r: (r["case"], r["harness"], r["version"])), "cells": sorted((dict(c) for c in cells), key=lambda c: (c["case"], c["harness"], c["version"], c["path"], c["scenario"])), @@ -294,19 +307,22 @@ def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[st timeout=timeout, budget=budget, env=env) runs.append(record) cells += found - return build_matrix(runs, cells, expectations_digest(expect)) + return build_matrix(runs, cells, expectations_digest(expect), fixtures_digest(fixtures)) def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: """One matrix from several: the runs and cells of each, and no run of the same known harness version twice. - The matrices must come from one revision of the expectations, else their documented states differ.""" + The matrices must come from one revision of the expectations and of the fixtures, else their states differ.""" if not matrices: raise ConformanceError("nothing to merge") digests = sorted({m["expectations_digest"] for m in matrices}) if len(digests) > 1: raise ConformanceError(f"the matrices come from {len(digests)} revisions of the expectations: " + ", ".join(d[:12] for d in digests)) + trees = sorted({m["fixtures_digest"] for m in matrices}) + if len(trees) > 1: + raise ConformanceError(f"the matrices come from {len(trees)} fixture trees: " + ", ".join(d[:12] for d in trees)) runs: list[dict[str, Any]] = [] cells: list[dict[str, Any]] = [] seen: set[tuple[str, str, str]] = set() @@ -321,7 +337,7 @@ def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: runs += matrix["runs"] cells += matrix["cells"] latest = max((m["generated_at"] for m in matrices), key=_instant) - return build_matrix(runs, cells, digests[0], generated_at=latest) + return build_matrix(runs, cells, digests[0], trees[0], generated_at=latest) def _instant(text: str) -> datetime: diff --git a/dokimi/dokimi/schemas/reach_matrix.v1.json b/dokimi/dokimi/schemas/reach_matrix.v1.json index 5691e2abe..b14a59dbb 100644 --- a/dokimi/dokimi/schemas/reach_matrix.v1.json +++ b/dokimi/dokimi/schemas/reach_matrix.v1.json @@ -4,12 +4,13 @@ "title": "Reach conformance matrix v1", "description": "The matrix of the reach conformance suite (docs/reach-matrix.md): for each harness run over each fixture repository, the state each guidance file was observed in, per reach scenario, next to what the harness's vendor documentation says (reach_conformance.v1). One matrix is written per harness and several are merged into one. A `skipped` run has no cells; a cell whose documented state is `unknown` makes no claim.", "type": "object", - "required": ["schema_version", "generated_at", "expectations", "expectations_digest", "runs", "cells", "summary"], + "required": ["schema_version", "generated_at", "expectations", "expectations_digest", "fixtures_digest", "runs", "cells", "summary"], "properties": { "schema_version": {"const": "reach_matrix.v1"}, "generated_at": {"type": "string"}, "expectations": {"const": "reach_conformance.v1", "description": "The schema of the expectations file the documented states come from."}, "expectations_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The sha256 of the expectations file's content as canonical JSON (sorted keys, no spaces); matrices with different digests are not merged."}, + "fixtures_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The sha256 of the fixture tree the runs observed (each file's relative path and bytes, a link's target); matrices with different digests are not merged."}, "runs": { "type": "array", "items": { diff --git a/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json index fbcc5fcc2..41e810f01 100644 --- a/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json +++ b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json @@ -3,6 +3,7 @@ "generated_at": "2026-10-31T04:37:00+00:00", "expectations": "reach_conformance.v1", "expectations_digest": "0000000000000000000000000000000000000000000000000000000000000000", + "fixtures_digest": "0000000000000000000000000000000000000000000000000000000000000000", "runs": [ { "harness": "codex", diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 3a970aaf4..ca23e1a83 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -10,6 +10,7 @@ import importlib.util import json import os +import shutil import tomllib from pathlib import Path @@ -66,10 +67,11 @@ def _cell(**fields) -> dict: DIGEST = "a" * 64 +TREE = "c" * 64 -def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00", digest=DIGEST) -> dict: - return cf.build_matrix([_run()] if runs is None else runs, [_cell()] if cells is None else cells, digest, +def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00", digest=DIGEST, tree=TREE) -> dict: + return cf.build_matrix([_run()] if runs is None else runs, [_cell()] if cells is None else cells, digest, tree, generated_at=generated_at) @@ -158,7 +160,7 @@ def test_build_matrix_counts_and_validates_against_the_schema(): runs = [_run(), _run(harness="copilot", status="skipped", reason="no key", version="unknown")] cells = [_cell(), _cell(agreement="disagrees", observed="not_loaded", documented="loaded", path="CLAUDE.md"), _cell(agreement="alias", path="LINK.md", alias_of="AGENTS.md")] - matrix = cf.build_matrix(runs, cells, DIGEST, generated_at="2026-10-31T00:00:00+00:00") + matrix = cf.build_matrix(runs, cells, DIGEST, TREE, generated_at="2026-10-31T00:00:00+00:00") assert matrix["summary"] == {"agrees": 1, "disagrees": 1, "inconclusive": 0, "no_claim": 0, "alias": 1, "runs": {"ran": 1, "errors": 0, "failed": 0, "skipped": 1}} assert schema_errors("reach_matrix.v1", matrix) == [] @@ -324,6 +326,31 @@ def test_merge_refuses_matrices_from_two_revisions_of_the_expectations(): cf.merge([_matrix(), second]) +def test_merge_refuses_matrices_observed_over_two_fixture_trees(tmp_path, monkeypatch): + expect = cf.load_expectations(FIXTURES / "expectations.yaml") + edited = tmp_path / "fixtures" + shutil.copytree(FIXTURES, edited, symlinks=True) + (edited / "nested-agents" / "packages" / "app" / "CLAUDE.md").write_text("x\n", encoding="utf-8") + monkeypatch.setattr(cf.process, "run", lambda *a, **k: pytest.fail("a skipped harness must not run")) + pristine, changed = (cf.run_harness(CODEX, "", fx, expect, tmp_path / name, env={}, which=_found) + for fx, name in ((FIXTURES, "a"), (edited, "b"))) + assert pristine["fixtures_digest"] == cf.fixtures_digest(FIXTURES) != changed["fixtures_digest"] + with pytest.raises(cf.ConformanceError, match="2 fixture trees"): + cf.merge([pristine, changed]) + + +def test_fixtures_digest_follows_each_file_and_link(tmp_path): + tree = tmp_path / "t" + (tree / "a").mkdir(parents=True) + (tree / "a" / "AGENTS.md").write_text("x\n", encoding="utf-8") + base = cf.fixtures_digest(tree) + assert cf.fixtures_digest(tree) == base + (tree / "a" / "AGENTS.md").write_text("y\n", encoding="utf-8") + edited = cf.fixtures_digest(tree) + (tree / "a" / "AGENTS.md").rename(tree / "a" / "CLAUDE.md") + assert len({base, edited, cf.fixtures_digest(tree)}) == 3 + + def test_expectations_digest_follows_the_content_not_the_key_order(): expect = cf.load_expectations(FIXTURES / "expectations.yaml") reordered = dict(reversed(list(expect.items()))) @@ -559,7 +586,7 @@ def no_call(*args, **kwargs): def test_cli_run_exits_1_only_when_a_run_failed(tmp_path, monkeypatch, status, exit_code): run = _run(status=status, reason="exit 1", version="unknown" if status == "failed" else "0.166.0", scenarios=0 if status == "failed" else 1) - monkeypatch.setattr(cli.cf, "run_harness", lambda *args, **kwargs: cf.build_matrix([run], [], DIGEST)) + monkeypatch.setattr(cli.cf, "run_harness", lambda *args, **kwargs: cf.build_matrix([run], [], DIGEST, TREE)) assert cli.main(["run", "--harness", "codex=m", "--out", str(tmp_path)]) == exit_code assert (tmp_path / "codex.matrix.json").is_file() From 1f06a8d442afa29d4f285af2ca8fca58570bfc8d Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 02:58:41 -0800 Subject: [PATCH 27/29] test(repo): hold the codex suite profile to the built-in one in every field The guard compared invoke, env_allow and egress only, so a change to the built-in profile's bin_companions, trace, tools or version_cmd passed while the nightly kept running the stale copy. The test now compares the whole document except the suite's config_files. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_reach_conformance.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index ca23e1a83..3a97e7e7b 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -637,6 +637,13 @@ def test_the_codex_suite_profile_authenticates_with_the_key_it_forwards(tmp_path assert provider["base_url"] == "https://api.openai.com/v1" +def test_the_codex_suite_profile_is_the_builtin_one_plus_its_provider(): + suite = yaml.safe_load(cf.CODEX_PROFILE.read_text(encoding="utf-8")) + builtin = yaml.safe_load((cf.CODEX_PROFILE.parents[1] / "profiles" / "codex.yaml").read_text(encoding="utf-8")) + assert list(suite.pop("config_files")) == ["config.toml"] + assert suite == builtin # the nightly measures the Codex setup `repo reach --harness codex` runs + + def test_the_codex_suite_profile_loads_through_the_package_resources(): # The installed-wheel case: the file is found through the package, not through the source tree. shipped = importlib.resources.files("dokimi.repos") / "conformance_profiles" / "codex.yaml" From 64d9b533ad0ca002212bcc55405be829bd8b8b22 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 14:01:03 +0000 Subject: [PATCH 28/29] docs(repo): say the fixtures digest covers the expectations and guidance files Signed-off-by: Anton Volkov --- dokimi/docs/reach-matrix.md | 5 +++-- dokimi/dokimi/repos/conformance.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md index 270b593f8..3e2fc8c05 100644 --- a/dokimi/docs/reach-matrix.md +++ b/dokimi/docs/reach-matrix.md @@ -108,8 +108,9 @@ authenticates the run. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and same fixture, so a merge cannot double-count (failed and skipped runs hold no cells and are kept as they are). Each matrix carries `expectations_digest`, the sha256 of the expectations it was scored against; `merge` refuses matrices with different digests, and the `merge` command refuses ones whose digest is not that of its `--expectations` file. Each also carries -`fixtures_digest`, the sha256 of the fixture tree its runs observed, and `merge` refuses matrices with different -ones: an edited fixture would otherwise read as a change between harness versions. +`fixtures_digest`, the sha256 of everything under the fixtures root its runs observed, and `merge` refuses matrices +with different ones: an edited fixture would otherwise read as a change between harness versions. The root also +holds `guidance.yaml` and, by default, `expectations.yaml`, so an edit to either, even to a comment, changes the digest. The offline tests (`tests/test_reach_conformance.py`) cover the agreement rules, scenario ids, alias handling, skipped and failed runs, merge, rendering, the drift checks, and the CLI, with the harness runs faked. No test diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 9f078afbb..474456acc 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -110,7 +110,7 @@ def expectations_digest(expect: Mapping[str, Any]) -> str: def fixtures_digest(fixtures: Path) -> str: - """The sha256 of the fixture tree (each file's path and bytes, a link's target): what the observed states come from.""" + """The sha256 of everything under the fixtures root (each file's path and bytes, a link's target), including its guidance file and, by default, its expectations file.""" h = hashlib.sha256() for p in sorted(fixtures.rglob("*"), key=lambda p: p.relative_to(fixtures).as_posix()): if not (p.is_symlink() or p.is_file()): From 150d55adc669f233ac692a5e5153b9b4cfab1c66 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 3 Nov 2026 08:20:57 -0800 Subject: [PATCH 29/29] test(reach): pin the fixtures digest's link, framing and walk-order rules The digest test only edited and renamed a regular file. Add three tests that each fail on one mutation of fixtures_digest (checked one by one): a link hashed by its target's bytes, a link hashed as a bare "L", links skipped, no length prefix on each part, and a walk-order-dependent sort. No code change. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_reach_conformance.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py index 3a97e7e7b..73d7cd458 100644 --- a/dokimi/tests/test_reach_conformance.py +++ b/dokimi/tests/test_reach_conformance.py @@ -351,6 +351,42 @@ def test_fixtures_digest_follows_each_file_and_link(tmp_path): assert len({base, edited, cf.fixtures_digest(tree)}) == 3 +def test_fixtures_digest_counts_a_link_by_its_target_text(tmp_path): + tree = tmp_path / "t" + tree.mkdir() + (tree / "real.md").write_text("same\n", encoding="utf-8") + (tree / "other.md").write_text("same\n", encoding="utf-8") + os.symlink("real.md", tree / "LINK.md") + linked = cf.fixtures_digest(tree) + (tree / "LINK.md").unlink() + os.symlink("other.md", tree / "LINK.md") # the same bytes behind it: only the link's text differs + retargeted = cf.fixtures_digest(tree) + assert retargeted != linked + (tree / "LINK.md").unlink() + assert cf.fixtures_digest(tree) not in {linked, retargeted} # a missing link is a change too + + +def test_fixtures_digest_frames_each_path_and_its_data(tmp_path): + one, two = tmp_path / "one", tmp_path / "two" + one.mkdir() + two.mkdir() + (one / "a").write_bytes(b"Fb") # path "a" then data "F" + "Fb": without length prefixes, "aFFb" + (two / "aF").write_bytes(b"b") # path "aF" then data "F" + "b": the same bytes, a different tree + assert cf.fixtures_digest(one) != cf.fixtures_digest(two) + + +def test_fixtures_digest_does_not_depend_on_the_walk_order(tmp_path, monkeypatch): + tree = tmp_path / "t" + tree.mkdir() + for name in ("a.md", "b.md", "c.md"): + (tree / name).write_text(name, encoding="utf-8") + os.symlink("a.md", tree / "L.md") + ordinary = cf.fixtures_digest(tree) + walk = type(tree).rglob + monkeypatch.setattr(type(tree), "rglob", lambda self, pattern: reversed(list(walk(self, pattern)))) + assert cf.fixtures_digest(tree) == ordinary # the same tree, walked in the other order: one digest + + def test_expectations_digest_follows_the_content_not_the_key_order(): expect = cf.load_expectations(FIXTURES / "expectations.yaml") reordered = dict(reversed(list(expect.items())))