From 09bcdfe1972f5a574d693d10aa7f4e81c2fe0562 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:44:46 -0700 Subject: [PATCH 01/15] feat(repo): effective-instructions receipt for repo reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `repo reach --format receipt` prints, and writes to effective_instructions.json in --out, the effective-instructions/v0 receipt (agentsmd/agents.md#213) of each probed scenario. It joins the probe's marker tokens with budget's start-up rules (Claude Code only): per source the path, sha256, scope, expected state, observed state, observation_method, the harness name and version, the displacements seen in the run, and the unknowns. followed is always unknown. The mapping and its limits are in docs/repo-evaluation-contract.md §7.1. The receipt is validated against schemas/effective_instructions.v1.json and registered in artifacts.SCHEMA_VERSIONS. The default output of repo reach is unchanged. Signed-off-by: Nikolay Petrov --- ...napetrov_effective-instructions-receipt.md | 8 + dokimi/docs/reference/cli-interface.json | 20 ++ dokimi/docs/repo-evaluation-contract.md | 64 +++++ dokimi/dokimi/artifacts.py | 4 +- dokimi/dokimi/commands/repo.py | 30 ++- dokimi/dokimi/repos/receipt.py | 229 ++++++++++++++++++ dokimi/dokimi/runs/kinds.py | 2 + .../schemas/effective_instructions.v1.json | 89 +++++++ .../artifacts/effective_instructions.v1.json | 104 ++++++++ dokimi/tests/test_repo_receipt.py | 197 +++++++++++++++ dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + 12 files changed, 745 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md create mode 100644 dokimi/dokimi/repos/receipt.py create mode 100644 dokimi/dokimi/schemas/effective_instructions.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/effective_instructions.v1.json create mode 100644 dokimi/tests/test_repo_receipt.py diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md new file mode 100644 index 000000000..0261c8a5e --- /dev/null +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -0,0 +1,8 @@ + + +### Added + +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`prompt_capture`), the harness's name and version, the displacements seen in the run, and the unknowns. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..550c2ec10 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10751,6 +10751,25 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": [ + "'matrix'", + "'receipt'" + ], + "const": null, + "default": "matrix", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--format", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -12884,6 +12903,7 @@ "'answers'", "'arms'", "'cli_routes'", + "'effective_instructions'", "'eval'", "'explore_runs'", "'guidance_reach'", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c2b873922..856b4616f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1049,6 +1049,70 @@ per scenario). token was in the request bytes. This settles cells where the model under-reports, which is how a small model produces false `truncated` cells. +### 7.1 Effective-instructions receipt (`effective_instructions.v1`) + +`repo reach --format receipt` prints, and writes to `effective_instructions.json` +in `--out`, a receipt in the shape of `effective-instructions/v0` +(agentsmd/agents.md#213). It is built from the reach payload and from the +start-up chain of `repo budget` (`repos/budget.py`). It makes no +LLM call. The schema is `schemas/effective_instructions.v1.json`; the code is +`repos/receipt.py`. + +One receipt is written per reach scenario, with `run_id` `:`. +Each source carries: + +| Receipt field | From | Meaning | +|---|---|---| +| `path`, `alias_of` | reach `files`, `aliases` | a link is `adapter` scope and takes its target's tokens | +| `sha256` | the text dokimi reads (LF-normalised UTF-8) | null when the file is not readable as text | +| `scope` | path | `repo` at the root or in `.claude/`; `subtree` for a nested AGENTS.md, CLAUDE.md or CLAUDE.local.md; else `unknown` | +| `expected` | budget `when` | see the table below | +| `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | +| `observation_method` | | `prompt_capture` for every observed source; `unknown` for an error | +| `states.accessed` | the Read tool | `true` if a Read-tool read was seen, `false` with tools off, else `unknown` | +| `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | +| `states.loaded` | the class | `true` for loaded, `false` for not_loaded, else `unknown`; a fetched file is not loaded | +| `states.followed` | | always `unknown` (the schema's `const`) | + +`runtime` and `model` are the harness's own name and version, as reach records +them. `observation.method` is `prompt_capture`: the model reports the tokens it +can see, and the request sent to the provider is not captured. A reported token +is evidence of loading, not a copy of the bytes received. + +**Expected state.** budget's start-up rules are Claude Code's, so they apply +only to `claude-code` runs; for other harnesses every `expected` is `unknown` +and the receipt says so. + +| budget `when` | `expected` | Reason | +|---|---|---| +| `start` | `true` | loaded at start-up in every run | +| `off` | `false` | a root CLAUDE\*.md turns AGENTS.md off | +| `lazy`, run directory is the file's directory or inside it | `true` | loaded when the work reaches that directory | +| `lazy`, run directory outside it | `false` | not reached by this run | +| `on demand` or absent | `unknown` | budget does not say | + +**Precedence.** `precedence_observed` lists a displacement only: a root +CLAUDE\*.md (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md`) observed +loaded in a run, with an `off` AGENTS.md observed not_loaded in that run. It +does not give an order of entry; budget's displacement rule is applied as +written, so nested AGENTS.md files that budget marks `off` appear too. + +**What the marker method can and cannot show.** It shows which tokens were +reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as +observations of that run. It cannot show: + +- that an instruction was followed (`followed` is always `unknown`); +- a read through a shell (`cat`, `rg`, `grep`), which the `Read` tool check + does not see; +- when a source entered the request, or in what order (no timestamps); +- start-up imports and skills from budget, which reach does not probe: they + appear with `observed: unknown` and a run-level unknown saying so; +- anything a model under-reports; a prose-heavy answer is flagged. + +Each run lists its own `unknowns` (no usable answer, a source with one token +reported, prose above the limit, unseeded tokens). The receipt-level `unknowns` +repeat the limits above. + ## 8. L0 scan (`repo_readiness.v1`) `repo scan ` checks every mechanically checkable claim in the guidance diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..893731d14 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -12,7 +12,8 @@ what moves it), ``repo_environment`` (static inspection of what a repository offers an agent), ``plan`` (the cells a run promises to measure), ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts -into an agent's context) and ``task_inspect`` (which eval-task format each +into an agent's context), ``effective_instructions`` (the effective-instructions/v0 +receipt of a reach probe) and ``task_inspect`` (which eval-task format each path under a tree is). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay @@ -56,6 +57,7 @@ "plan": "plan.v1", "result": "result.v1", "repo_budget": "repo_budget.v1", + "effective_instructions": "effective_instructions.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", } diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 88f64ad09..7a526da73 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -269,7 +269,6 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: from dokimi.repos.source import SourceArgumentError from dokimi.repos.reach import ( ReachError, - format_matrix, planned, probe_files, run_reach, @@ -325,12 +324,34 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) path = save_reach(payload, out_dir) - print(format_matrix(payload)) - print(f"wrote {path}") + _print_reach_output(tree, profile, payload, out_dir, path, receipt=args.format == "receipt") if any(s["error"] for s in payload["scenarios"]): sys.exit(1) +def _print_reach_output(tree, profile, payload: dict, out_dir: Path, path: Path, receipt: bool) -> None: + """The matrix on stdout; with ``receipt``, the effective-instructions receipt on stdout and the note on stderr.""" + from dokimi.repos.reach import format_matrix + + if receipt: + _print_receipt(tree, profile, payload, out_dir) + print(f"wrote {path}", file=sys.stderr) + else: + print(format_matrix(payload)) + print(f"wrote {path}") + + +def _print_receipt(tree, profile, payload: dict, out_dir: Path) -> None: + """The effective-instructions receipt of a probe: budget's start-up chain, reach's observed states.""" + from dokimi.artifacts import save_artifact + from dokimi.repos.budget import budget + from dokimi.repos.receipt import build_receipt + + receipt = build_receipt(payload, budget(tree, profile.guidance_names, profile.import_max_depth), tree) + written = save_artifact("effective_instructions", receipt, out_dir / "effective_instructions.json") + print(json.dumps(written, indent=2)) + + def cmd_repo_budget(args: argparse.Namespace) -> None: """What the repository puts into an agent's context at start-up, lazily and on demand.""" from dokimi.artifacts import save_artifact, stamp @@ -535,6 +556,9 @@ def register(sub, positive_int) -> None: reach_p.add_argument("--budget", type=float, default=0.10, help="Per-scenario USD cap (default: 0.10)") reach_p.add_argument("--timeout", type=positive_int, default=300, help="Per-scenario seconds") reach_p.add_argument("--keep", action="store_true", help="Keep the temporary workspace") + reach_p.add_argument("--format", choices=["matrix", "receipt"], default="matrix", + help="matrix: the table (default). receipt: print the effective-instructions/v0 receipt as JSON and " + "write it to effective_instructions.json in --out (docs/repo-evaluation-contract.md §7.1)") reach_p.add_argument("--dry-run", action="store_true", dest="dry_run", help="Print the files and the exact argv of every scenario; run nothing") reach_p.add_argument("--cache-dir", default=None, dest="cache_dir", help=cache_help) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py new file mode 100644 index 000000000..04531dc4c --- /dev/null +++ b/dokimi/dokimi/repos/receipt.py @@ -0,0 +1,229 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The effective-instructions receipt (``effective-instructions/v0``, agentsmd/agents.md#213) of one reach probe. + +Built from a ``guidance_reach.v1`` payload and the start-up chain of ``repo_budget.v1``. Per run (one reach +scenario) and per guidance source it gives the expected state (budget's start-up rules), the observed state (the +probe's marker tokens) and the states ``accessed``, ``selected``, ``loaded`` and ``followed``. What the probe +cannot establish is ``unknown``; ``followed`` always is. docs/repo-evaluation-contract.md §7.1 holds the mapping +and its limits. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + +from .reach import PROSE_LIMIT +from .source import RepoTree + +SCHEMA = "effective-instructions/v0" +SCHEMA_VERSION = "effective_instructions.v1" +UNKNOWN = "unknown" +# The probe's method: the model reports the marker tokens it can see; the request itself is not captured. +METHOD = "prompt_capture" +# Root-level files that turn AGENTS.md off in Claude Code (budget.py): the displacing set. +CLAUDE_ROOT = ("CLAUDE.md", "CLAUDE.local.md", ".claude/CLAUDE.md") +ROOT_NESTED = (".claude/CLAUDE.md", ".claude/AGENTS.md") # project memory in .claude/ applies repo-wide +SUBTREE_NAMES = ("AGENTS.md", "CLAUDE.md", "CLAUDE.local.md") +_STATE = {"loaded": "loaded", "fetched": "fetched", "not_loaded": "not_loaded"} +_LOADED: dict[str | None, bool] = {"loaded": True, "not_loaded": False} +DETAIL = ("the model is asked to report the marker tokens it can see; the request sent to the provider is not " + "captured, so a reported token is evidence of loading, not a copy of the bytes it received") +NOT_PROBED = "start-up imports and skills from repo_budget are not probed by reach; their observed state is unknown" + + +@dataclass(frozen=True) +class _Run: + row: dict[str, Any] # the reach scenario row + records: dict[tuple[str, str], dict[str, Any]] # (file, scenario id) -> record + when_of: dict[str, str] # canonical path -> budget's when + digests: dict[str, str | None] + workdir: str # the directory the run works in + rules: bool # budget's rules apply (Claude Code) + + +def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) -> dict[str, Any]: + """The ``effective_instructions.v1`` artifact: one ``effective-instructions/v0`` receipt per reach scenario.""" + harness = reach["harness"] + rules = harness["name"] == "claude-code" # budget's start-up rules are Claude Code's + when_of = _budget_when(reach, budget) if rules else {} + start_up = [p for p, when in when_of.items() if when in ("start", "off") and p not in _known(reach)] + digests = source_digests(tree, [p for p, _ in _probed(reach)] + start_up) + records = {(r["file"], r["scenario"]): r for r in reach["records"]} + runtime = {"name": harness["name"], "version": harness["version"]} + receipts = [] + for row in reach["scenarios"]: + run = _Run(row, records, when_of, digests, _workdir(row), rules) + receipts.append(_receipt(reach, run, runtime, start_up)) + return { + "schema_version": SCHEMA_VERSION, + "generated_at": reach["generated_at"], + "repo": reach["repo"], + "runtime": runtime, + "model": reach["model"], + "observation": {"method": METHOD, "detail": DETAIL}, + "unknowns": _global_unknowns(harness["name"], rules), + "receipts": receipts, + } + + +def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: + """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" + out: dict[str, str | None] = {} + for path in dict.fromkeys(paths): + try: + out[path] = hashlib.sha256(tree.read_text(path).encode("utf-8")).hexdigest() + except (OSError, ValueError, RuntimeError): # SourceError and UnicodeDecodeError are ValueErrors + out[path] = None + return out + + +def _probed(reach: dict[str, Any]) -> list[tuple[str, str | None]]: + """(path, alias_of) for every probed path: each guidance file, then the links to it.""" + out: list[tuple[str, str | None]] = [] + for item in reach["files"]: + out.append((item["path"], None)) + out.extend((alias, item["path"]) for alias in item.get("aliases", [])) + return out + + +def _known(reach: dict[str, Any]) -> set[str]: + return {path for path, _ in _probed(reach)} + + +def _budget_when(reach: dict[str, Any], budget: dict[str, Any]) -> dict[str, str]: + """budget's ``when`` of each guidance file and import, keyed by the file the path resolves to.""" + target_of = {alias: item["path"] for item in reach["files"] for alias in item.get("aliases", [])} + return {target_of.get(e["path"], e["path"]): e["when"] + for e in budget["entries"] if e["kind"] in ("guidance", "import")} + + +def _dir(path: str) -> str: + parent = str(PurePosixPath(path).parent) + return "" if parent == "." else parent + + +def _workdir(row: dict[str, Any]) -> str: + """The directory a run works in: the one holding the file it is told to read, else where it starts.""" + return _dir(row["read"]) if row.get("read") else row["cwd"] + + +def _receipt(reach: dict[str, Any], run: _Run, runtime: dict[str, str], start_up: list[str]) -> dict[str, Any]: + sources = [_source(path, alias_of, run) for path, alias_of in _probed(reach)] + sources += [_budget_only(path, run) for path in start_up] + off = {p for p, when in run.when_of.items() if when == "off"} + row = run.row + return { + "schema": SCHEMA, + "run_id": f"{reach['salt']}:{row['id']}", + "runtime": runtime, + "cwd": row["cwd"] or ".", + "target": row.get("read"), + "sources": sources, + "precedence_observed": _precedence(sources, off), + "unknowns": _run_unknowns(row, sources), + } + + +def _source(path: str, alias_of: str | None, run: _Run) -> dict[str, Any]: + target = alias_of or path + rec = run.records.get((target, run.row["id"])) + cls = rec["class"] if rec else None + observed, method = _observation(cls) + return { + "path": path, + **({"alias_of": alias_of} if alias_of else {}), + "sha256": run.digests.get(path), + "scope": _scope(path, alias_of is not None), + "expected": _expected(target, run.when_of.get(target), run), + "observed": observed, + "observation_method": method, + "states": _states(cls, run.row["tools"] == "none", bool(rec and rec["read_by_tool"])), + } + + +def _budget_only(path: str, run: _Run) -> dict[str, Any]: + """A start-up import or guidance file budget found and reach did not probe: expected, never observed.""" + return {"path": path, "sha256": run.digests.get(path), "scope": _scope(path, False), + "expected": _expected(path, run.when_of[path], run), "observed": UNKNOWN, + "observation_method": UNKNOWN, "states": _states(None, False, False)} + + +def _observation(cls: str | None) -> tuple[str, str]: + """(observed state, observation_method) of one reach class. A truncated cell saw one token: an observation, + no verdict.""" + if cls in _STATE: + return _STATE[cls], METHOD + if cls == "truncated": + return UNKNOWN, METHOD + return UNKNOWN, UNKNOWN # error, or no record (not probed in this run) + + +def _states(cls: str | None, tools_off: bool, read_by_tool: bool) -> dict[str, Any]: + """accessed: a Read-tool read was seen (false with no tools at all); selected: the probe saw the source enter the + request, by the harness or by the agent's own read; loaded: at the harness's own load step, not through a read.""" + accessed: bool | str = True if read_by_tool else (False if tools_off else UNKNOWN) + selected: bool | str = True if cls in ("loaded", "fetched") else UNKNOWN + return {"accessed": accessed, "selected": selected, "loaded": _LOADED.get(cls, UNKNOWN), "followed": UNKNOWN} + + +def _expected(target: str, when: str | None, run: _Run) -> bool | str: + """Start-up files are expected in every run; off ones in none; a nested lazy file in a run whose directory is its + subtree (or inside it), since the harness loads it when the work reaches that directory.""" + if not run.rules or when is None: + return UNKNOWN + if when in ("start", "off"): + return when == "start" + if when == "lazy": + d = _dir(target) + return run.workdir == d or run.workdir.startswith(d + "/") + return UNKNOWN + + +def _scope(path: str, alias: bool) -> str: + if alias: + return "adapter" # a link to another guidance file: its tokens are the target's + if "/" not in path or path in ROOT_NESTED: + return "repo" + return "subtree" if path.rsplit("/", 1)[-1] in SUBTREE_NAMES else UNKNOWN + + +def _precedence(sources: list[dict[str, Any]], off: set[str]) -> list[dict[str, str]]: + """A file budget says a root CLAUDE* file displaces, seen not loaded while that file was seen loaded in the run.""" + winners = sorted(s["path"] for s in sources if s["path"] in CLAUDE_ROOT and s["observed"] == "loaded") + if not winners: + return [] + return [{"winner": w, "displaced": s["path"]} + for s in sources if (s.get("alias_of") or s["path"]) in off and s["observed"] == "not_loaded" + for w in winners] + + +def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]]) -> list[str]: + out = [] + if row.get("error"): + out.append("no usable answer from this run: every observed state is unknown") + out += [f"{s['path']}: one of its two marker tokens was reported; loaded is not established" + for s in sources if s["observed"] == UNKNOWN and s["observation_method"] == METHOD] + if (row.get("prose") or 0) > PROSE_LIMIT: + out.append(f"answered with {row['prose']} words besides tokens; its not_loaded states may be under-reports") + if row.get("unknown"): + out.append("reported tokens that were never seeded: " + ", ".join(row["unknown"])) + return out + + +def _global_unknowns(name: str, rules: bool) -> list[str]: + out = [ + "followed is not observed: no marker shows what the model did with a loaded source", + "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", + "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", + "no load timestamps: the probe does not record when a source entered the request", + "the order in which sources entered the request is not observed; precedence_observed lists displacements only", + NOT_PROBED, + ] + if not rules: + out.append(f"budget's start-up rules are Claude Code's: no source has an expected state for {name}") + return out diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..b9b3d2cb1 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -63,6 +63,8 @@ class KindFields: counts=("cells",)), "result": KindFields(time=("generated_at",), counts=("cells",)), "repo_budget": KindFields(subject=("tree",), counts=("entries",)), + "effective_instructions": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",), + counts=("receipts",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), } diff --git a/dokimi/dokimi/schemas/effective_instructions.v1.json b/dokimi/dokimi/schemas/effective_instructions.v1.json new file mode 100644 index 000000000..b31064f18 --- /dev/null +++ b/dokimi/dokimi/schemas/effective_instructions.v1.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/effective_instructions.v1.json", + "title": "Effective-instructions receipt v1", + "description": "The effective-instructions/v0 receipt (agentsmd/agents.md#213) of one `repo reach` probe (docs/repo-evaluation-contract.md §7.1): one receipt per scenario, each source's expected state from budget's start-up rules and its observed state from the probe's marker tokens. `followed` is always unknown: reach never proves it.", + "type": "object", + "required": ["schema_version", "generated_at", "repo", "runtime", "model", "observation", "unknowns", "receipts"], + "properties": { + "schema_version": {"const": "effective_instructions.v1"}, + "generated_at": {"type": "string"}, + "repo": { + "type": "object", + "required": ["id", "tree"], + "properties": { + "id": {"type": "string"}, + "tree": {"type": "string"}, + "sha": {"type": ["string", "null"]}, + "manifest": {"type": "string"} + } + }, + "runtime": {"$ref": "#/$defs/runtime"}, + "model": {"type": "string"}, + "observation": { + "type": "object", + "required": ["method", "detail"], + "properties": { + "method": {"enum": ["prompt_capture"], "description": "The probe's method for every observed source."}, + "detail": {"type": "string"} + } + }, + "unknowns": {"type": "array", "items": {"type": "string"}}, + "receipts": { + "type": "array", + "items": { + "type": "object", + "required": ["schema", "run_id", "runtime", "cwd", "target", "sources", "precedence_observed", "unknowns"], + "properties": { + "schema": {"const": "effective-instructions/v0"}, + "run_id": {"type": "string", "description": "The reach salt and the scenario id, so the receipt names its record in reach.json."}, + "runtime": {"$ref": "#/$defs/runtime"}, + "cwd": {"type": "string", "description": "The directory the run started in; `.` is the repository root."}, + "target": {"type": ["string", "null"], "description": "The file the run was told to read (the probe file of a read= scenario), else null."}, + "sources": {"type": "array", "items": {"$ref": "#/$defs/source"}}, + "precedence_observed": { + "type": "array", + "description": "Displacements seen in this run: a root CLAUDE* file loaded while the AGENTS.md it displaces was not.", + "items": { + "type": "object", + "required": ["winner", "displaced"], + "properties": {"winner": {"type": "string"}, "displaced": {"type": "string"}} + } + }, + "unknowns": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "$defs": { + "runtime": { + "type": "object", + "required": ["name", "version"], + "properties": {"name": {"type": "string"}, "version": {"type": "string"}} + }, + "tristate": {"enum": [true, false, "unknown"]}, + "source": { + "type": "object", + "required": ["path", "sha256", "scope", "expected", "observed", "observation_method", "states"], + "properties": { + "path": {"type": "string"}, + "alias_of": {"type": "string", "description": "Set when the path is a link to another probed file; its tokens and its observation are the target's."}, + "sha256": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$", "description": "Null when the file cannot be read as text."}, + "scope": {"enum": ["repo", "subtree", "user", "adapter", "unknown"]}, + "expected": {"$ref": "#/$defs/tristate", "description": "Budget's start-up rules: true for start, false for off, true for a lazy file whose subtree the run works in. unknown without Claude Code's rules."}, + "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, + "observation_method": {"enum": ["runtime_event", "prompt_capture", "hook", "unknown"]}, + "states": { + "type": "object", + "required": ["accessed", "selected", "loaded", "followed"], + "properties": { + "accessed": {"$ref": "#/$defs/tristate"}, + "selected": {"$ref": "#/$defs/tristate"}, + "loaded": {"$ref": "#/$defs/tristate"}, + "followed": {"const": "unknown", "description": "Reach never proves that an instruction was followed."} + } + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json new file mode 100644 index 000000000..eee6afc56 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json @@ -0,0 +1,104 @@ +{ + "schema_version": "effective_instructions.v1", + "generated_at": "2026-10-31T00:00:00+00:00", + "repo": { + "id": "toy", + "tree": "toy", + "sha": null, + "manifest": "evals/repo-eval.yaml" + }, + "runtime": { + "name": "claude-code", + "version": "2.1.0" + }, + "model": "m", + "observation": { + "method": "prompt_capture", + "detail": "the model is asked to report the marker tokens it can see; the request sent to the provider is not captured, so a reported token is evidence of loading, not a copy of the bytes it received" + }, + "unknowns": [ + "followed is not observed: no marker shows what the model did with a loaded source", + "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", + "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", + "no load timestamps: the probe does not record when a source entered the request", + "the order in which sources entered the request is not observed; precedence_observed lists displacements only", + "start-up imports and skills from repo_budget are not probed by reach; their observed state is unknown" + ], + "receipts": [ + { + "schema": "effective-instructions/v0", + "run_id": "salt:cwd=.", + "runtime": { + "name": "claude-code", + "version": "2.1.0" + }, + "cwd": ".", + "target": null, + "sources": [ + { + "path": "AGENTS.md", + "sha256": "87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7", + "scope": "repo", + "expected": false, + "observed": "not_loaded", + "observation_method": "prompt_capture", + "states": { + "accessed": false, + "selected": "unknown", + "loaded": false, + "followed": "unknown" + } + }, + { + "path": "CLAUDE.md", + "sha256": "a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478", + "scope": "repo", + "expected": true, + "observed": "loaded", + "observation_method": "prompt_capture", + "states": { + "accessed": false, + "selected": true, + "loaded": true, + "followed": "unknown" + } + }, + { + "path": "sub/AGENTS.md", + "sha256": "cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556", + "scope": "subtree", + "expected": false, + "observed": "not_loaded", + "observation_method": "prompt_capture", + "states": { + "accessed": false, + "selected": "unknown", + "loaded": false, + "followed": "unknown" + } + }, + { + "path": "docs/style.md", + "sha256": "8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be", + "scope": "unknown", + "expected": true, + "observed": "unknown", + "observation_method": "unknown", + "states": { + "accessed": "unknown", + "selected": "unknown", + "loaded": "unknown", + "followed": "unknown" + } + } + ], + "precedence_observed": [ + { + "winner": "CLAUDE.md", + "displaced": "AGENTS.md" + } + ], + "unknowns": [] + } + ] +} diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py new file mode 100644 index 000000000..ebd8c6833 --- /dev/null +++ b/dokimi/tests/test_repo_receipt.py @@ -0,0 +1,197 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The effective-instructions receipt of a reach probe (no LLM): its mapping, its schema and the CLI.""" + +from __future__ import annotations + +import hashlib +import json + +import pytest + +from dokimi.artifacts import ArtifactValidationError, validate_artifact +from dokimi.cli import build_parser +from dokimi.repos import WorkTree +from dokimi.repos.receipt import build_receipt, source_digests +from tests.test_repo_reach import _profile, _repo + +ROOT_ROW = {"id": "cwd=.", "cwd": "", "tools": "none", "read": None, "error": None, "prose": 0, "unknown": []} +SUB_ROW = {"id": "read=sub", "cwd": "", "tools": "read", "read": "sub/REACH_PROBE.txt", "error": None, + "prose": 0, "unknown": []} +BUDGET = {"entries": [ + {"path": "CLAUDE.md", "kind": "guidance", "when": "start", "chars": 10}, + {"path": "AGENTS.md", "kind": "guidance", "when": "off", "chars": 10}, + {"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "docs/style.md", "kind": "import", "when": "start", "chars": 10}, +]} + + +def _record(file: str, scenario: str, cls: str, read_by_tool: bool = False) -> dict: + return {"file": file, "scenario": scenario, "harness": "claude-code", "harness_version": "2.1.0", + "model": "m", "head": cls in ("loaded", "fetched", "truncated"), "tail": cls in ("loaded", "fetched"), + "read_by_tool": read_by_tool, "class": cls} + + +def _reach(records: list[dict], scenarios: list[dict], harness: str = "claude-code", + files: list[dict] | None = None) -> dict: + files = files or [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + return {"generated_at": "2026-10-31T00:00:00+00:00", "repo": {"id": "toy", "tree": "toy", "sha": None, + "manifest": "evals/repo-eval.yaml"}, "harness": {"name": harness, "version": "2.1.0"}, "model": "m", + "salt": "salt", "files": files, "scenarios": scenarios, "records": records, "summary": {}} + + +def _tree(tmp_path, files: dict[str, str | bytes]) -> WorkTree: + for rel, text in files.items(): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + if isinstance(text, bytes): + (tmp_path / rel).write_bytes(text) + else: + (tmp_path / rel).write_text(text, encoding="utf-8") + return WorkTree(tmp_path) + + +def _sources(receipt: dict, run_id: str) -> dict[str, dict]: + run = next(r for r in receipt["receipts"] if r["run_id"].endswith(run_id)) + return {s["path"]: s for s in run["sources"]} + + +CLASS_MAP = { # reach class -> (observed, observation_method, selected, loaded) + "loaded": ("loaded", "prompt_capture", True, True), + "fetched": ("fetched", "prompt_capture", True, "unknown"), + "not_loaded": ("not_loaded", "prompt_capture", "unknown", False), + "truncated": ("unknown", "prompt_capture", "unknown", "unknown"), + "error": ("unknown", "unknown", "unknown", "unknown"), +} + + +@pytest.mark.parametrize("cls", sorted(CLASS_MAP)) +def test_each_reach_class_maps_to_its_observed_and_thread_states(tmp_path, cls): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", cls, read_by_tool=cls == "fetched")], [ROOT_ROW]) + src = _sources(build_receipt(reach, {"entries": []}, tree), "cwd=.")["AGENTS.md"] + observed, method, selected, loaded = CLASS_MAP[cls] + assert (src["observed"], src["observation_method"]) == (observed, method) + assert (src["states"]["selected"], src["states"]["loaded"]) == (selected, loaded) + assert src["states"]["followed"] == "unknown" + + +def test_start_up_expectations_follow_budget_and_the_run_directory(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n", "docs/style.md": "d\n"}) + records = [_record(f, s, "not_loaded") for f in ("AGENTS.md", "CLAUDE.md", "sub/AGENTS.md") + for s in ("cwd=.", "read=sub")] + reach = _reach(records, [ROOT_ROW, SUB_ROW]) + receipt = build_receipt(reach, BUDGET, tree) + root = _sources(receipt, "cwd=.") + assert root["CLAUDE.md"]["expected"] is True # start-up + assert root["AGENTS.md"]["expected"] is False # off: a root CLAUDE.md displaces it + assert root["sub/AGENTS.md"]["expected"] is False # lazy, and the run starts at the root + assert root["docs/style.md"]["expected"] is True and root["docs/style.md"]["observed"] == "unknown" + assert _sources(receipt, "read=sub")["sub/AGENTS.md"]["expected"] is True # the run reads inside its subtree + + +def test_a_displacement_seen_in_the_run_is_precedence_observed(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + records = [_record("CLAUDE.md", "cwd=.", "loaded"), _record("AGENTS.md", "cwd=.", "not_loaded")] + receipt = build_receipt(_reach(records, [ROOT_ROW]), BUDGET, tree) + run = receipt["receipts"][0] + assert run["precedence_observed"] == [{"winner": "CLAUDE.md", "displaced": "AGENTS.md"}] + + +def test_no_displacement_claim_without_the_winner_loaded(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + records = [_record("CLAUDE.md", "cwd=.", "not_loaded"), _record("AGENTS.md", "cwd=.", "not_loaded")] + assert build_receipt(_reach(records, [ROOT_ROW]), BUDGET, tree)["receipts"][0]["precedence_observed"] == [] + + +def test_other_harnesses_get_no_expectations_and_say_so(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + records = [_record("AGENTS.md", "cwd=.", "loaded")] + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + receipt = build_receipt(_reach(records, [ROOT_ROW], harness="codex", files=files), BUDGET, tree) + src = _sources(receipt, "cwd=.") + assert set(src) == {"AGENTS.md", "CLAUDE.md"} # budget's start-up chain is Claude Code's, so none is added + assert all(s["expected"] == "unknown" for s in src.values()) + assert any("no source has an expected state for codex" in u for u in receipt["unknowns"]) + + +def test_a_link_is_an_adapter_with_the_targets_observation(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "sub/AGENTS.md": "s\n"}) + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002", "aliases": ["CLAUDE.md"]}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + records = [_record("AGENTS.md", "cwd=.", "loaded")] + src = _sources(build_receipt(_reach(records, [ROOT_ROW], files=files), {"entries": []}, tree), "cwd=.") + assert src["CLAUDE.md"]["alias_of"] == "AGENTS.md" + assert src["CLAUDE.md"]["scope"] == "adapter" and src["CLAUDE.md"]["observed"] == "loaded" + assert src["sub/AGENTS.md"]["scope"] == "subtree" and src["sub/AGENTS.md"]["observed"] == "unknown" + + +def test_a_scenario_with_no_answer_is_unknown_and_says_why(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + row = dict(ROOT_ROW, error="exit 1") + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "error")], [row]), {"entries": []}, tree) + run = receipt["receipts"][0] + assert all(s["observed"] == "unknown" for s in run["sources"]) + assert any("no usable answer" in u for u in run["unknowns"]) + + +def test_a_prose_heavy_or_unseeded_answer_is_flagged(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + row = dict(ROOT_ROW, prose=40, unknown=["RC0000FF"]) + run = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "not_loaded")], [row]), {"entries": []}, tree) + text = " ".join(run["receipts"][0]["unknowns"]) + assert "under-reports" in text and "RC0000FF" in text + + +def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\r\nb\n", "blob.md": b"\xff\xfe\x00bin"}) + digests = source_digests(tree, ["AGENTS.md", "blob.md", "missing.md"]) + assert digests["AGENTS.md"] == hashlib.sha256("a\nb\n".encode("utf-8")).hexdigest() # LF, as dokimi reads it + assert digests["blob.md"] is None and digests["missing.md"] is None + + +def test_the_receipt_is_a_valid_artifact_and_a_schema_rejects_a_followed_claim(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + validate_artifact("effective_instructions", receipt) + receipt["receipts"][0]["sources"][0]["states"]["followed"] = "loaded" + with pytest.raises(ArtifactValidationError, match="followed"): + validate_artifact("effective_instructions", receipt) + + +def _end_to_end_args(tmp_path, monkeypatch, *extra): + monkeypatch.setenv("FAKE_MODE", "ok") + root = _repo(tmp_path) + profile = _profile(tmp_path, name="claude-code", guidance={"names": ["CLAUDE.md", "AGENTS.md"]}) + work = tmp_path / "work" + work.mkdir() + return ["repo", "reach", str(root), "--manifest", "evals/repo-eval.yaml", "--harness", str(profile), + "--model", "m", "--out", str(tmp_path / "out"), "--work-dir", str(work), "--salt", "s", + "--scenarios", "root", *extra] + + +def test_cli_receipt_format_prints_and_writes_a_valid_receipt(tmp_path, monkeypatch, capsys): + args = build_parser().parse_args(_end_to_end_args(tmp_path, monkeypatch, "--format", "receipt")) + args.func(args) + out = capsys.readouterr().out + data = json.loads(out) + validate_artifact("effective_instructions", data) + assert json.loads((tmp_path / "out" / "effective_instructions.json").read_text(encoding="utf-8")) == data + run = data["receipts"][0] + assert run["run_id"] == "s:cwd=." and run["runtime"] == {"name": "claude-code", "version": "9.8.7"} + states = {s["path"]: s["observed"] for s in run["sources"]} + assert states["CLAUDE.md"] == "loaded" and states["AGENTS.md"] == "not_loaded" + # budget turns off every AGENTS.md while a root CLAUDE.md exists, nested ones included + assert sorted(p["displaced"] for p in run["precedence_observed"]) == ["AGENTS.md", "sub/AGENTS.md"] + assert all(p["winner"] == "CLAUDE.md" for p in run["precedence_observed"]) + assert out.lstrip().startswith("{") and "wrote" not in out # the matrix is not printed in receipt mode + + +def test_default_format_still_prints_the_matrix(tmp_path, monkeypatch, capsys): + args = build_parser().parse_args(_end_to_end_args(tmp_path, monkeypatch)) + args.func(args) + out = capsys.readouterr().out + assert "wrote" in out and not (tmp_path / "out" / "effective_instructions.json").exists() diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..3a24afd19 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -46,6 +46,7 @@ "plan.v1": ({}, "skill:onetbb", "claude-sonnet-4-6", "2026-10-21T00:00:00Z"), "result.v1": ({}, None, None, "2026-10-21T01:00:00Z"), "repo_budget.v1": ({}, "toy", None, None), + "effective_instructions.v1": ({}, "toy", "m", "2026-10-31T00:00:00+00:00"), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..170c1138b 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -32,6 +32,7 @@ "plugin_delta.v1": {"arms": 2}, "questions.v2": {"questions": 3}, "repo_budget.v1": {"entries": 9}, + "effective_instructions.v1": {"receipts": 1}, "repo_environment.v1": {"instructions": 4, "findings": 13}, "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6}, From 6260ae497096cb43f4c834ba03c962b341c6cfba Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:46:50 -0700 Subject: [PATCH 02/15] fix(repo): keep local directories out of the receipt's tree label A local source's tree label is its absolute path, and the receipt copied it verbatim: a shared effective-instructions receipt then carried the home path. An absolute or drive-letter or ~ label now keeps only its last directory; URLs and relative labels are unchanged. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/receipt.py | 14 ++++++++++++-- dokimi/tests/test_repo_receipt.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index 04531dc4c..b3f972815 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -14,7 +14,7 @@ import hashlib from dataclasses import dataclass -from pathlib import PurePosixPath +from pathlib import PurePosixPath, PureWindowsPath from typing import Any from .reach import PROSE_LIMIT @@ -62,7 +62,7 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) return { "schema_version": SCHEMA_VERSION, "generated_at": reach["generated_at"], - "repo": reach["repo"], + "repo": {**reach["repo"], "tree": _shareable_tree(reach["repo"]["tree"])}, "runtime": runtime, "model": reach["model"], "observation": {"method": METHOD, "detail": DETAIL}, @@ -71,6 +71,16 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) } +def _shareable_tree(label: str) -> str: + """The tree label with a local directory cut to its last component, so no home path reaches a shared receipt.""" + name, at, sha = label.rpartition("@") + if not at: + name, sha = label, "" + if not (PureWindowsPath(name).drive or PurePosixPath(name).is_absolute() or name.startswith("~")): + return label + return (PurePosixPath(name.replace("\\", "/")).name or "tree") + at + sha + + def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" out: dict[str, str | None] = {} diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index ebd8c6833..73e5d4cdc 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -153,6 +153,22 @@ def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): assert digests["blob.md"] is None and digests["missing.md"] is None +@pytest.mark.parametrize("label, shared", [ + ("/home/alice/src/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("C:\\Users\\alice\\toy", "toy"), + ("~alice/toy", "toy"), + ("https://example.com/org/toy.git@4ff761e2d943", "https://example.com/org/toy.git@4ff761e2d943"), + ("toy@4ff761e2d943", "toy@4ff761e2d943"), +]) +def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["tree"] = label + receipt = build_receipt(reach, {"entries": []}, tree) + assert receipt["repo"]["tree"] == shared + assert "alice" not in json.dumps(receipt) + + def test_the_receipt_is_a_valid_artifact_and_a_schema_rejects_a_followed_claim(tmp_path): tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) From 8dd7f27ec67caad161bf5af2911b31d41fff59b7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:18:29 -0700 Subject: [PATCH 03/15] feat(repo): reach --from reads a saved probe; receipt names its marker method The receipt's observation is dokimi's marker self-report, not a captured prompt: observation_method is unknown (the thread vocabulary has no method that fits), and dokimi_method is marker_self_report on every source the probe recorded. Schema v1 is edited in place, unreleased. repo reach --from builds the matrix or the receipt from a saved probe without a harness run. The source is read to check the file's repository, harness and commit; a mismatch, a missing or malformed file, or a probe-only flag is a usage error (exit 2). The receipt is written beside the saved file. --model is required only when probing. Signed-off-by: Nikolay Petrov --- ...napetrov_effective-instructions-receipt.md | 2 +- dokimi/docs/reference/cli-interface.json | 20 ++- dokimi/docs/repo-evaluation-contract.md | 26 ++-- dokimi/dokimi/commands/repo.py | 73 +++++++++- dokimi/dokimi/repos/receipt.py | 46 ++++--- .../schemas/effective_instructions.v1.json | 10 +- .../artifacts/effective_instructions.v1.json | 14 +- dokimi/tests/test_repo_receipt.py | 128 ++++++++++++++++-- 8 files changed, 267 insertions(+), 52 deletions(-) diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md index 0261c8a5e..ef71bd50c 100644 --- a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`prompt_capture`), the harness's name and version, the displacements seen in the run, and the unknowns. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 550c2ec10..e5fba1563 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10770,6 +10770,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--from", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -10879,7 +10895,7 @@ "kind": "option", "nargs": null, "position": null, - "required": true, + "required": false, "type": null }, { @@ -10902,7 +10918,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": "reach-out", + "default": null, "group_key": null, "group_required": null, "hidden": false, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 856b4616f..e4952c75d 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1055,8 +1055,13 @@ per scenario). in `--out`, a receipt in the shape of `effective-instructions/v0` (agentsmd/agents.md#213). It is built from the reach payload and from the start-up chain of `repo budget` (`repos/budget.py`). It makes no -LLM call. The schema is `schemas/effective_instructions.v1.json`; the code is -`repos/receipt.py`. +LLM call. `repo reach --from --format receipt` builds the same +receipt from a saved probe without running the harness: the source is read +only to check that the file is for the same repository, harness and commit +(a different one is a usage error, exit 2). The receipt is written beside the +saved file. The probe-only flags (`--model`, `--budget`, `--scenarios`, `--only` +and the rest) cannot be combined with `--from`. The schema is +`schemas/effective_instructions.v1.json`; the code is `repos/receipt.py`. One receipt is written per reach scenario, with `run_id` `:`. Each source carries: @@ -1068,16 +1073,19 @@ Each source carries: | `scope` | path | `repo` at the root or in `.claude/`; `subtree` for a nested AGENTS.md, CLAUDE.md or CLAUDE.local.md; else `unknown` | | `expected` | budget `when` | see the table below | | `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | -| `observation_method` | | `prompt_capture` for every observed source; `unknown` for an error | +| `observation_method` | | `unknown` for every source: the thread vocabulary's method, since no runtime event, prompt capture or hook backs the probe | +| `dokimi_method` | | `marker_self_report` for a source the probe recorded; null for a start-up file budget found and reach did not probe | | `states.accessed` | the Read tool | `true` if a Read-tool read was seen, `false` with tools off, else `unknown` | | `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | | `states.loaded` | the class | `true` for loaded, `false` for not_loaded, else `unknown`; a fetched file is not loaded | | `states.followed` | | always `unknown` (the schema's `const`) | `runtime` and `model` are the harness's own name and version, as reach records -them. `observation.method` is `prompt_capture`: the model reports the tokens it -can see, and the request sent to the provider is not captured. A reported token -is evidence of loading, not a copy of the bytes received. +them. `observation.method` is `unknown` and `observation.dokimi_method` is +`marker_self_report`: the model reports the tokens it can see, and the request +sent to the provider is not captured. A reported token is evidence of loading, +not a copy of the bytes received. The name is not `prompt_capture`, because +dokimi does not capture the prompt. **Expected state.** budget's start-up rules are Claude Code's, so they apply only to `claude-code` runs; for other harnesses every `expected` is `unknown` @@ -1094,8 +1102,10 @@ and the receipt says so. **Precedence.** `precedence_observed` lists a displacement only: a root CLAUDE\*.md (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md`) observed loaded in a run, with an `off` AGENTS.md observed not_loaded in that run. It -does not give an order of entry; budget's displacement rule is applied as -written, so nested AGENTS.md files that budget marks `off` appear too. +does not give an order of entry. The displacement follows budget's rule as +written (`repos/budget.py`: every AGENTS.md is `off` while a root CLAUDE\*.md +exists, nested ones included), so nested AGENTS.md files appear too. If budget's +rule changes, the receipt follows it. **What the marker method can and cannot show.** It shows which tokens were reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 7a526da73..5f3f7d596 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -250,6 +250,67 @@ def _check_reach_flags(args: argparse.Namespace, kinds: tuple[str, ...]) -> None raise ReachError(f"unknown --scenarios kind(s) {', '.join(unknown_kinds)} (known: root, cwd, read)") if not math.isfinite(args.budget) or args.budget <= 0: raise ReachError(f"--budget must be > 0, got {args.budget:g}") + if args.model is None: + raise ReachError("--model is required to probe (leave it out only with --from)") + + +# The probe's flags and their defaults: --from refuses any of them set to another value. +_PROBE_ONLY = {"only": None, "salt": None, "work_dir": None, "keep": False, "bin": None, "dry_run": False, + "model": None, "scenarios": "root,cwd,read", "max_turns": 4, "budget": 0.10, "timeout": 300, + "out": None} + + +def _reach_entry(args: argparse.Namespace) -> None: + """`repo reach`: probe the repository, or with --from read a saved reach.json.""" + if args.from_file: + _reach_from_saved(args) + else: + cmd_repo_reach(args) + + +def _probe_flags_given(args: argparse.Namespace) -> list[str]: + return [f"--{k.replace('_', '-')}" for k, default in _PROBE_ONLY.items() if getattr(args, k) != default] + + +def _reach_from_saved(args: argparse.Namespace) -> None: + """The matrix or the receipt of a saved reach.json, checked against the source; no harness runs.""" + from dokimi.repos import ManifestError, RepoSource, SourceError, open_source + from dokimi.repos.harness_profile import ProfileError, load_profile + from dokimi.repos.receipt import check_saved_reach + from dokimi.repos.reach import ReachError, format_matrix, load_reach + from dokimi.repos.source import SourceArgumentError + + try: # the arguments and the saved file as typed: a usage error + given = _probe_flags_given(args) + if given: + raise ReachError(f"--from reads a saved probe, so {', '.join(given)} cannot be used with it") + profile = load_profile(args.harness) + source = RepoSource.parse(args.source) + reach = load_reach(Path(args.from_file)) + except (SourceError, ProfileError, ReachError, OSError, ValueError) as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + try: # the clone and fetch: a runtime failure + tree = open_source(source, _cache_dir(args)) + except SourceArgumentError as exc: # a pinned local path that is not the repository root + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + except (SourceError, OSError, ValueError) as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(1) + try: # the manifest and the file's provenance: a usage error; a failed git read is a fetch failure (1) + manifest = _load_manifest(tree, args, guidance_default=True) + check_saved_reach(reach, manifest.id, profile.name, getattr(tree, "sha", None)) + except (SourceError, ManifestError, ReachError) as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(_tree_read_exit_code(exc)) + if args.format == "receipt": + _print_receipt(tree, profile, reach, Path(args.from_file).parent) + print(f"wrote {Path(args.from_file).parent / 'effective_instructions.json'}", file=sys.stderr) + else: + print(format_matrix(reach)) + if any(s["error"] for s in reach["scenarios"]): + sys.exit(1) def _tree_read_exit_code(exc: Exception) -> int: @@ -312,7 +373,7 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: sys.exit(_tree_read_exit_code(exc)) print(f"{len(files)} guidance files, {planned_count} scenarios, " f"at most ${args.budget:g} each", file=sys.stderr) - out_dir = Path(args.out) + out_dir = Path(args.out or "reach-out") try: # the harness runs: a runtime failure payload = run_reach( tree=tree, manifest=manifest, profile=profile, model=args.model, out_dir=out_dir, @@ -541,9 +602,13 @@ def register(sub, positive_int) -> None: "at --manifest (for a repository that has not added it yet)") reach_p.add_argument("--harness", default="claude-code", help="Built-in profile name or a harness_profile.v1 YAML path (default: claude-code)") - reach_p.add_argument("--model", required=True, help="Model string passed to the harness as it is") + reach_p.add_argument("--model", default=None, help="Model string passed to the harness as it is (required " + "unless --from)") + reach_p.add_argument("--from", default=None, dest="from_file", metavar="REACH_JSON", + help="Read a saved reach.json instead of probing: no harness runs, no cost. The source is " + "still read, to check the file's repository, harness and sha; the receipt goes beside the file") reach_p.add_argument("--bin", default=None, help="Harness executable (default: the profile's `bin`)") - reach_p.add_argument("--out", default="reach-out", help="Output directory: reach.json and traces/") + reach_p.add_argument("--out", default=None, help="Output directory: reach.json and traces/ (default: reach-out)") reach_p.add_argument("--scenarios", default="root,cwd,read", help="Comma list of scenario kinds: root, cwd, read (default: all)") reach_p.add_argument("--only", action="append", default=None, metavar="SCENARIO", @@ -562,7 +627,7 @@ def register(sub, positive_int) -> None: reach_p.add_argument("--dry-run", action="store_true", dest="dry_run", help="Print the files and the exact argv of every scenario; run nothing") reach_p.add_argument("--cache-dir", default=None, dest="cache_dir", help=cache_help) - reach_p.set_defaults(func=cmd_repo_reach) + reach_p.set_defaults(func=_reach_entry) budget_p = repo_sub.add_parser( "budget", diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index b3f972815..1c7075deb 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -17,19 +17,21 @@ from pathlib import PurePosixPath, PureWindowsPath from typing import Any -from .reach import PROSE_LIMIT +from .reach import PROSE_LIMIT, ReachError from .source import RepoTree SCHEMA = "effective-instructions/v0" SCHEMA_VERSION = "effective_instructions.v1" UNKNOWN = "unknown" -# The probe's method: the model reports the marker tokens it can see; the request itself is not captured. -METHOD = "prompt_capture" +# The thread vocabulary's observation_method: no runtime event, prompt capture or hook backs the probe. +OBSERVATION_METHOD = UNKNOWN +# dokimi's own method: the model reports the marker tokens it can see; the request itself is not captured. +DOKIMI_METHOD = "marker_self_report" # Root-level files that turn AGENTS.md off in Claude Code (budget.py): the displacing set. CLAUDE_ROOT = ("CLAUDE.md", "CLAUDE.local.md", ".claude/CLAUDE.md") ROOT_NESTED = (".claude/CLAUDE.md", ".claude/AGENTS.md") # project memory in .claude/ applies repo-wide SUBTREE_NAMES = ("AGENTS.md", "CLAUDE.md", "CLAUDE.local.md") -_STATE = {"loaded": "loaded", "fetched": "fetched", "not_loaded": "not_loaded"} +_STATE: dict[str | None, str] = {"loaded": "loaded", "fetched": "fetched", "not_loaded": "not_loaded"} _LOADED: dict[str | None, bool] = {"loaded": True, "not_loaded": False} DETAIL = ("the model is asked to report the marker tokens it can see; the request sent to the provider is not " "captured, so a reported token is evidence of loading, not a copy of the bytes it received") @@ -65,7 +67,7 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) "repo": {**reach["repo"], "tree": _shareable_tree(reach["repo"]["tree"])}, "runtime": runtime, "model": reach["model"], - "observation": {"method": METHOD, "detail": DETAIL}, + "observation": {"method": OBSERVATION_METHOD, "dokimi_method": DOKIMI_METHOD, "detail": DETAIL}, "unknowns": _global_unknowns(harness["name"], rules), "receipts": receipts, } @@ -81,6 +83,17 @@ def _shareable_tree(label: str) -> str: return (PurePosixPath(name.replace("\\", "/")).name or "tree") + at + sha +def check_saved_reach(reach: dict[str, Any], repo_id: str, harness_name: str, tree_sha: str | None) -> None: + """Refuse a saved reach.json probed on another repository, harness or commit than the one it is read against.""" + if reach["repo"]["id"] != repo_id: + raise ReachError(f"reach.json is for repository {reach['repo']['id']!r}, not {repo_id!r}") + if reach["harness"]["name"] != harness_name: + raise ReachError(f"reach.json was probed with harness {reach['harness']['name']!r}, not {harness_name!r}") + probed = reach["repo"].get("sha") + if probed and tree_sha and probed != tree_sha: + raise ReachError(f"reach.json was probed at {probed[:12]}, the source is at {tree_sha[:12]}") + + def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" out: dict[str, str | None] = {} @@ -143,15 +156,15 @@ def _source(path: str, alias_of: str | None, run: _Run) -> dict[str, Any]: target = alias_of or path rec = run.records.get((target, run.row["id"])) cls = rec["class"] if rec else None - observed, method = _observation(cls) return { "path": path, **({"alias_of": alias_of} if alias_of else {}), "sha256": run.digests.get(path), "scope": _scope(path, alias_of is not None), "expected": _expected(target, run.when_of.get(target), run), - "observed": observed, - "observation_method": method, + "observed": _STATE.get(cls, UNKNOWN), # a truncated cell saw one token: an observation, no verdict + "observation_method": OBSERVATION_METHOD, + "dokimi_method": DOKIMI_METHOD if rec else None, "states": _states(cls, run.row["tools"] == "none", bool(rec and rec["read_by_tool"])), } @@ -160,17 +173,8 @@ def _budget_only(path: str, run: _Run) -> dict[str, Any]: """A start-up import or guidance file budget found and reach did not probe: expected, never observed.""" return {"path": path, "sha256": run.digests.get(path), "scope": _scope(path, False), "expected": _expected(path, run.when_of[path], run), "observed": UNKNOWN, - "observation_method": UNKNOWN, "states": _states(None, False, False)} - - -def _observation(cls: str | None) -> tuple[str, str]: - """(observed state, observation_method) of one reach class. A truncated cell saw one token: an observation, - no verdict.""" - if cls in _STATE: - return _STATE[cls], METHOD - if cls == "truncated": - return UNKNOWN, METHOD - return UNKNOWN, UNKNOWN # error, or no record (not probed in this run) + "observation_method": OBSERVATION_METHOD, "dokimi_method": None, + "states": _states(None, False, False)} def _states(cls: str | None, tools_off: bool, read_by_tool: bool) -> dict[str, Any]: @@ -216,8 +220,9 @@ def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]]) -> list[st out = [] if row.get("error"): out.append("no usable answer from this run: every observed state is unknown") + marked = [s for s in sources if s["observed"] == UNKNOWN and s["dokimi_method"] == DOKIMI_METHOD] out += [f"{s['path']}: one of its two marker tokens was reported; loaded is not established" - for s in sources if s["observed"] == UNKNOWN and s["observation_method"] == METHOD] + for s in marked if not row.get("error")] if (row.get("prose") or 0) > PROSE_LIMIT: out.append(f"answered with {row['prose']} words besides tokens; its not_loaded states may be under-reports") if row.get("unknown"): @@ -228,6 +233,7 @@ def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]]) -> list[st def _global_unknowns(name: str, rules: bool) -> list[str]: out = [ "followed is not observed: no marker shows what the model did with a loaded source", + "observation_method is unknown: no prompt is captured; dokimi_method names the marker self-report instead", "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", "no load timestamps: the probe does not record when a source entered the request", diff --git a/dokimi/dokimi/schemas/effective_instructions.v1.json b/dokimi/dokimi/schemas/effective_instructions.v1.json index b31064f18..bf81c3a69 100644 --- a/dokimi/dokimi/schemas/effective_instructions.v1.json +++ b/dokimi/dokimi/schemas/effective_instructions.v1.json @@ -22,9 +22,10 @@ "model": {"type": "string"}, "observation": { "type": "object", - "required": ["method", "detail"], + "required": ["method", "dokimi_method", "detail"], "properties": { - "method": {"enum": ["prompt_capture"], "description": "The probe's method for every observed source."}, + "method": {"enum": ["unknown"], "description": "The thread vocabulary's observation_method: no runtime event, prompt capture or hook backs the probe."}, + "dokimi_method": {"const": "marker_self_report", "description": "dokimi's method: the model reports the marker tokens it can see; the request is not captured."}, "detail": {"type": "string"} } }, @@ -64,7 +65,7 @@ "tristate": {"enum": [true, false, "unknown"]}, "source": { "type": "object", - "required": ["path", "sha256", "scope", "expected", "observed", "observation_method", "states"], + "required": ["path", "sha256", "scope", "expected", "observed", "observation_method", "dokimi_method", "states"], "properties": { "path": {"type": "string"}, "alias_of": {"type": "string", "description": "Set when the path is a link to another probed file; its tokens and its observation are the target's."}, @@ -72,7 +73,8 @@ "scope": {"enum": ["repo", "subtree", "user", "adapter", "unknown"]}, "expected": {"$ref": "#/$defs/tristate", "description": "Budget's start-up rules: true for start, false for off, true for a lazy file whose subtree the run works in. unknown without Claude Code's rules."}, "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, - "observation_method": {"enum": ["runtime_event", "prompt_capture", "hook", "unknown"]}, + "observation_method": {"enum": ["runtime_event", "prompt_capture", "hook", "unknown"], "description": "The thread vocabulary's method; unknown for every source, since no runtime event backs the probe."}, + "dokimi_method": {"enum": ["marker_self_report", null], "description": "Set when the probe saw a marker token for the source; null for a source it did not probe."}, "states": { "type": "object", "required": ["accessed", "selected", "loaded", "followed"], diff --git a/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json index eee6afc56..11a646f66 100644 --- a/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json +++ b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json @@ -13,11 +13,13 @@ }, "model": "m", "observation": { - "method": "prompt_capture", + "method": "unknown", + "dokimi_method": "marker_self_report", "detail": "the model is asked to report the marker tokens it can see; the request sent to the provider is not captured, so a reported token is evidence of loading, not a copy of the bytes it received" }, "unknowns": [ "followed is not observed: no marker shows what the model did with a loaded source", + "observation_method is unknown: no prompt is captured; dokimi_method names the marker self-report instead", "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", "no load timestamps: the probe does not record when a source entered the request", @@ -41,7 +43,8 @@ "scope": "repo", "expected": false, "observed": "not_loaded", - "observation_method": "prompt_capture", + "observation_method": "unknown", + "dokimi_method": "marker_self_report", "states": { "accessed": false, "selected": "unknown", @@ -55,7 +58,8 @@ "scope": "repo", "expected": true, "observed": "loaded", - "observation_method": "prompt_capture", + "observation_method": "unknown", + "dokimi_method": "marker_self_report", "states": { "accessed": false, "selected": true, @@ -69,7 +73,8 @@ "scope": "subtree", "expected": false, "observed": "not_loaded", - "observation_method": "prompt_capture", + "observation_method": "unknown", + "dokimi_method": "marker_self_report", "states": { "accessed": false, "selected": "unknown", @@ -84,6 +89,7 @@ "expected": true, "observed": "unknown", "observation_method": "unknown", + "dokimi_method": null, "states": { "accessed": "unknown", "selected": "unknown", diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index 73e5d4cdc..6c5fea595 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -13,7 +13,8 @@ from dokimi.artifacts import ArtifactValidationError, validate_artifact from dokimi.cli import build_parser from dokimi.repos import WorkTree -from dokimi.repos.receipt import build_receipt, source_digests +from dokimi.repos.receipt import build_receipt, check_saved_reach, source_digests +from dokimi.repos.reach import ReachError, format_matrix, load_reach from tests.test_repo_reach import _profile, _repo ROOT_ROW = {"id": "cwd=.", "cwd": "", "tools": "none", "read": None, "error": None, "prose": 0, "unknown": []} @@ -58,12 +59,12 @@ def _sources(receipt: dict, run_id: str) -> dict[str, dict]: return {s["path"]: s for s in run["sources"]} -CLASS_MAP = { # reach class -> (observed, observation_method, selected, loaded) - "loaded": ("loaded", "prompt_capture", True, True), - "fetched": ("fetched", "prompt_capture", True, "unknown"), - "not_loaded": ("not_loaded", "prompt_capture", "unknown", False), - "truncated": ("unknown", "prompt_capture", "unknown", "unknown"), - "error": ("unknown", "unknown", "unknown", "unknown"), +CLASS_MAP = { # reach class -> (observed, selected, loaded) + "loaded": ("loaded", True, True), + "fetched": ("fetched", True, "unknown"), + "not_loaded": ("not_loaded", "unknown", False), + "truncated": ("unknown", "unknown", "unknown"), + "error": ("unknown", "unknown", "unknown"), } @@ -72,12 +73,24 @@ def test_each_reach_class_maps_to_its_observed_and_thread_states(tmp_path, cls): tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) reach = _reach([_record("AGENTS.md", "cwd=.", cls, read_by_tool=cls == "fetched")], [ROOT_ROW]) src = _sources(build_receipt(reach, {"entries": []}, tree), "cwd=.")["AGENTS.md"] - observed, method, selected, loaded = CLASS_MAP[cls] - assert (src["observed"], src["observation_method"]) == (observed, method) + observed, selected, loaded = CLASS_MAP[cls] + assert src["observed"] == observed + assert src["observation_method"] == "unknown" # the thread vocabulary: no runtime event backs the probe + assert src["dokimi_method"] == "marker_self_report" # the probe saw a marker record for this source assert (src["states"]["selected"], src["states"]["loaded"]) == (selected, loaded) assert src["states"]["followed"] == "unknown" +def test_a_source_the_probe_did_not_record_has_no_dokimi_method(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "docs/style.md": "d\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + receipt = build_receipt(reach, BUDGET, tree) + src = _sources(receipt, "cwd=.") + assert src["docs/style.md"]["dokimi_method"] is None and src["docs/style.md"]["observation_method"] == "unknown" + assert receipt["observation"] == {"method": "unknown", "dokimi_method": "marker_self_report", + "detail": receipt["observation"]["detail"]} + + def test_start_up_expectations_follow_budget_and_the_run_directory(tmp_path): tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n", "docs/style.md": "d\n"}) records = [_record(f, s, "not_loaded") for f in ("AGENTS.md", "CLAUDE.md", "sub/AGENTS.md") @@ -211,3 +224,100 @@ def test_default_format_still_prints_the_matrix(tmp_path, monkeypatch, capsys): args.func(args) out = capsys.readouterr().out assert "wrote" in out and not (tmp_path / "out" / "effective_instructions.json").exists() + + +def _from_argv(live: list[str], *extra: str) -> list[str]: + """The argv that reads a saved reach.json against the same source and harness as a live run's argv.""" + return ["repo", "reach", live[2], "--manifest", "evals/repo-eval.yaml", "--harness", + live[live.index("--harness") + 1], *extra] + + +def _saved(tmp_path) -> str: + return str(tmp_path / "out" / "reach.json") + + +def test_from_a_saved_reach_gives_the_receipt_of_the_live_run(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch, "--format", "receipt") + args = build_parser().parse_args(live) + args.func(args) + live_receipt = json.loads(capsys.readouterr().out) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + args.func(args) + assert json.loads(capsys.readouterr().out) == live_receipt + assert (tmp_path / "out" / "effective_instructions.json").exists() # beside the saved file + + +def test_from_prints_the_matrix_of_the_saved_probe_and_runs_nothing(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(live) + args.func(args) + capsys.readouterr() + + def no_probe(**_kwargs): + raise AssertionError("a saved probe must not run the harness") + + monkeypatch.setattr("dokimi.repos.reach.run_reach", no_probe) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path))) + args.func(args) + assert capsys.readouterr().out == format_matrix(load_reach(tmp_path / "out" / "reach.json")) + "\n" + + +def test_from_a_missing_file_is_a_usage_error(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(_from_argv(live, "--from", str(tmp_path / "nope.json"))) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "nope.json" in capsys.readouterr().err + + +@pytest.mark.parametrize("text", ["{not json", "[1, 2]", '{"schema": "guidance_reach.v1"}']) +def test_from_a_malformed_or_wrong_shaped_file_is_a_usage_error(tmp_path, monkeypatch, capsys, text): + live = _end_to_end_args(tmp_path, monkeypatch) + bad = tmp_path / "bad.json" + bad.write_text(text, encoding="utf-8") + args = build_parser().parse_args(_from_argv(live, "--from", str(bad))) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and capsys.readouterr().err.startswith("\u2717") + + +def test_from_refuses_a_saved_probe_of_another_repository_or_harness(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + for key, value, text in (("repo", {**data["repo"], "id": "other"}, "for repository 'other'"), + ("harness", {**data["harness"], "name": "codex"}, "harness 'codex'")): + other = tmp_path / f"{key}.json" + other.write_text(json.dumps({**data, key: value}), encoding="utf-8") + args = build_parser().parse_args(_from_argv(live, "--from", str(other))) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and text in capsys.readouterr().err + + +def test_from_refuses_the_probe_only_flags(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--budget", "0.5", "--model", "m")) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "--model, --budget" in err # both named, in one message + + +def test_a_probe_needs_a_model(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + no_model = live[:live.index("--model")] + live[live.index("--model") + 2:] + args = build_parser().parse_args(no_model) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "--model is required" in capsys.readouterr().err + + +def test_a_saved_probe_must_match_the_commit_it_is_read_against(tmp_path): + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["sha"] = "a" * 40 + check_saved_reach(reach, "toy", "claude-code", "a" * 40) # the same commit is accepted + check_saved_reach(reach, "toy", "claude-code", None) # a tree without a commit cannot be compared + with pytest.raises(ReachError, match="probed at aaaaaaaaaaaa, the source is at bbbbbbbbbbbb"): + check_saved_reach(reach, "toy", "claude-code", "b" * 40) From 0fe06ccbb07ba83196d136feaa5c31a222de4fe6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:19:02 -0700 Subject: [PATCH 04/15] docs(schema): dokimi_method is set for every source the probe recorded Signed-off-by: Nikolay Petrov --- dokimi/dokimi/schemas/effective_instructions.v1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/schemas/effective_instructions.v1.json b/dokimi/dokimi/schemas/effective_instructions.v1.json index bf81c3a69..7e79f6066 100644 --- a/dokimi/dokimi/schemas/effective_instructions.v1.json +++ b/dokimi/dokimi/schemas/effective_instructions.v1.json @@ -74,7 +74,7 @@ "expected": {"$ref": "#/$defs/tristate", "description": "Budget's start-up rules: true for start, false for off, true for a lazy file whose subtree the run works in. unknown without Claude Code's rules."}, "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, "observation_method": {"enum": ["runtime_event", "prompt_capture", "hook", "unknown"], "description": "The thread vocabulary's method; unknown for every source, since no runtime event backs the probe."}, - "dokimi_method": {"enum": ["marker_self_report", null], "description": "Set when the probe saw a marker token for the source; null for a source it did not probe."}, + "dokimi_method": {"enum": ["marker_self_report", null], "description": "Set when the probe recorded the source in this run, whether or not a marker token came back; null for a source it did not probe."}, "states": { "type": "object", "required": ["accessed", "selected", "loaded", "followed"], From 7968811e5c603ef54e14d10c0b6c6550e2e29439 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:11:54 -0700 Subject: [PATCH 05/15] fix(repo): receipt labels, lazy scope, skill listings, saved-probe fields - The receipt's tree label keeps its pinned revision as the label's own hex suffix, and a local directory is cut to its basename, so an @ in a parent directory no longer reaches a shared receipt. - A lazy expectation is given only for a nested AGENTS/CLAUDE file that budget lists under its own path. A link or an import is unknown, since budget's flattened when does not say where it is activated. - A start-up skill listing is a source (observed unknown). Skill bodies, which load on demand, stay out of the receipt. - repo reach --from exits 2 with a message when the saved probe lacks salt or generated_at (receipt) or summary.matrix (matrix), instead of a traceback. Thread: PRRT_kwDOVBg7g86yWZRT Thread: PRRT_kwDOVBg7g86yWaSw Thread: PRRT_kwDOVBg7g86yWZRX Thread: PRRT_kwDOVBg7g86yWaS_ Thread: PRRT_kwDOVBg7g86yWZRQ Thread: PRRT_kwDOVBg7g86yWaSc Thread: PRRT_kwDOVBg7g86yWaS7 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo.py | 5 ++- dokimi/dokimi/repos/reach.py | 7 ++++ dokimi/dokimi/repos/receipt.py | 36 ++++++++++++++------ dokimi/tests/test_repo_receipt.py | 56 +++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 5f3f7d596..d410af2c5 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -277,7 +277,7 @@ def _reach_from_saved(args: argparse.Namespace) -> None: from dokimi.repos import ManifestError, RepoSource, SourceError, open_source from dokimi.repos.harness_profile import ProfileError, load_profile from dokimi.repos.receipt import check_saved_reach - from dokimi.repos.reach import ReachError, format_matrix, load_reach + from dokimi.repos.reach import ReachError, format_matrix, load_reach, saved_gaps from dokimi.repos.source import SourceArgumentError try: # the arguments and the saved file as typed: a usage error @@ -287,6 +287,9 @@ def _reach_from_saved(args: argparse.Namespace) -> None: profile = load_profile(args.harness) source = RepoSource.parse(args.source) reach = load_reach(Path(args.from_file)) + gaps = saved_gaps(reach, args.format) + if gaps: + raise ReachError(f"{args.from_file} has no {', '.join(gaps)}, which --format {args.format} needs") except (SourceError, ProfileError, ReachError, OSError, ValueError) as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 599b48451..7c2b85e52 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -405,6 +405,13 @@ def _matrix(records: list[dict[str, Any]]) -> dict[str, dict[str, str]]: return matrix +def saved_gaps(payload: dict[str, Any], fmt: str) -> list[str]: + """The fields a saved probe must carry to be rendered as ``fmt``: the schema leaves them optional.""" + if fmt == "receipt": + return [key for key in ("salt", "generated_at") if key not in payload] + return [] if "matrix" in payload["summary"] else ["summary.matrix"] + + def format_matrix(payload: dict[str, Any]) -> str: """A plain-text table: one row per guidance file, one column per scenario.""" short = {"loaded": "L", "truncated": "T", "fetched": "F", "not_loaded": ".", "error": "E"} diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index 1c7075deb..fb3d1629e 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -13,6 +13,7 @@ from __future__ import annotations import hashlib +import re from dataclasses import dataclass from pathlib import PurePosixPath, PureWindowsPath from typing import Any @@ -35,7 +36,8 @@ _LOADED: dict[str | None, bool] = {"loaded": True, "not_loaded": False} DETAIL = ("the model is asked to report the marker tokens it can see; the request sent to the provider is not " "captured, so a reported token is evidence of loading, not a copy of the bytes it received") -NOT_PROBED = "start-up imports and skills from repo_budget are not probed by reach; their observed state is unknown" +NOT_PROBED = ("start-up imports and skill listings from repo_budget are not probed by reach; their observed state is " + "unknown, and skill bodies (loaded on demand) are not listed") @dataclass(frozen=True) @@ -43,6 +45,7 @@ class _Run: row: dict[str, Any] # the reach scenario row records: dict[tuple[str, str], dict[str, Any]] # (file, scenario id) -> record when_of: dict[str, str] # canonical path -> budget's when + own: frozenset[str] # guidance files budget lists under their own path (not a link to another file) digests: dict[str, str | None] workdir: str # the directory the run works in rules: bool # budget's rules apply (Claude Code) @@ -53,13 +56,14 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) harness = reach["harness"] rules = harness["name"] == "claude-code" # budget's start-up rules are Claude Code's when_of = _budget_when(reach, budget) if rules else {} + own = _own_guidance(reach, budget) start_up = [p for p, when in when_of.items() if when in ("start", "off") and p not in _known(reach)] digests = source_digests(tree, [p for p, _ in _probed(reach)] + start_up) records = {(r["file"], r["scenario"]): r for r in reach["records"]} runtime = {"name": harness["name"], "version": harness["version"]} receipts = [] for row in reach["scenarios"]: - run = _Run(row, records, when_of, digests, _workdir(row), rules) + run = _Run(row, records, when_of, own, digests, _workdir(row), rules) receipts.append(_receipt(reach, run, runtime, start_up)) return { "schema_version": SCHEMA_VERSION, @@ -73,14 +77,17 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) } +_REV_SUFFIX = re.compile(r"@[0-9a-f]{7,40}$") # the pinned revision a git source's label ends with + + def _shareable_tree(label: str) -> str: - """The tree label with a local directory cut to its last component, so no home path reaches a shared receipt.""" - name, at, sha = label.rpartition("@") - if not at: - name, sha = label, "" + """The tree label with a local directory cut to its last component, so no home path reaches a shared receipt. + The revision is the label's own hex suffix, so an ``@`` in a parent directory is part of the path.""" + match = _REV_SUFFIX.search(label) + name, rev = (label[: match.start()], match.group()) if match else (label, "") if not (PureWindowsPath(name).drive or PurePosixPath(name).is_absolute() or name.startswith("~")): return label - return (PurePosixPath(name.replace("\\", "/")).name or "tree") + at + sha + return (PurePosixPath(name.replace("\\", "/")).name or "tree") + rev def check_saved_reach(reach: dict[str, Any], repo_id: str, harness_name: str, tree_sha: str | None) -> None: @@ -122,7 +129,13 @@ def _budget_when(reach: dict[str, Any], budget: dict[str, Any]) -> dict[str, str """budget's ``when`` of each guidance file and import, keyed by the file the path resolves to.""" target_of = {alias: item["path"] for item in reach["files"] for alias in item.get("aliases", [])} return {target_of.get(e["path"], e["path"]): e["when"] - for e in budget["entries"] if e["kind"] in ("guidance", "import")} + for e in budget["entries"] if e["kind"] in ("guidance", "import", "skill listing")} + + +def _own_guidance(reach: dict[str, Any], budget: dict[str, Any]) -> frozenset[str]: + """Guidance files budget lists under their own path: a link or an import has no subtree of its own.""" + aliases = {alias for item in reach["files"] for alias in item.get("aliases", [])} + return frozenset(e["path"] for e in budget["entries"] if e["kind"] == "guidance" and e["path"] not in aliases) def _dir(path: str) -> str: @@ -192,8 +205,10 @@ def _expected(target: str, when: str | None, run: _Run) -> bool | str: return UNKNOWN if when in ("start", "off"): return when == "start" - if when == "lazy": + if when == "lazy": # budget's flattened when cannot say where a link or an import is activated: only a subtree file d = _dir(target) + if target not in run.own or not d or target.rsplit("/", 1)[-1] not in SUBTREE_NAMES: + return UNKNOWN return run.workdir == d or run.workdir.startswith(d + "/") return UNKNOWN @@ -234,7 +249,8 @@ def _global_unknowns(name: str, rules: bool) -> list[str]: out = [ "followed is not observed: no marker shows what the model did with a loaded source", "observation_method is unknown: no prompt is captured; dokimi_method names the marker self-report instead", - "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", + "accessed is observed only where the harness reports a read event; " + "a shell read it does not normalise is not seen", "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", "no load timestamps: the probe does not record when a source entered the request", "the order in which sources entered the request is not observed; precedence_observed lists displacements only", diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index 6c5fea595..e1ee42553 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -172,6 +172,9 @@ def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): ("~alice/toy", "toy"), ("https://example.com/org/toy.git@4ff761e2d943", "https://example.com/org/toy.git@4ff761e2d943"), ("toy@4ff761e2d943", "toy@4ff761e2d943"), + ("/home/alice@corp/src/toy", "toy"), + ("/home/alice@corp.com/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("C:\\Users\\alice@corp\\toy", "toy"), ]) def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) @@ -321,3 +324,56 @@ def test_a_saved_probe_must_match_the_commit_it_is_read_against(tmp_path): check_saved_reach(reach, "toy", "claude-code", None) # a tree without a commit cannot be compared with pytest.raises(ReachError, match="probed at aaaaaaaaaaaa, the source is at bbbbbbbbbbbb"): check_saved_reach(reach, "toy", "claude-code", "b" * 40) + + +def test_a_lazy_guidance_is_expected_only_in_the_subtree_it_activates(tmp_path): + tree = _tree(tmp_path, {"shared.md": "x\n", "sub/AGENTS.md": "s\n"}) + records = [_record(f, s, "not_loaded") for f in ("shared.md", "sub/AGENTS.md") for s in ("cwd=.", "read=sub")] + files = [{"path": "shared.md", "head": "RC000001", "tail": "RC000002", "aliases": ["sub/CLAUDE.md"]}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + budget = {"entries": [ # sub/CLAUDE.md is a link to shared.md: budget's lazy is the link's activation, not shared.md's + {"path": "sub/CLAUDE.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW], files=files), budget, tree) + root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") + assert root["sub/CLAUDE.md"]["expected"] == "unknown" and sub["sub/CLAUDE.md"]["expected"] == "unknown" + assert root["shared.md"]["expected"] == "unknown" and sub["shared.md"]["expected"] == "unknown" + assert root["sub/AGENTS.md"]["expected"] is False and sub["sub/AGENTS.md"]["expected"] is True + + +def test_a_start_up_skill_listing_is_a_source_and_its_body_is_not(tmp_path): + tree = _tree(tmp_path, {"CLAUDE.md": "c\n", ".claude/skills/fmt/SKILL.md": "---\nname: fmt\n---\n"}) + budget = {"entries": [ + {"path": "CLAUDE.md", "kind": "guidance", "when": "start", "chars": 10}, + {"path": ".claude/skills/fmt/SKILL.md", "kind": "skill listing", "when": "start", "chars": 10}, + {"path": ".claude/skills/fmt/SKILL.md", "kind": "skill body", "when": "on demand", "chars": 10}]} + files = [{"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + receipt = build_receipt(_reach([_record("CLAUDE.md", "cwd=.", "loaded")], [ROOT_ROW], files=files), budget, tree) + sources = receipt["receipts"][0]["sources"] + skill = [s for s in sources if s["path"] == ".claude/skills/fmt/SKILL.md"] + assert len(skill) == 1 and skill[0]["expected"] is True and skill[0]["observed"] == "unknown" + assert skill[0]["sha256"] is not None + + +@pytest.mark.parametrize("fmt, drop, key", [ + ("receipt", "salt", "salt"), + ("receipt", "generated_at", "generated_at"), + ("matrix", "matrix", "summary.matrix"), +]) +def test_a_schema_valid_probe_without_what_its_format_needs_is_a_usage_error(tmp_path, monkeypatch, capsys, + fmt, drop, key): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + if drop == "matrix": + del data["summary"]["matrix"] + else: + del data[drop] + bad = tmp_path / "partial.json" + bad.write_text(json.dumps(data), encoding="utf-8") + argv = _from_argv(live, "--from", str(bad), *(["--format", "receipt"] if fmt == "receipt" else [])) + args = build_parser().parse_args(argv) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and key in err and "Traceback" not in err From 7411bac4d5f88bb58d16d98ed3bdeb4c02ee5d5e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:11:54 -0700 Subject: [PATCH 06/15] =?UTF-8?q?docs(repo):=20make=20=C2=A77.1=20receipt?= =?UTF-8?q?=20limits=20match=20the=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A saved probe is converted against the source as it is now: the receipt is the same only for unchanged source contents and harness configuration. - states.loaded is unknown for a fetched source, not false. - model is the --model string, not the harness's name or version. - Shell reads are normalised only where the harness reports read events (Codex and Copilot, §7); the rest are not seen. - Start-up skill listings are included; skill bodies are not. The not-probed notice is a receipt-level unknown. Thread: PRRT_kwDOVBg7g86yWZRF Thread: PRRT_kwDOVBg7g86yWZRH Thread: PRRT_kwDOVBg7g86yWZRM Thread: PRRT_kwDOVBg7g86yWZRN Thread: PRRT_kwDOVBg7g86yWZRO Thread: PRRT_kwDOVBg7g86yWaS7 Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 38 +++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e4952c75d..e69b02956 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1056,12 +1056,15 @@ in `--out`, a receipt in the shape of `effective-instructions/v0` (agentsmd/agents.md#213). It is built from the reach payload and from the start-up chain of `repo budget` (`repos/budget.py`). It makes no LLM call. `repo reach --from --format receipt` builds the same -receipt from a saved probe without running the harness: the source is read -only to check that the file is for the same repository, harness and commit -(a different one is a usage error, exit 2). The receipt is written beside the -saved file. The probe-only flags (`--model`, `--budget`, `--scenarios`, `--only` -and the rest) cannot be combined with `--from`. The schema is -`schemas/effective_instructions.v1.json`; the code is `repos/receipt.py`. +receipt from a saved probe without running the harness. The source is read +again: the file must be for the same repository, harness and commit (a +different one is a usage error, exit 2), and `sha256` and the expectations are +computed from the source as it is at conversion time. So the two receipts are +the same only for unchanged source contents and harness configuration. The +receipt is written beside the saved file. The probe-only flags (`--model`, +`--budget`, `--scenarios`, `--only` and the rest) cannot be combined with +`--from`. The schema is `schemas/effective_instructions.v1.json`; the code is +`repos/receipt.py`. One receipt is written per reach scenario, with `run_id` `:`. Each source carries: @@ -1075,13 +1078,14 @@ Each source carries: | `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | | `observation_method` | | `unknown` for every source: the thread vocabulary's method, since no runtime event, prompt capture or hook backs the probe | | `dokimi_method` | | `marker_self_report` for a source the probe recorded; null for a start-up file budget found and reach did not probe | -| `states.accessed` | the Read tool | `true` if a Read-tool read was seen, `false` with tools off, else `unknown` | +| `states.accessed` | the harness's read events | `true` if the harness reported a read of the file (the Read tool; Codex and Copilot also report supported shell reads, §7), `false` with tools off, else `unknown` | | `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | -| `states.loaded` | the class | `true` for loaded, `false` for not_loaded, else `unknown`; a fetched file is not loaded | +| `states.loaded` | the class | `true` for loaded, `false` for not_loaded, `unknown` for fetched, truncated and error: a tool read is not proof that start-up loading did not happen | | `states.followed` | | always `unknown` (the schema's `const`) | -`runtime` and `model` are the harness's own name and version, as reach records -them. `observation.method` is `unknown` and `observation.dokimi_method` is +`runtime` is the harness's name and version, as reach records them. `model` is +the string passed with `--model`, not the harness's name or a resolved model +version. `observation.method` is `unknown` and `observation.dokimi_method` is `marker_self_report`: the model reports the tokens it can see, and the request sent to the provider is not captured. A reported token is evidence of loading, not a copy of the bytes received. The name is not `prompt_capture`, because @@ -1097,6 +1101,7 @@ and the receipt says so. | `off` | `false` | a root CLAUDE\*.md turns AGENTS.md off | | `lazy`, run directory is the file's directory or inside it | `true` | loaded when the work reaches that directory | | `lazy`, run directory outside it | `false` | not reached by this run | +| `lazy` on a link or an import, or at the root | `unknown` | budget's flattened `when` does not say which directory activates the file, so only a nested AGENTS/CLAUDE file in its own subtree gets a verdict | | `on demand` or absent | `unknown` | budget does not say | **Precedence.** `precedence_observed` lists a displacement only: a root @@ -1112,16 +1117,19 @@ reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as observations of that run. It cannot show: - that an instruction was followed (`followed` is always `unknown`); -- a read through a shell (`cat`, `rg`, `grep`), which the `Read` tool check - does not see; +- a read through a shell that the harness does not normalise into a read + event (`cat`, `rg`, `grep` under Claude Code; Codex and Copilot normalise + only the shell commands §7 lists); - when a source entered the request, or in what order (no timestamps); -- start-up imports and skills from budget, which reach does not probe: they - appear with `observed: unknown` and a run-level unknown saying so; +- start-up imports and skill listings from budget, which reach does not probe: + they appear with `observed: unknown`. A skill body is loaded on demand, so it + is not listed; the receipt-level unknown says both; - anything a model under-reports; a prose-heavy answer is flagged. Each run lists its own `unknowns` (no usable answer, a source with one token reported, prose above the limit, unseeded tokens). The receipt-level `unknowns` -repeat the limits above. +repeat the limits above and carry the notice that start-up imports and skill +listings are not probed. ## 8. L0 scan (`repo_readiness.v1`) From c4d78e2b1254eb34b30183e01b0a35c06d949546 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:41:51 -0700 Subject: [PATCH 07/15] fix(repo): refuse a receipt rebuilt from an unpinned source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit repo reach --from --format receipt rebuilt sha256 and expectations from the source as it is now. A working tree can change after the probe, and a probe of one has no commit to check against, so the receipt mixed old observations with current digests. Refuse it with exit 2 unless the source is pinned and the saved probe was made at a commit. The matrix is unchanged. Tests: unpinned and edited, pinned probe read against a working tree, and the pinned round trip. docs §7.1 and the changelog say the limit. Signed-off-by: Nikolay Petrov --- ...napetrov_effective-instructions-receipt.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 14 ++-- dokimi/dokimi/commands/repo.py | 4 +- dokimi/dokimi/repos/receipt.py | 12 ++++ dokimi/tests/test_repo_receipt.py | 64 ++++++++++++++++++- 5 files changed, 88 insertions(+), 8 deletions(-) diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md index ef71bd50c..b42d783c0 100644 --- a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source; both must be at a commit (a working tree is a usage error, exit 2). `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e69b02956..d0045cc37 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1058,10 +1058,16 @@ start-up chain of `repo budget` (`repos/budget.py`). It makes no LLM call. `repo reach --from --format receipt` builds the same receipt from a saved probe without running the harness. The source is read again: the file must be for the same repository, harness and commit (a -different one is a usage error, exit 2), and `sha256` and the expectations are -computed from the source as it is at conversion time. So the two receipts are -the same only for unchanged source contents and harness configuration. The -receipt is written beside the saved file. The probe-only flags (`--model`, +different one is a usage error, exit 2). A receipt also needs a pinned source +(`@`) and a saved probe made at a commit: a working tree, or a probe +of one (`repo.sha` null), is a usage error (exit 2), because `sha256` and the +expectations are computed from the source as it is at conversion time and a +working tree can change after the probe. The matrix (without `--format +receipt`) still accepts an unpinned source. The harness configuration is not +pinned: `expected` comes from the harness profile as it is at conversion time, +and its guidance names and import depth are not compared with the probe's, so +a receipt is the same only for an unchanged profile. The receipt is written +beside the saved file. The probe-only flags (`--model`, `--budget`, `--scenarios`, `--only` and the rest) cannot be combined with `--from`. The schema is `schemas/effective_instructions.v1.json`; the code is `repos/receipt.py`. diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index d410af2c5..d3c84ff72 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -276,7 +276,7 @@ def _reach_from_saved(args: argparse.Namespace) -> None: """The matrix or the receipt of a saved reach.json, checked against the source; no harness runs.""" from dokimi.repos import ManifestError, RepoSource, SourceError, open_source from dokimi.repos.harness_profile import ProfileError, load_profile - from dokimi.repos.receipt import check_saved_reach + from dokimi.repos.receipt import check_receipt_provenance, check_saved_reach from dokimi.repos.reach import ReachError, format_matrix, load_reach, saved_gaps from dokimi.repos.source import SourceArgumentError @@ -304,6 +304,8 @@ def _reach_from_saved(args: argparse.Namespace) -> None: try: # the manifest and the file's provenance: a usage error; a failed git read is a fetch failure (1) manifest = _load_manifest(tree, args, guidance_default=True) check_saved_reach(reach, manifest.id, profile.name, getattr(tree, "sha", None)) + if args.format == "receipt": + check_receipt_provenance(reach, getattr(tree, "sha", None)) except (SourceError, ManifestError, ReachError) as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(_tree_read_exit_code(exc)) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index fb3d1629e..e54d2abc5 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -101,6 +101,18 @@ def check_saved_reach(reach: dict[str, Any], repo_id: str, harness_name: str, tr raise ReachError(f"reach.json was probed at {probed[:12]}, the source is at {tree_sha[:12]}") +def check_receipt_provenance(reach: dict[str, Any], tree_sha: str | None) -> None: + """Refuse a receipt rebuilt from a saved probe unless both sides are at a commit. The digests and expectations + are read from the source now, so they describe the probed text only when that text is pinned: a working tree + may have changed since the probe, and a probe of a working tree has no commit to check the source against.""" + if not tree_sha: + raise ReachError("--format receipt needs a pinned source: a working tree can change after the probe, so give " + "the repository at a revision (@)") + if not reach["repo"].get("sha"): + raise ReachError("reach.json was probed from a working tree (repo.sha is null), so its receipt cannot be " + "tied to a commit; probe a pinned source to get a receipt") + + def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" out: dict[str, str | None] = {} diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index e1ee42553..cb9ec3e47 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -7,13 +7,14 @@ import hashlib import json +import subprocess import pytest from dokimi.artifacts import ArtifactValidationError, validate_artifact from dokimi.cli import build_parser from dokimi.repos import WorkTree -from dokimi.repos.receipt import build_receipt, check_saved_reach, source_digests +from dokimi.repos.receipt import build_receipt, check_receipt_provenance, check_saved_reach, source_digests from dokimi.repos.reach import ReachError, format_matrix, load_reach from tests.test_repo_reach import _profile, _repo @@ -239,8 +240,17 @@ def _saved(tmp_path) -> str: return str(tmp_path / "out" / "reach.json") +def _pinned(live: list[str]) -> list[str]: + """The same argv with its source committed and pinned to HEAD: a receipt needs a commit on both sides.""" + root = live[2] + for args in (["init", "-q"], ["add", "-A"], + ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--no-gpg-sign", "-m", "x"]): + subprocess.run(["git", "-C", root, *args], check=True, capture_output=True) + return [live[0], live[1], f"{root}@HEAD", *live[3:]] + + def test_from_a_saved_reach_gives_the_receipt_of_the_live_run(tmp_path, monkeypatch, capsys): - live = _end_to_end_args(tmp_path, monkeypatch, "--format", "receipt") + live = _pinned(_end_to_end_args(tmp_path, monkeypatch, "--format", "receipt")) args = build_parser().parse_args(live) args.func(args) live_receipt = json.loads(capsys.readouterr().out) @@ -326,6 +336,56 @@ def test_a_saved_probe_must_match_the_commit_it_is_read_against(tmp_path): check_saved_reach(reach, "toy", "claude-code", "b" * 40) +def test_a_receipt_needs_a_pinned_probe_and_a_pinned_source(tmp_path): + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["sha"] = "a" * 40 + check_receipt_provenance(reach, "a" * 40) # both at the same commit: the digests describe the probed text + with pytest.raises(ReachError, match="pinned"): # a working tree can change after the probe + check_receipt_provenance(reach, None) + reach["repo"]["sha"] = None # probed from a working tree: its observations have no commit to check against + with pytest.raises(ReachError, match="pinned"): + check_receipt_provenance(reach, "a" * 40) + + +def test_from_a_receipt_refuses_a_pinned_probe_read_against_a_working_tree(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) # probed at HEAD: repo.sha is set + capsys.readouterr() + working = _from_argv(live, "--from", _saved(tmp_path), "--format", "receipt") + working[2] = str(tmp_path / "repo") # the same directory, read as a working tree (no revision) + args = build_parser().parse_args(working) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "--format receipt needs a pinned source" in capsys.readouterr().err + + +def test_from_a_receipt_refuses_an_unpinned_source_where_the_matrix_runs(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) # an unpinned probe: repo.sha is null + capsys.readouterr() + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "--format receipt needs a pinned source" in err + assert not (tmp_path / "out" / "effective_instructions.json").exists() + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path))) # the matrix needs no commit + args.func(args) + assert capsys.readouterr().out == format_matrix(load_reach(tmp_path / "out" / "reach.json")) + "\n" + + +def test_from_a_receipt_refuses_an_unpinned_probe_read_against_a_pinned_source(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) # probed from a working tree: repo.sha null + capsys.readouterr() + pinned = _pinned(live) # the same repository, committed, now read at HEAD + args = build_parser().parse_args(_from_argv(pinned, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "probed from a working tree (repo.sha is null)" in err + + def test_a_lazy_guidance_is_expected_only_in_the_subtree_it_activates(tmp_path): tree = _tree(tmp_path, {"shared.md": "x\n", "sub/AGENTS.md": "s\n"}) records = [_record(f, s, "not_loaded") for f in ("shared.md", "sub/AGENTS.md") for s in ("cwd=.", "read=sub")] From 5c2c6da6d82db405c99d3ea96c855d3a90d5dc51 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:23:23 -0700 Subject: [PATCH 08/15] fix(repo): --from never overwrites the probe it reads - repo reach --from refuses a receipt path that would replace the saved probe (usage error; also for renamed or symlinked paths) - a live receipt that cannot be written is a runtime failure (exit 1), not a traceback; a receipt path that resolves to reach.json is refused - a probe saved without `unknown` tokens renders its matrix - a lazy guidance file imported by a nested file is expected only where that file is read - contract section 7.1 and the changelog: the pinned-source rule is for --from, collision and write-failure exit codes, one aggregate artifact Signed-off-by: Nikolay Petrov --- ...napetrov_effective-instructions-receipt.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 34 +++++---- dokimi/dokimi/commands/repo.py | 27 +++++++- dokimi/dokimi/repos/reach.py | 2 +- dokimi/dokimi/repos/receipt.py | 12 ++-- dokimi/tests/test_repo_receipt.py | 69 +++++++++++++++++++ 6 files changed, 124 insertions(+), 22 deletions(-) diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md index b42d783c0..96b4cd668 100644 --- a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source; both must be at a commit (a working tree is a usage error, exit 2). `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source; with `--from`, both must be at a commit (a working tree is a usage error, exit 2), and the receipt may not be written over the saved file. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d0045cc37..71477fb2c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1058,33 +1058,37 @@ start-up chain of `repo budget` (`repos/budget.py`). It makes no LLM call. `repo reach --from --format receipt` builds the same receipt from a saved probe without running the harness. The source is read again: the file must be for the same repository, harness and commit (a -different one is a usage error, exit 2). A receipt also needs a pinned source -(`@`) and a saved probe made at a commit: a working tree, or a probe -of one (`repo.sha` null), is a usage error (exit 2), because `sha256` and the -expectations are computed from the source as it is at conversion time and a -working tree can change after the probe. The matrix (without `--format -receipt`) still accepts an unpinned source. The harness configuration is not +different one is a usage error, exit 2). With `--from`, a receipt also needs a +pinned source (`@`) and a saved probe made at a commit: a working +tree, or a probe of one (`repo.sha` null), is a usage error (exit 2), because +`sha256` and the expectations are computed from the source as it is at +conversion time and a working tree can change after the probe. A live +`repo reach --format receipt` run is not held to this: it builds the receipt +from the source it probes. The matrix (without `--format receipt`) still +accepts an unpinned source. The harness configuration is not pinned: `expected` comes from the harness profile as it is at conversion time, and its guidance names and import depth are not compared with the probe's, so a receipt is the same only for an unchanged profile. The receipt is written -beside the saved file. The probe-only flags (`--model`, +beside the saved file; a name that resolves to the saved probe is a usage error +(exit 2), and a write that fails is a runtime failure (exit 1). The probe-only flags (`--model`, `--budget`, `--scenarios`, `--only` and the rest) cannot be combined with `--from`. The schema is `schemas/effective_instructions.v1.json`; the code is `repos/receipt.py`. -One receipt is written per reach scenario, with `run_id` `:`. +One artifact is written to `effective_instructions.json`, with one entry in its +`receipts` array per reach scenario and `run_id` `:`. Each source carries: | Receipt field | From | Meaning | |---|---|---| | `path`, `alias_of` | reach `files`, `aliases` | a link is `adapter` scope and takes its target's tokens | | `sha256` | the text dokimi reads (LF-normalised UTF-8) | null when the file is not readable as text | -| `scope` | path | `repo` at the root or in `.claude/`; `subtree` for a nested AGENTS.md, CLAUDE.md or CLAUDE.local.md; else `unknown` | +| `scope` | path | `repo` for a root-level file and for `.claude/CLAUDE.md` and `.claude/AGENTS.md` only; `subtree` for a nested AGENTS.md, CLAUDE.md or CLAUDE.local.md (`.claude/CLAUDE.local.md` included); else `unknown` (a skill, for one) | | `expected` | budget `when` | see the table below | | `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | | `observation_method` | | `unknown` for every source: the thread vocabulary's method, since no runtime event, prompt capture or hook backs the probe | | `dokimi_method` | | `marker_self_report` for a source the probe recorded; null for a start-up file budget found and reach did not probe | -| `states.accessed` | the harness's read events | `true` if the harness reported a read of the file (the Read tool; Codex and Copilot also report supported shell reads, §7), `false` with tools off, else `unknown` | +| `states.accessed` | the harness's read events | `true` if the harness reported a read of the file (the Read tool; Codex and Copilot also report supported shell reads, §7); for a probed source `false` with tools off, else `unknown`; always `unknown` for a start-up file budget found and reach did not probe | | `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | | `states.loaded` | the class | `true` for loaded, `false` for not_loaded, `unknown` for fetched, truncated and error: a tool read is not proof that start-up loading did not happen | | `states.followed` | | always `unknown` (the schema's `const`) | @@ -1107,16 +1111,18 @@ and the receipt says so. | `off` | `false` | a root CLAUDE\*.md turns AGENTS.md off | | `lazy`, run directory is the file's directory or inside it | `true` | loaded when the work reaches that directory | | `lazy`, run directory outside it | `false` | not reached by this run | -| `lazy` on a link or an import, or at the root | `unknown` | budget's flattened `when` does not say which directory activates the file, so only a nested AGENTS/CLAUDE file in its own subtree gets a verdict | +| `lazy` on a link, an import, a root file, or a file another guidance file `@`-imports | `unknown` | budget's flattened `when` does not say which directory activates the file, so only a nested AGENTS/CLAUDE file that budget lists under its own path, and that no guidance file imports, gets a verdict | | `on demand` or absent | `unknown` | budget does not say | **Precedence.** `precedence_observed` lists a displacement only: a root CLAUDE\*.md (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md`) observed loaded in a run, with an `off` AGENTS.md observed not_loaded in that run. It does not give an order of entry. The displacement follows budget's rule as -written (`repos/budget.py`: every AGENTS.md is `off` while a root CLAUDE\*.md -exists, nested ones included), so nested AGENTS.md files appear too. If budget's -rule changes, the receipt follows it. +written (`repos/budget.py`): an AGENTS.md is `off` while a root CLAUDE\*.md +exists, nested ones included, unless a root CLAUDE\* file aliases it or +`@`-imports it (`start`), or a nested CLAUDE\* file `@`-imports it (`lazy`). So +nested AGENTS.md files can appear, and an AGENTS.md at the root is not always +`off`. If budget's rule changes, the receipt follows it. **What the marker method can and cannot show.** It shows which tokens were reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index d3c84ff72..becaf925b 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -290,6 +290,9 @@ def _reach_from_saved(args: argparse.Namespace) -> None: gaps = saved_gaps(reach, args.format) if gaps: raise ReachError(f"{args.from_file} has no {', '.join(gaps)}, which --format {args.format} needs") + if args.format == "receipt" and _receipt_would_overwrite(Path(args.from_file), _receipt_path(args.from_file)): + raise ReachError(f"the receipt would replace {args.from_file}, the probe it is read from: " + "rename the saved file") except (SourceError, ProfileError, ReachError, OSError, ValueError) as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) @@ -311,7 +314,7 @@ def _reach_from_saved(args: argparse.Namespace) -> None: sys.exit(_tree_read_exit_code(exc)) if args.format == "receipt": _print_receipt(tree, profile, reach, Path(args.from_file).parent) - print(f"wrote {Path(args.from_file).parent / 'effective_instructions.json'}", file=sys.stderr) + print(f"wrote {_receipt_path(args.from_file)}", file=sys.stderr) else: print(format_matrix(reach)) if any(s["error"] for s in reach["scenarios"]): @@ -379,6 +382,10 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: print(f"{len(files)} guidance files, {planned_count} scenarios, " f"at most ${args.budget:g} each", file=sys.stderr) out_dir = Path(args.out or "reach-out") + if args.format == "receipt" and _receipt_would_overwrite(out_dir / "reach.json", out_dir / RECEIPT_NAME): + print(f"✗ {out_dir / RECEIPT_NAME} is the probe itself (a link to {out_dir / 'reach.json'}): " + "remove the link", file=sys.stderr) + sys.exit(EXIT_USAGE) try: # the harness runs: a runtime failure payload = run_reach( tree=tree, manifest=manifest, profile=profile, model=args.model, out_dir=out_dir, @@ -407,6 +414,18 @@ def _print_reach_output(tree, profile, payload: dict, out_dir: Path, path: Path, print(f"wrote {path}") +RECEIPT_NAME = "effective_instructions.json" + + +def _receipt_path(from_file: str) -> Path: + return Path(from_file).parent / RECEIPT_NAME + + +def _receipt_would_overwrite(probe: Path, receipt: Path) -> bool: + """A receipt written at the probe's path, directly or through a link, would replace the probe it is built from.""" + return probe.resolve() == receipt.resolve() + + def _print_receipt(tree, profile, payload: dict, out_dir: Path) -> None: """The effective-instructions receipt of a probe: budget's start-up chain, reach's observed states.""" from dokimi.artifacts import save_artifact @@ -414,7 +433,11 @@ def _print_receipt(tree, profile, payload: dict, out_dir: Path) -> None: from dokimi.repos.receipt import build_receipt receipt = build_receipt(payload, budget(tree, profile.guidance_names, profile.import_max_depth), tree) - written = save_artifact("effective_instructions", receipt, out_dir / "effective_instructions.json") + try: # a directory that is not writable, or one already named effective_instructions.json: a runtime failure + written = save_artifact("effective_instructions", receipt, out_dir / RECEIPT_NAME) + except OSError as exc: + print(f"✗ cannot write {out_dir / RECEIPT_NAME}: {exc}", file=sys.stderr) + sys.exit(1) print(json.dumps(written, indent=2)) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 7c2b85e52..5ce97ee87 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -438,7 +438,7 @@ def format_matrix(payload: dict[str, Any]) -> str: if (s.get("prose") or 0) > PROSE_LIMIT: lines.append(f"warning: {s['id']} answered with {s['prose']} words besides tokens; " "read its trace before trusting its not_loaded cells") - if s["unknown"]: + if s.get("unknown"): # schema-optional: a scenario saved without it reports no unknown tokens lines.append(f"unknown tokens in {s['id']}: {', '.join(s['unknown'])}") return "\n".join(lines) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index e54d2abc5..87546a06a 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -18,6 +18,7 @@ from pathlib import PurePosixPath, PureWindowsPath from typing import Any +from .budget import _imports, _target from .reach import PROSE_LIMIT, ReachError from .source import RepoTree @@ -56,7 +57,7 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) harness = reach["harness"] rules = harness["name"] == "claude-code" # budget's start-up rules are Claude Code's when_of = _budget_when(reach, budget) if rules else {} - own = _own_guidance(reach, budget) + own = _own_guidance(reach, budget, tree) start_up = [p for p, when in when_of.items() if when in ("start", "off") and p not in _known(reach)] digests = source_digests(tree, [p for p, _ in _probed(reach)] + start_up) records = {(r["file"], r["scenario"]): r for r in reach["records"]} @@ -144,10 +145,13 @@ def _budget_when(reach: dict[str, Any], budget: dict[str, Any]) -> dict[str, str for e in budget["entries"] if e["kind"] in ("guidance", "import", "skill listing")} -def _own_guidance(reach: dict[str, Any], budget: dict[str, Any]) -> frozenset[str]: - """Guidance files budget lists under their own path: a link or an import has no subtree of its own.""" +def _own_guidance(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) -> frozenset[str]: + """Guidance files budget lists under their own path, and that no other guidance file @-imports: a link or an + import is activated where its importer is, so the file has no subtree of its own.""" aliases = {alias for item in reach["files"] for alias in item.get("aliases", [])} - return frozenset(e["path"] for e in budget["entries"] if e["kind"] == "guidance" and e["path"] not in aliases) + guidance = [e["path"] for e in budget["entries"] if e["kind"] == "guidance"] + imported = {_target(tree, p) for g in guidance for p in _imports(tree, [g])[0]} + return frozenset(p for p in guidance if p not in aliases and p not in imported) def _dir(path: str) -> str: diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index cb9ec3e47..443bcb6f1 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -437,3 +437,72 @@ def test_a_schema_valid_probe_without_what_its_format_needs_is_a_usage_error(tmp args.func(args) err = capsys.readouterr().err assert exc.value.code == 2 and key in err and "Traceback" not in err + + +@pytest.mark.parametrize("link", [False, True], ids=["renamed", "symlink"]) +def test_from_never_writes_the_receipt_over_the_probe_it_reads(tmp_path, monkeypatch, capsys, link): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + capsys.readouterr() + saved = tmp_path / "out" / "reach.json" + before = saved.read_text(encoding="utf-8") + receipt = tmp_path / "out" / "effective_instructions.json" + if link: + receipt.symlink_to(saved) + else: + saved.rename(receipt) + saved = receipt + args = build_parser().parse_args(_from_argv(live, "--from", str(saved), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "Traceback" not in capsys.readouterr().err + assert receipt.resolve().read_text(encoding="utf-8") == before + + +def test_a_receipt_that_cannot_be_written_is_a_runtime_failure_not_a_traceback(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + capsys.readouterr() + (tmp_path / "out" / "effective_instructions.json").mkdir() # a directory where the receipt goes + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 1 and "cannot write" in err and "Traceback" not in err + + +def test_a_live_receipt_that_cannot_be_written_is_a_runtime_failure(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch, "--format", "receipt") + (tmp_path / "out").mkdir() + (tmp_path / "out" / "effective_instructions.json").mkdir() + args = build_parser().parse_args(live) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 and "cannot write" in capsys.readouterr().err + + +def test_a_matrix_of_a_probe_saved_without_unknown_tokens_renders(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + for scenario in data["scenarios"]: + scenario.pop("unknown", None) # schema-optional + (tmp_path / "out" / "bare.json").write_text(json.dumps(data), encoding="utf-8") + capsys.readouterr() + args = build_parser().parse_args(_from_argv(live, "--from", str(tmp_path / "out" / "bare.json"))) + args.func(args) + assert capsys.readouterr().out == format_matrix(data) + "\n" + + +def test_a_lazy_guidance_imported_by_a_nested_file_is_expected_only_where_that_file_is(tmp_path): + tree = _tree(tmp_path, {"sub/CLAUDE.md": "@../other/AGENTS.md\n", "other/AGENTS.md": "o\n"}) + records = [_record(f, s, "not_loaded") for f in ("other/AGENTS.md", "sub/CLAUDE.md") for s in ("cwd=.", "read=sub")] + files = [{"path": "other/AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "sub/CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + budget = {"entries": [ # the import activates other/AGENTS.md in sub/, not where the file lies + {"path": "sub/CLAUDE.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "other/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW], files=files), budget, tree) + root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") + assert root["other/AGENTS.md"]["expected"] == "unknown" and sub["other/AGENTS.md"]["expected"] == "unknown" + assert root["sub/CLAUDE.md"]["expected"] is False and sub["sub/CLAUDE.md"]["expected"] is True From a84f717014ad4f86db02c1973aebaafae1e6254a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:53:01 -0700 Subject: [PATCH 09/15] refactor(repo): split the reach helpers under the complexity cap _reach_from_saved and cmd_repo_reach were at complexity 11 over the cap of 10 with no baseline entry. The saved-file checks and the receipt guard move into helpers; no behaviour change. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo.py | 53 ++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index becaf925b..7d9561b4e 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -272,27 +272,37 @@ def _probe_flags_given(args: argparse.Namespace) -> list[str]: return [f"--{k.replace('_', '-')}" for k, default in _PROBE_ONLY.items() if getattr(args, k) != default] +def _saved_reach_inputs(args: argparse.Namespace): + """The harness profile, the source and the saved reach.json of ``--from``, as typed; raises on a usage error.""" + from dokimi.repos import RepoSource + from dokimi.repos.harness_profile import load_profile + from dokimi.repos.reach import ReachError, load_reach, saved_gaps + + given = _probe_flags_given(args) + if given: + raise ReachError(f"--from reads a saved probe, so {', '.join(given)} cannot be used with it") + profile = load_profile(args.harness) + source = RepoSource.parse(args.source) + reach = load_reach(Path(args.from_file)) + gaps = saved_gaps(reach, args.format) + if gaps: + raise ReachError(f"{args.from_file} has no {', '.join(gaps)}, which --format {args.format} needs") + if args.format == "receipt" and _receipt_would_overwrite(Path(args.from_file), _receipt_path(args.from_file)): + raise ReachError(f"the receipt would replace {args.from_file}, the probe it is read from: " + "rename the saved file") + return profile, source, reach + + def _reach_from_saved(args: argparse.Namespace) -> None: """The matrix or the receipt of a saved reach.json, checked against the source; no harness runs.""" - from dokimi.repos import ManifestError, RepoSource, SourceError, open_source - from dokimi.repos.harness_profile import ProfileError, load_profile + from dokimi.repos import ManifestError, SourceError, open_source + from dokimi.repos.harness_profile import ProfileError from dokimi.repos.receipt import check_receipt_provenance, check_saved_reach - from dokimi.repos.reach import ReachError, format_matrix, load_reach, saved_gaps + from dokimi.repos.reach import ReachError, format_matrix from dokimi.repos.source import SourceArgumentError try: # the arguments and the saved file as typed: a usage error - given = _probe_flags_given(args) - if given: - raise ReachError(f"--from reads a saved probe, so {', '.join(given)} cannot be used with it") - profile = load_profile(args.harness) - source = RepoSource.parse(args.source) - reach = load_reach(Path(args.from_file)) - gaps = saved_gaps(reach, args.format) - if gaps: - raise ReachError(f"{args.from_file} has no {', '.join(gaps)}, which --format {args.format} needs") - if args.format == "receipt" and _receipt_would_overwrite(Path(args.from_file), _receipt_path(args.from_file)): - raise ReachError(f"the receipt would replace {args.from_file}, the probe it is read from: " - "rename the saved file") + profile, source, reach = _saved_reach_inputs(args) except (SourceError, ProfileError, ReachError, OSError, ValueError) as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) @@ -331,6 +341,14 @@ def _tree_read_exit_code(exc: Exception) -> int: return 1 if git_failure else EXIT_USAGE +def _refuse_receipt_over_probe(args: argparse.Namespace, out_dir: Path) -> None: + """Exit as a usage error when ``--format receipt`` would write the receipt over the probe it is built from.""" + if args.format == "receipt" and _receipt_would_overwrite(out_dir / "reach.json", out_dir / RECEIPT_NAME): + print(f"✗ {out_dir / RECEIPT_NAME} is the probe itself (a link to {out_dir / 'reach.json'}): " + "remove the link", file=sys.stderr) + sys.exit(EXIT_USAGE) + + def cmd_repo_reach(args: argparse.Namespace) -> None: """L1 reach probe: which guidance files one harness/version/model loads.""" from dokimi.repos import ManifestError, RepoSource, SourceError, open_source @@ -382,10 +400,7 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: print(f"{len(files)} guidance files, {planned_count} scenarios, " f"at most ${args.budget:g} each", file=sys.stderr) out_dir = Path(args.out or "reach-out") - if args.format == "receipt" and _receipt_would_overwrite(out_dir / "reach.json", out_dir / RECEIPT_NAME): - print(f"✗ {out_dir / RECEIPT_NAME} is the probe itself (a link to {out_dir / 'reach.json'}): " - "remove the link", file=sys.stderr) - sys.exit(EXIT_USAGE) + _refuse_receipt_over_probe(args, out_dir) try: # the harness runs: a runtime failure payload = run_reach( tree=tree, manifest=manifest, profile=profile, model=args.model, out_dir=out_dir, From 43492eb48d0318c243ddfcccf715bf1643af5bd7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:33:16 -0800 Subject: [PATCH 10/15] fix(repo): receipt labels, output links and saved matrix cells - the tree label of a receipt cuts every local source (relative and file://) to its last component; only URLs and git@host:path stay whole - a receipt output path that is a link loop is a usage error, not a traceback (live and --from) - a saved reach.json whose matrix has a null row or a non-class cell is a usage error before the matrix renders - a receipt of a working tree says its source is read after the run - tests: accessed states, lazy scope in a deeper and a sibling directory, a nested link, .claude/CLAUDE.md scope Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo.py | 28 +++++++-- dokimi/dokimi/repos/reach.py | 20 ++++++- dokimi/dokimi/repos/receipt.py | 25 ++++++-- dokimi/tests/test_repo_receipt.py | 99 +++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 7d9561b4e..326fbdd4c 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -276,7 +276,7 @@ def _saved_reach_inputs(args: argparse.Namespace): """The harness profile, the source and the saved reach.json of ``--from``, as typed; raises on a usage error.""" from dokimi.repos import RepoSource from dokimi.repos.harness_profile import load_profile - from dokimi.repos.reach import ReachError, load_reach, saved_gaps + from dokimi.repos.reach import ReachError, load_reach, saved_gaps, saved_matrix_problem given = _probe_flags_given(args) if given: @@ -287,6 +287,8 @@ def _saved_reach_inputs(args: argparse.Namespace): gaps = saved_gaps(reach, args.format) if gaps: raise ReachError(f"{args.from_file} has no {', '.join(gaps)}, which --format {args.format} needs") + if args.format == "matrix" and (problem := saved_matrix_problem(reach)): + raise ReachError(f"{args.from_file} cannot be shown as a matrix: {problem}") if args.format == "receipt" and _receipt_would_overwrite(Path(args.from_file), _receipt_path(args.from_file)): raise ReachError(f"the receipt would replace {args.from_file}, the probe it is read from: " "rename the saved file") @@ -342,8 +344,18 @@ def _tree_read_exit_code(exc: Exception) -> int: def _refuse_receipt_over_probe(args: argparse.Namespace, out_dir: Path) -> None: - """Exit as a usage error when ``--format receipt`` would write the receipt over the probe it is built from.""" - if args.format == "receipt" and _receipt_would_overwrite(out_dir / "reach.json", out_dir / RECEIPT_NAME): + """Exit as a usage error when ``--format receipt`` would write the receipt over the probe it is built from, or + when an output path is a link that cannot be resolved (a loop).""" + from dokimi.repos.reach import ReachError + + if args.format != "receipt": + return + try: + overwrite = _receipt_would_overwrite(out_dir / "reach.json", out_dir / RECEIPT_NAME) + except ReachError as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if overwrite: print(f"✗ {out_dir / RECEIPT_NAME} is the probe itself (a link to {out_dir / 'reach.json'}): " "remove the link", file=sys.stderr) sys.exit(EXIT_USAGE) @@ -437,8 +449,14 @@ def _receipt_path(from_file: str) -> Path: def _receipt_would_overwrite(probe: Path, receipt: Path) -> bool: - """A receipt written at the probe's path, directly or through a link, would replace the probe it is built from.""" - return probe.resolve() == receipt.resolve() + """A receipt written at the probe's path, directly or through a link, would replace the probe it is built from. + A link loop cannot be resolved (RuntimeError before Python 3.13, OSError after): a usage error, not a traceback.""" + from dokimi.repos.reach import ReachError + + try: + return probe.resolve() == receipt.resolve() + except (OSError, RuntimeError) as exc: + raise ReachError(f"cannot resolve {receipt}: {exc}") from exc def _print_receipt(tree, profile, payload: dict, out_dir: Path) -> None: diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 5ce97ee87..98a0ea65f 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -412,9 +412,27 @@ def saved_gaps(payload: dict[str, Any], fmt: str) -> list[str]: return [] if "matrix" in payload["summary"] else ["summary.matrix"] +_SHORT = {"loaded": "L", "truncated": "T", "fetched": "F", "not_loaded": ".", "error": "E"} # a cell -> its letter + + +def saved_matrix_problem(payload: dict[str, Any]) -> str | None: + """What keeps a saved ``summary.matrix`` from rendering: a row that is not an object, or a cell that is not one of + the classes the table shows (format_matrix would raise on either). None when every cell can be shown.""" + matrix = payload["summary"]["matrix"] + if not isinstance(matrix, dict): + return "summary.matrix is not an object" + for path, row in matrix.items(): + if not isinstance(row, dict): + return f"summary.matrix[{path!r}] is not an object" + for scenario, cell in row.items(): + if not isinstance(cell, str) or cell not in _SHORT: + return f"summary.matrix[{path!r}][{scenario!r}] is {cell!r}, not a reach class" + return None + + def format_matrix(payload: dict[str, Any]) -> str: """A plain-text table: one row per guidance file, one column per scenario.""" - short = {"loaded": "L", "truncated": "T", "fetched": "F", "not_loaded": ".", "error": "E"} + short = _SHORT ids = [s["id"] for s in payload["scenarios"]] label = {f["path"]: " = ".join([f["path"], ", ".join(f["aliases"])]) if f.get("aliases") else f["path"] for f in payload["files"]} diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index 87546a06a..e55a8b7e5 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -15,7 +15,7 @@ import hashlib import re from dataclasses import dataclass -from pathlib import PurePosixPath, PureWindowsPath +from pathlib import PurePosixPath from typing import Any from .budget import _imports, _target @@ -73,22 +73,31 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) "runtime": runtime, "model": reach["model"], "observation": {"method": OBSERVATION_METHOD, "dokimi_method": DOKIMI_METHOD, "detail": DETAIL}, - "unknowns": _global_unknowns(harness["name"], rules), + "unknowns": _global_unknowns(harness["name"], rules, pinned=bool(getattr(tree, "sha", None))), "receipts": receipts, } _REV_SUFFIX = re.compile(r"@[0-9a-f]{7,40}$") # the pinned revision a git source's label ends with +_URL_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://") # https://, ssh://, file://, ... +_SCP_LIKE = re.compile(r"^[^/\\\s]+@[^/\\\s]+:") # git@host:path def _shareable_tree(label: str) -> str: """The tree label with a local directory cut to its last component, so no home path reaches a shared receipt. - The revision is the label's own hex suffix, so an ``@`` in a parent directory is part of the path.""" + A remote label (any URL scheme but file://, or git@host:path) is kept whole; any other label names a local + directory, absolute or relative, and is cut. The revision is the label's own hex suffix, so an ``@`` in a parent + directory is part of the path.""" match = _REV_SUFFIX.search(label) name, rev = (label[: match.start()], match.group()) if match else (label, "") - if not (PureWindowsPath(name).drive or PurePosixPath(name).is_absolute() or name.startswith("~")): + if name.lower().startswith("file://"): + local = True + else: + local = not (_URL_SCHEME.match(name) or _SCP_LIKE.match(name)) + if not local: return label - return (PurePosixPath(name.replace("\\", "/")).name or "tree") + rev + base = PurePosixPath(name.replace("\\", "/")).name + return (base if base not in ("", ".", "..") else "tree") + rev def check_saved_reach(reach: dict[str, Any], repo_id: str, harness_name: str, tree_sha: str | None) -> None: @@ -261,7 +270,7 @@ def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]]) -> list[st return out -def _global_unknowns(name: str, rules: bool) -> list[str]: +def _global_unknowns(name: str, rules: bool, pinned: bool) -> list[str]: out = [ "followed is not observed: no marker shows what the model did with a loaded source", "observation_method is unknown: no prompt is captured; dokimi_method names the marker self-report instead", @@ -274,4 +283,8 @@ def _global_unknowns(name: str, rules: bool) -> list[str]: ] if not rules: out.append(f"budget's start-up rules are Claude Code's: no source has an expected state for {name}") + if not pinned: + out.append("the source is a working tree: sha256 and expected are read after the harness exits, so an edit " + "made while the run was in flight is not reflected in the observations; pin the source to a " + "revision for a stable receipt") return out diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index 443bcb6f1..19244274c 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -176,6 +176,11 @@ def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): ("/home/alice@corp/src/toy", "toy"), ("/home/alice@corp.com/toy@4ff761e2d943", "toy@4ff761e2d943"), ("C:\\Users\\alice@corp\\toy", "toy"), + ("../alice/private/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("alice/private/toy", "toy"), + ("file:///home/alice/toy.git@4ff761e2d943", "toy.git@4ff761e2d943"), + ("git@github.com:org/toy.git@4ff761e2d943", "git@github.com:org/toy.git@4ff761e2d943"), + ("ssh://git@example.com/org/toy.git@4ff761e2d943", "ssh://git@example.com/org/toy.git@4ff761e2d943"), ]) def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) @@ -186,6 +191,12 @@ def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): assert "alice" not in json.dumps(receipt) +def test_a_working_tree_receipt_says_its_source_is_read_after_the_run(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), {"entries": []}, tree) + assert any("working tree" in u and "pin the source" in u for u in receipt["unknowns"]) + + def test_the_receipt_is_a_valid_artifact_and_a_schema_rejects_a_followed_claim(tmp_path): tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) @@ -212,6 +223,7 @@ def test_cli_receipt_format_prints_and_writes_a_valid_receipt(tmp_path, monkeypa out = capsys.readouterr().out data = json.loads(out) validate_artifact("effective_instructions", data) + assert any("working tree" in u for u in data["unknowns"]) # live on a working tree, unpinned assert json.loads((tmp_path / "out" / "effective_instructions.json").read_text(encoding="utf-8")) == data run = data["receipts"][0] assert run["run_id"] == "s:cwd=." and run["runtime"] == {"name": "claude-code", "version": "9.8.7"} @@ -254,6 +266,7 @@ def test_from_a_saved_reach_gives_the_receipt_of_the_live_run(tmp_path, monkeypa args = build_parser().parse_args(live) args.func(args) live_receipt = json.loads(capsys.readouterr().out) + assert not any("working tree" in u for u in live_receipt["unknowns"]) # pinned: nothing is read after the run args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) args.func(args) assert json.loads(capsys.readouterr().out) == live_receipt @@ -506,3 +519,89 @@ def test_a_lazy_guidance_imported_by_a_nested_file_is_expected_only_where_that_f root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") assert root["other/AGENTS.md"]["expected"] == "unknown" and sub["other/AGENTS.md"]["expected"] == "unknown" assert root["sub/CLAUDE.md"]["expected"] is False and sub["sub/CLAUDE.md"]["expected"] is True + + +def test_accessed_is_false_without_tools_unknown_when_a_read_is_unseen_and_true_when_seen(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + seen = dict(SUB_ROW, id="read=seen") + records = [_record("AGENTS.md", "cwd=.", "not_loaded"), _record("AGENTS.md", "read=sub", "not_loaded"), + _record("AGENTS.md", "read=seen", "not_loaded", read_by_tool=True)] + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW, seen]), {"entries": []}, tree) + accessed = {r["run_id"].split(":")[1]: _sources(receipt, r["run_id"].split(":")[1])["AGENTS.md"]["states"] + ["accessed"] for r in receipt["receipts"]} + assert accessed == {"cwd=.": False, "read=sub": "unknown", "read=seen": True} + + +def test_a_lazy_subtree_file_is_expected_in_its_directory_and_below_not_in_a_sibling(tmp_path): + tree = _tree(tmp_path, {"sub/AGENTS.md": "s\n"}) + deep = dict(SUB_ROW, id="read=sub/deeper", read="sub/deeper/REACH_PROBE.txt") + sibling = dict(SUB_ROW, id="read=sub2", read="sub2/REACH_PROBE.txt") + rows = [ROOT_ROW, SUB_ROW, deep, sibling] + records = [_record("sub/AGENTS.md", row["id"], "not_loaded") for row in rows] + files = [{"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + budget = {"entries": [{"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, rows, files=files), budget, tree) + expected = {row["id"]: _sources(receipt, row["id"])["sub/AGENTS.md"]["expected"] for row in rows} + assert expected == {"cwd=.": False, "read=sub": True, "read=sub/deeper": True, "read=sub2": False} + + +def test_a_nested_link_has_the_expectation_of_the_file_it_links_to(tmp_path): + tree = _tree(tmp_path, {"sub/AGENTS.md": "s\n"}) + records = [_record("sub/AGENTS.md", r, "not_loaded") for r in ("cwd=.", "read=sub")] + files = [{"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006", "aliases": ["sub/CLAUDE.md"]}] + budget = {"entries": [{"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW], files=files), budget, tree) + root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") + assert root["sub/CLAUDE.md"]["scope"] == "adapter" and sub["sub/CLAUDE.md"]["scope"] == "adapter" + assert root["sub/CLAUDE.md"]["expected"] is False and sub["sub/CLAUDE.md"]["expected"] is True + + +@pytest.mark.parametrize("path, scope", [(".claude/CLAUDE.md", "repo"), (".claude/AGENTS.md", "repo"), + ("docs/CLAUDE.md", "subtree"), ("README.md", "repo")]) +def test_project_memory_in_dot_claude_applies_to_the_repo_not_a_subtree(tmp_path, path, scope): + tree = _tree(tmp_path, {path: "x\n"}) + files = [{"path": path, "head": "RC000001", "tail": "RC000002"}] + receipt = build_receipt(_reach([_record(path, "cwd=.", "loaded")], [ROOT_ROW], files=files), {"entries": []}, tree) + assert _sources(receipt, "cwd=.")[path]["scope"] == scope + + +@pytest.mark.parametrize("bad", [{"CLAUDE.md": None}, {"CLAUDE.md": {"cwd=.": "bogus"}}], + ids=["row-not-an-object", "cell-not-a-class"]) +def test_a_saved_matrix_with_a_malformed_row_or_cell_is_a_usage_error(tmp_path, monkeypatch, capsys, bad): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + data["summary"]["matrix"] = bad + (tmp_path / "out" / "malformed.json").write_text(json.dumps(data), encoding="utf-8") + capsys.readouterr() + args = build_parser().parse_args(_from_argv(live, "--from", str(tmp_path / "out" / "malformed.json"))) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "cannot be shown as a matrix" in err and "Traceback" not in err + + +def test_a_receipt_output_link_that_loops_is_a_usage_error_before_the_harness_runs(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch, "--format", "receipt") + (tmp_path / "out").mkdir() + loop = tmp_path / "out" / "effective_instructions.json" + loop.symlink_to(loop) # a link to itself: it cannot be resolved + args = build_parser().parse_args(live) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "cannot resolve" in err and "Traceback" not in err + assert not (tmp_path / "out" / "reach.json").exists() # the probe never ran + + +def test_from_a_receipt_output_link_that_loops_is_a_usage_error_not_a_traceback(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + capsys.readouterr() + loop = tmp_path / "out" / "effective_instructions.json" + loop.symlink_to(loop) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "cannot resolve" in err and "Traceback" not in err From a724a75c5d6f47fea276f3556f8573e9ea59cd8b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:33:16 -0800 Subject: [PATCH 11/15] docs(repo): say when a live receipt reads its source after the run Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 71477fb2c..0379368ab 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1064,8 +1064,12 @@ tree, or a probe of one (`repo.sha` null), is a usage error (exit 2), because `sha256` and the expectations are computed from the source as it is at conversion time and a working tree can change after the probe. A live `repo reach --format receipt` run is not held to this: it builds the receipt -from the source it probes. The matrix (without `--format receipt`) still -accepts an unpinned source. The harness configuration is not +from the source it probes, but it reads `sha256` and the expectations after +the harness exits. An edit to a guidance file made while the run is in flight +therefore appears in the receipt next to the observations of the old text; +the receipt's `unknowns` says so for a working tree, and a pinned source +(`@`) has no such window. The matrix (without `--format receipt`) +still accepts an unpinned source. The harness configuration is not pinned: `expected` comes from the harness profile as it is at conversion time, and its guidance names and import depth are not compared with the probe's, so a receipt is the same only for an unchanged profile. The receipt is written From 95c090ec167df63b375282cf321b34ac93493032 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 10:30:50 -0800 Subject: [PATCH 12/15] fix(repo): resolve a receipt link loop strictly on Python 3.14 Non-strict Path.resolve() returns the looping path on Python 3.13 and later instead of raising, so the overwrite check passed and the probe ran; the two loop tests failed on the py3.14 CI job with exit 1. Resolve strictly (with a fallback for a missing last component) so a loop is the usage error on every version. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 326fbdd4c..a4c7f4cee 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -448,13 +448,23 @@ def _receipt_path(from_file: str) -> Path: return Path(from_file).parent / RECEIPT_NAME +def _strict_resolve(path: Path) -> Path: + """``path`` with every link resolved, a missing or dangling last component included. Non-strict resolve() of a link + loop raises RuntimeError before Python 3.13 and returns the looping path from 3.13 on; strict resolve raises + OSError on a loop on every version.""" + try: + return path.resolve(strict=True) + except FileNotFoundError: + return path.resolve() + + def _receipt_would_overwrite(probe: Path, receipt: Path) -> bool: """A receipt written at the probe's path, directly or through a link, would replace the probe it is built from. - A link loop cannot be resolved (RuntimeError before Python 3.13, OSError after): a usage error, not a traceback.""" + A link loop cannot be resolved (OSError, or RuntimeError before Python 3.13): a usage error, not a traceback.""" from dokimi.repos.reach import ReachError try: - return probe.resolve() == receipt.resolve() + return _strict_resolve(probe) == _strict_resolve(receipt) except (OSError, RuntimeError) as exc: raise ReachError(f"cannot resolve {receipt}: {exc}") from exc From 6f3a4889831a11a168f383accde978c1665c7628 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:42:13 -0800 Subject: [PATCH 13/15] fix(repo): --from refuses any probe flag given and an empty file name; split receipt unknowns by class The probe flags have no parser default, so --from can tell a value that was given (even at the default) from one that was not. An empty --from name is a usage error, not a probe. The receipt's marker-token unknown is written for truncated records only; an error record gets its own unknown. Signed-off-by: Nikolay Petrov --- ...napetrov_effective-instructions-receipt.md | 2 +- dokimi/dokimi/commands/repo.py | 22 +++++++++----- dokimi/dokimi/repos/receipt.py | 19 ++++++++---- dokimi/tests/test_repo_receipt.py | 29 +++++++++++++++++++ 4 files changed, 58 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md index 96b4cd668..c7978d1e7 100644 --- a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source; with `--from`, both must be at a commit (a working tree is a usage error, exit 2), and the receipt may not be written over the saved file. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source; with `--from`, both must be at a commit (a working tree is a usage error, exit 2), and the receipt may not be written over the saved file. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. With `--from`, every probe flag is refused when given, including at its default value, and an empty `--from` name is a usage error (exit 2); the receipt's unknowns name a truncated record and an error record separately. diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index a4c7f4cee..9f61497ca 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -254,17 +254,20 @@ def _check_reach_flags(args: argparse.Namespace, kinds: tuple[str, ...]) -> None raise ReachError("--model is required to probe (leave it out only with --from)") -# The probe's flags and their defaults: --from refuses any of them set to another value. +# The probe's flags, unset by default: --from refuses any of them given, even at its default value. _PROBE_ONLY = {"only": None, "salt": None, "work_dir": None, "keep": False, "bin": None, "dry_run": False, - "model": None, "scenarios": "root,cwd,read", "max_turns": 4, "budget": 0.10, "timeout": 300, - "out": None} + "model": None, "scenarios": None, "max_turns": None, "budget": None, "timeout": None, "out": None} +_PROBE_DEFAULTS = {"scenarios": "root,cwd,read", "max_turns": 4, "budget": 0.10, "timeout": 300} def _reach_entry(args: argparse.Namespace) -> None: """`repo reach`: probe the repository, or with --from read a saved reach.json.""" - if args.from_file: + if args.from_file is not None: # an empty name is still a name: a usage error, never a probe _reach_from_saved(args) else: + for key, value in _PROBE_DEFAULTS.items(): + if getattr(args, key) is None: + setattr(args, key, value) cmd_repo_reach(args) @@ -278,6 +281,8 @@ def _saved_reach_inputs(args: argparse.Namespace): from dokimi.repos.harness_profile import load_profile from dokimi.repos.reach import ReachError, load_reach, saved_gaps, saved_matrix_problem + if not args.from_file: + raise ReachError("--from needs a file name") given = _probe_flags_given(args) if given: raise ReachError(f"--from reads a saved probe, so {', '.join(given)} cannot be used with it") @@ -680,7 +685,7 @@ def register(sub, positive_int) -> None: "still read, to check the file's repository, harness and sha; the receipt goes beside the file") reach_p.add_argument("--bin", default=None, help="Harness executable (default: the profile's `bin`)") reach_p.add_argument("--out", default=None, help="Output directory: reach.json and traces/ (default: reach-out)") - reach_p.add_argument("--scenarios", default="root,cwd,read", + reach_p.add_argument("--scenarios", default=None, help="Comma list of scenario kinds: root, cwd, read (default: all)") reach_p.add_argument("--only", action="append", default=None, metavar="SCENARIO", help="Run only this scenario id (for example read=cpp); repeatable") @@ -688,9 +693,10 @@ def register(sub, positive_int) -> None: help="Parent of the temporary workspace (default: the system temp dir). " "No ancestor may hold guidance files.") reach_p.add_argument("--salt", default=None, help="Token salt (default: random per run)") - reach_p.add_argument("--max-turns", type=positive_int, default=4, dest="max_turns") - reach_p.add_argument("--budget", type=float, default=0.10, help="Per-scenario USD cap (default: 0.10)") - reach_p.add_argument("--timeout", type=positive_int, default=300, help="Per-scenario seconds") + reach_p.add_argument("--max-turns", type=positive_int, default=None, dest="max_turns", + help="Turn cap per scenario (default: 4)") + reach_p.add_argument("--budget", type=float, default=None, help="Per-scenario USD cap (default: 0.10)") + reach_p.add_argument("--timeout", type=positive_int, default=None, help="Per-scenario seconds (default: 300)") reach_p.add_argument("--keep", action="store_true", help="Keep the temporary workspace") reach_p.add_argument("--format", choices=["matrix", "receipt"], default="matrix", help="matrix: the table (default). receipt: print the effective-instructions/v0 receipt as JSON and " diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index e55a8b7e5..3440bc4e2 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -186,7 +186,7 @@ def _receipt(reach: dict[str, Any], run: _Run, runtime: dict[str, str], start_up "target": row.get("read"), "sources": sources, "precedence_observed": _precedence(sources, off), - "unknowns": _run_unknowns(row, sources), + "unknowns": _run_unknowns(row, sources, run), } @@ -256,13 +256,22 @@ def _precedence(sources: list[dict[str, Any]], off: set[str]) -> list[dict[str, for w in winners] -def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]]) -> list[str]: +def _recorded_class(source: dict[str, Any], run: _Run) -> str | None: + """The reach class of the record this source's observation came from (its alias's, for a link).""" + rec = run.records.get((source.get("alias_of") or source["path"], run.row["id"])) + return rec["class"] if rec else None + + +def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]], run: _Run) -> list[str]: out = [] if row.get("error"): out.append("no usable answer from this run: every observed state is unknown") - marked = [s for s in sources if s["observed"] == UNKNOWN and s["dokimi_method"] == DOKIMI_METHOD] - out += [f"{s['path']}: one of its two marker tokens was reported; loaded is not established" - for s in marked if not row.get("error")] + else: + # a truncated record saw one of the two marker tokens; an error record saw neither, so it says nothing + out += [f"{s['path']}: one of its two marker tokens was reported; loaded is not established" + for s in sources if _recorded_class(s, run) == "truncated"] + out += [f"{s['path']}: the probe recorded an error for this file; no state is established" + for s in sources if _recorded_class(s, run) == "error"] if (row.get("prose") or 0) > PROSE_LIMIT: out.append(f"answered with {row['prose']} words besides tokens; its not_loaded states may be under-reports") if row.get("unknown"): diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index 19244274c..86c81fe6b 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -605,3 +605,32 @@ def test_from_a_receipt_output_link_that_loops_is_a_usage_error_not_a_traceback( args.func(args) err = capsys.readouterr().err assert exc.value.code == 2 and "cannot resolve" in err and "Traceback" not in err + + +@pytest.mark.parametrize("cls", ["truncated", "error"]) +def test_the_unknowns_name_a_truncated_record_and_an_error_record_apart(tmp_path, cls): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", cls)], [ROOT_ROW]) + run = next(r for r in build_receipt(reach, {"entries": []}, tree)["receipts"] if r["run_id"].endswith("cwd=.")) + text = " | ".join(run["unknowns"]) + assert ("AGENTS.md: one of its two marker tokens was reported" in text) == (cls == "truncated") + assert ("AGENTS.md: the probe recorded an error for this file" in text) == (cls == "error") + + +@pytest.mark.parametrize("flags", [["--budget", "0.10"], ["--max-turns", "4"], ["--scenarios", "root,cwd,read"], + ["--timeout", "300"]]) +def test_from_refuses_a_probe_flag_given_at_its_default_value(tmp_path, monkeypatch, capsys, flags): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), *flags)) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and flags[0] in capsys.readouterr().err + + +def test_from_an_empty_file_name_is_a_usage_error_and_runs_no_probe(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args([*live, "--from", ""]) # a shell variable that holds no file name + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "--from needs a file name" in capsys.readouterr().err + assert not (tmp_path / "out" / "reach.json").exists() From 57ebe3a75d5c1ea82c9d84882a573d57e805604b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:27:06 -0800 Subject: [PATCH 14/15] test(repo): pin the documented defaults of repo reach; regenerate the CLI snapshot The probe flags are unset in the parser and filled in by _reach_entry, so the snapshot recorded null for --scenarios, --max-turns, --budget and --timeout and failed the CLI snapshot check. Regenerate it with scripts/cli_surface.py dump. A bare `repo reach --dry-run --model m` now has a test that pins the budget 0.1, max turns 4, timeout 300 and the three scenario kinds, with a mutation check for each default. The test is a new file because tests/test_repo_reach.py is at the 1000-line cap. Signed-off-by: Nikolay Petrov --- dokimi/docs/reference/cli-interface.json | 8 +++--- dokimi/tests/test_repo_reach_defaults.py | 31 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 dokimi/tests/test_repo_reach_defaults.py diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index e5fba1563..4cf6d1484 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10707,7 +10707,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 0.1, + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -10870,7 +10870,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 4, + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -10950,7 +10950,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": "root,cwd,read", + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -10966,7 +10966,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 300, + "default": null, "group_key": null, "group_required": null, "hidden": false, diff --git a/dokimi/tests/test_repo_reach_defaults.py b/dokimi/tests/test_repo_reach_defaults.py new file mode 100644 index 000000000..5291f7b57 --- /dev/null +++ b/dokimi/tests/test_repo_reach_defaults.py @@ -0,0 +1,31 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The documented defaults of `repo reach`: the parser leaves the probe flags unset, and a probe fills them in.""" + +from __future__ import annotations + +import json + +import pytest +from tests.test_repo_reach import _repo + +from dokimi.cli import build_parser + + +@pytest.mark.outcome("cjm-repo-reach") +def test_repo_reach_bare_dry_run_plans_the_documented_defaults(tmp_path, capsys): + args = build_parser().parse_args(["repo", "reach", str(_repo(tmp_path)), "--manifest", "evals/repo-eval.yaml", + "--model", "m", "--dry-run"]) + args.func(args) + runs = json.loads(capsys.readouterr().out)["runs"] + assert runs + ids = [run["scenario"] for run in runs] + assert "cwd=." in ids # the root kind + assert any(i.startswith("cwd=") and i != "cwd=." for i in ids) # the cwd kind + assert any(i.startswith("read=") for i in ids) # the read kind + for run in runs: + argv = run["argv"] + assert argv[argv.index("--max-budget-usd") + 1] == "0.1" + assert argv[argv.index("--max-turns") + 1] == "4" + assert (args.timeout, args.scenarios, args.budget, args.max_turns) == (300, "root,cwd,read", 0.1, 4) From b55e3d0eaba40ecc31d91c39be30848304503007 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 16:46:06 -0800 Subject: [PATCH 15/15] fix(repo): drop URL credentials from the receipt's tree label A remote tree label was copied into effective_instructions.json whole, so https://user:token@host/... put the token in a receipt meant to be shared. Drop user:secret userinfo, and all userinfo over http(s); keep a bare ssh user. --- dokimi/dokimi/repos/receipt.py | 12 +++++++++++- dokimi/tests/test_repo_receipt.py | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index 3440bc4e2..0b60ca44a 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -81,6 +81,7 @@ def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) _REV_SUFFIX = re.compile(r"@[0-9a-f]{7,40}$") # the pinned revision a git source's label ends with _URL_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://") # https://, ssh://, file://, ... _SCP_LIKE = re.compile(r"^[^/\\\s]+@[^/\\\s]+:") # git@host:path +_USERINFO = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*://)([^/]*)@") # scheme://user[:secret]@, up to the last @ before the host def _shareable_tree(label: str) -> str: @@ -95,11 +96,20 @@ def _shareable_tree(label: str) -> str: else: local = not (_URL_SCHEME.match(name) or _SCP_LIKE.match(name)) if not local: - return label + return _USERINFO.sub(_drop_secret, name, count=1) + rev base = PurePosixPath(name.replace("\\", "/")).name return (base if base not in ("", ".", "..") else "tree") + rev +def _drop_secret(match: re.Match[str]) -> str: + """Userinfo that can carry a credential is dropped: any ``user:secret``, and all of it over http(s), where a + token is often the user name. A bare ssh user (``ssh://git@``) names no secret and is kept.""" + scheme, user = match.groups() + if ":" in user or scheme.lower().startswith("http"): + return scheme + return match.group() + + def check_saved_reach(reach: dict[str, Any], repo_id: str, harness_name: str, tree_sha: str | None) -> None: """Refuse a saved reach.json probed on another repository, harness or commit than the one it is read against.""" if reach["repo"]["id"] != repo_id: diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py index 86c81fe6b..700935d71 100644 --- a/dokimi/tests/test_repo_receipt.py +++ b/dokimi/tests/test_repo_receipt.py @@ -181,6 +181,10 @@ def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): ("file:///home/alice/toy.git@4ff761e2d943", "toy.git@4ff761e2d943"), ("git@github.com:org/toy.git@4ff761e2d943", "git@github.com:org/toy.git@4ff761e2d943"), ("ssh://git@example.com/org/toy.git@4ff761e2d943", "ssh://git@example.com/org/toy.git@4ff761e2d943"), + ("https://alice:s3cret@example.com/org/toy.git@4ff761e2d943", "https://example.com/org/toy.git@4ff761e2d943"), + ("https://alice@example.com/org/toy.git", "https://example.com/org/toy.git"), + ("ssh://alice:s3cret@example.com/org/toy.git", "ssh://example.com/org/toy.git"), + ("https://alice:p@ss@example.com/org/toy.git", "https://example.com/org/toy.git"), ]) def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): tree = _tree(tmp_path, {"AGENTS.md": "a\n"})