diff --git a/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md new file mode 100644 index 00000000..c7978d1e --- /dev/null +++ b/dokimi/changelog.d/20261031_132853_napetrov_effective-instructions-receipt.md @@ -0,0 +1,8 @@ + + +### Added + +- **An effective-instructions receipt for a reach probe** — `repo reach --format receipt` prints, and writes to `effective_instructions.json` in `--out`, the receipt of `effective-instructions/v0` (agentsmd/agents.md#213) for each probed scenario: each guidance source's path, sha256, scope, expected state (from `repo budget`'s start-up rules, Claude Code only), observed state (`loaded`, `fetched`, `not_loaded` or `unknown`), `observation_method` (`unknown`) with dokimi's own `dokimi_method` (`marker_self_report`: the model reports the marker tokens it sees; the prompt is not captured), the harness's name and version, the displacements seen in the run, and the unknowns. `repo reach --from --format receipt` builds the same receipt from a saved probe without a harness run, after checking the file's repository, harness and commit against the source; with `--from`, both must be at a commit (a working tree is a usage error, exit 2), and the receipt may not be written over the saved file. `followed` is always `unknown`: the probe does not show what the model did with a loaded source. The receipt makes no LLM call and is validated against `schemas/effective_instructions.v1.json`. The mapping and its limits are in `docs/repo-evaluation-contract.md` §7.1. The default output of `repo reach` is unchanged. With `--from`, every probe flag is refused when given, including at its default value, and an empty `--from` name is a usage error (exit 2); the receipt's unknowns name a truncated record and an error record separately. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 8ab11f97..db6e5d52 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -10859,7 +10859,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 0.1, + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -10903,6 +10903,41 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": [ + "'matrix'", + "'receipt'" + ], + "const": null, + "default": "matrix", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--format", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--from", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -10987,7 +11022,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 4, + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -11012,7 +11047,7 @@ "kind": "option", "nargs": null, "position": null, - "required": true, + "required": false, "type": null }, { @@ -11035,7 +11070,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": "reach-out", + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -11067,7 +11102,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": "root,cwd,read", + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -11083,7 +11118,7 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 300, + "default": null, "group_key": null, "group_required": null, "hidden": false, @@ -13036,6 +13071,7 @@ "'answers'", "'arms'", "'cli_routes'", + "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index a26a0c92..6377283a 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1077,6 +1077,104 @@ per scenario). token was in the request bytes. This settles cells where the model under-reports, which is how a small model produces false `truncated` cells. +### 7.1 Effective-instructions receipt (`effective_instructions.v1`) + +`repo reach --format receipt` prints, and writes to `effective_instructions.json` +in `--out`, a receipt in the shape of `effective-instructions/v0` +(agentsmd/agents.md#213). It is built from the reach payload and from the +start-up chain of `repo budget` (`repos/budget.py`). It makes no +LLM call. `repo reach --from --format receipt` builds the same +receipt from a saved probe without running the harness. The source is read +again: the file must be for the same repository, harness and commit (a +different one is a usage error, exit 2). With `--from`, a receipt also needs a +pinned source (`@`) and a saved probe made at a commit: a working +tree, or a probe of one (`repo.sha` null), is a usage error (exit 2), because +`sha256` and the expectations are computed from the source as it is at +conversion time and a working tree can change after the probe. A live +`repo reach --format receipt` run is not held to this: it builds the receipt +from the source it probes, but it reads `sha256` and the expectations after +the harness exits. An edit to a guidance file made while the run is in flight +therefore appears in the receipt next to the observations of the old text; +the receipt's `unknowns` says so for a working tree, and a pinned source +(`@`) has no such window. The matrix (without `--format receipt`) +still accepts an unpinned source. The harness configuration is not +pinned: `expected` comes from the harness profile as it is at conversion time, +and its guidance names and import depth are not compared with the probe's, so +a receipt is the same only for an unchanged profile. The receipt is written +beside the saved file; a name that resolves to the saved probe is a usage error +(exit 2), and a write that fails is a runtime failure (exit 1). The probe-only flags (`--model`, +`--budget`, `--scenarios`, `--only` and the rest) cannot be combined with +`--from`. The schema is `schemas/effective_instructions.v1.json`; the code is +`repos/receipt.py`. + +One artifact is written to `effective_instructions.json`, with one entry in its +`receipts` array per reach scenario and `run_id` `:`. +Each source carries: + +| Receipt field | From | Meaning | +|---|---|---| +| `path`, `alias_of` | reach `files`, `aliases` | a link is `adapter` scope and takes its target's tokens | +| `sha256` | the text dokimi reads (LF-normalised UTF-8) | null when the file is not readable as text | +| `scope` | path | `repo` for a root-level file and for `.claude/CLAUDE.md` and `.claude/AGENTS.md` only; `subtree` for a nested AGENTS.md, CLAUDE.md or CLAUDE.local.md (`.claude/CLAUDE.local.md` included); else `unknown` (a skill, for one) | +| `expected` | budget `when` | see the table below | +| `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | +| `observation_method` | | `unknown` for every source: the thread vocabulary's method, since no runtime event, prompt capture or hook backs the probe | +| `dokimi_method` | | `marker_self_report` for a source the probe recorded; null for a start-up file budget found and reach did not probe | +| `states.accessed` | the harness's read events | `true` if the harness reported a read of the file (the Read tool; Codex and Copilot also report supported shell reads, §7); for a probed source `false` with tools off, else `unknown`; always `unknown` for a start-up file budget found and reach did not probe | +| `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | +| `states.loaded` | the class | `true` for loaded, `false` for not_loaded, `unknown` for fetched, truncated and error: a tool read is not proof that start-up loading did not happen | +| `states.followed` | | always `unknown` (the schema's `const`) | + +`runtime` is the harness's name and version, as reach records them. `model` is +the string passed with `--model`, not the harness's name or a resolved model +version. `observation.method` is `unknown` and `observation.dokimi_method` is +`marker_self_report`: the model reports the tokens it can see, and the request +sent to the provider is not captured. A reported token is evidence of loading, +not a copy of the bytes received. The name is not `prompt_capture`, because +dokimi does not capture the prompt. + +**Expected state.** budget's start-up rules are Claude Code's, so they apply +only to `claude-code` runs; for other harnesses every `expected` is `unknown` +and the receipt says so. + +| budget `when` | `expected` | Reason | +|---|---|---| +| `start` | `true` | loaded at start-up in every run | +| `off` | `false` | a root CLAUDE\*.md turns AGENTS.md off | +| `lazy`, run directory is the file's directory or inside it | `true` | loaded when the work reaches that directory | +| `lazy`, run directory outside it | `false` | not reached by this run | +| `lazy` on a link, an import, a root file, or a file another guidance file `@`-imports | `unknown` | budget's flattened `when` does not say which directory activates the file, so only a nested AGENTS/CLAUDE file that budget lists under its own path, and that no guidance file imports, gets a verdict | +| `on demand` or absent | `unknown` | budget does not say | + +**Precedence.** `precedence_observed` lists a displacement only: a root +CLAUDE\*.md (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/CLAUDE.md`) observed +loaded in a run, with an `off` AGENTS.md observed not_loaded in that run. It +does not give an order of entry. The displacement follows budget's rule as +written (`repos/budget.py`): an AGENTS.md is `off` while a root CLAUDE\*.md +exists, nested ones included, unless a root CLAUDE\* file aliases it or +`@`-imports it (`start`), or a nested CLAUDE\* file `@`-imports it (`lazy`). So +nested AGENTS.md files can appear, and an AGENTS.md at the root is not always +`off`. If budget's rule changes, the receipt follows it. + +**What the marker method can and cannot show.** It shows which tokens were +reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as +observations of that run. It cannot show: + +- that an instruction was followed (`followed` is always `unknown`); +- a read through a shell that the harness does not normalise into a read + event (`cat`, `rg`, `grep` under Claude Code; Codex and Copilot normalise + only the shell commands §7 lists); +- when a source entered the request, or in what order (no timestamps); +- start-up imports and skill listings from budget, which reach does not probe: + they appear with `observed: unknown`. A skill body is loaded on demand, so it + is not listed; the receipt-level unknown says both; +- anything a model under-reports; a prose-heavy answer is flagged. + +Each run lists its own `unknowns` (no usable answer, a source with one token +reported, prose above the limit, unseeded tokens). The receipt-level `unknowns` +repeat the limits above and carry the notice that start-up imports and skill +listings are not probed. + ## 8. L0 scan (`repo_readiness.v1`) `repo scan ` checks every mechanically checkable claim in the guidance diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 55fd25cf..aeb88bab 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -12,7 +12,8 @@ what moves it), ``repo_environment`` (static inspection of what a repository offers an agent), ``plan`` (the cells a run promises to measure), ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts -into an agent's context), ``need_graph`` and ``eval_coverage`` (a frozen need graph and an +into an agent's context), ``effective_instructions`` (the effective-instructions/v0 +receipt of a reach probe), ``need_graph`` and ``eval_coverage`` (a frozen need graph and an eval set's coverage of it) and ``task_inspect`` (which eval-task format each path under a tree is). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in @@ -57,6 +58,7 @@ "plan": "plan.v1", "result": "result.v1", "repo_budget": "repo_budget.v1", + "effective_instructions": "effective_instructions.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", "need_graph": "need_graph.v1", diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 88f64ad0..9f61497c 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -250,6 +250,92 @@ def _check_reach_flags(args: argparse.Namespace, kinds: tuple[str, ...]) -> None raise ReachError(f"unknown --scenarios kind(s) {', '.join(unknown_kinds)} (known: root, cwd, read)") if not math.isfinite(args.budget) or args.budget <= 0: raise ReachError(f"--budget must be > 0, got {args.budget:g}") + if args.model is None: + raise ReachError("--model is required to probe (leave it out only with --from)") + + +# The probe's flags, unset by default: --from refuses any of them given, even at its default value. +_PROBE_ONLY = {"only": None, "salt": None, "work_dir": None, "keep": False, "bin": None, "dry_run": False, + "model": None, "scenarios": None, "max_turns": None, "budget": None, "timeout": None, "out": None} +_PROBE_DEFAULTS = {"scenarios": "root,cwd,read", "max_turns": 4, "budget": 0.10, "timeout": 300} + + +def _reach_entry(args: argparse.Namespace) -> None: + """`repo reach`: probe the repository, or with --from read a saved reach.json.""" + if args.from_file is not None: # an empty name is still a name: a usage error, never a probe + _reach_from_saved(args) + else: + for key, value in _PROBE_DEFAULTS.items(): + if getattr(args, key) is None: + setattr(args, key, value) + cmd_repo_reach(args) + + +def _probe_flags_given(args: argparse.Namespace) -> list[str]: + return [f"--{k.replace('_', '-')}" for k, default in _PROBE_ONLY.items() if getattr(args, k) != default] + + +def _saved_reach_inputs(args: argparse.Namespace): + """The harness profile, the source and the saved reach.json of ``--from``, as typed; raises on a usage error.""" + from dokimi.repos import RepoSource + from dokimi.repos.harness_profile import load_profile + from dokimi.repos.reach import ReachError, load_reach, saved_gaps, saved_matrix_problem + + if not args.from_file: + raise ReachError("--from needs a file name") + given = _probe_flags_given(args) + if given: + raise ReachError(f"--from reads a saved probe, so {', '.join(given)} cannot be used with it") + profile = load_profile(args.harness) + source = RepoSource.parse(args.source) + reach = load_reach(Path(args.from_file)) + gaps = saved_gaps(reach, args.format) + if gaps: + raise ReachError(f"{args.from_file} has no {', '.join(gaps)}, which --format {args.format} needs") + if args.format == "matrix" and (problem := saved_matrix_problem(reach)): + raise ReachError(f"{args.from_file} cannot be shown as a matrix: {problem}") + if args.format == "receipt" and _receipt_would_overwrite(Path(args.from_file), _receipt_path(args.from_file)): + raise ReachError(f"the receipt would replace {args.from_file}, the probe it is read from: " + "rename the saved file") + return profile, source, reach + + +def _reach_from_saved(args: argparse.Namespace) -> None: + """The matrix or the receipt of a saved reach.json, checked against the source; no harness runs.""" + from dokimi.repos import ManifestError, SourceError, open_source + from dokimi.repos.harness_profile import ProfileError + from dokimi.repos.receipt import check_receipt_provenance, check_saved_reach + from dokimi.repos.reach import ReachError, format_matrix + from dokimi.repos.source import SourceArgumentError + + try: # the arguments and the saved file as typed: a usage error + profile, source, reach = _saved_reach_inputs(args) + except (SourceError, ProfileError, ReachError, OSError, ValueError) as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + try: # the clone and fetch: a runtime failure + tree = open_source(source, _cache_dir(args)) + except SourceArgumentError as exc: # a pinned local path that is not the repository root + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + except (SourceError, OSError, ValueError) as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(1) + try: # the manifest and the file's provenance: a usage error; a failed git read is a fetch failure (1) + manifest = _load_manifest(tree, args, guidance_default=True) + check_saved_reach(reach, manifest.id, profile.name, getattr(tree, "sha", None)) + if args.format == "receipt": + check_receipt_provenance(reach, getattr(tree, "sha", None)) + except (SourceError, ManifestError, ReachError) as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(_tree_read_exit_code(exc)) + if args.format == "receipt": + _print_receipt(tree, profile, reach, Path(args.from_file).parent) + print(f"wrote {_receipt_path(args.from_file)}", file=sys.stderr) + else: + print(format_matrix(reach)) + if any(s["error"] for s in reach["scenarios"]): + sys.exit(1) def _tree_read_exit_code(exc: Exception) -> int: @@ -262,6 +348,24 @@ def _tree_read_exit_code(exc: Exception) -> int: return 1 if git_failure else EXIT_USAGE +def _refuse_receipt_over_probe(args: argparse.Namespace, out_dir: Path) -> None: + """Exit as a usage error when ``--format receipt`` would write the receipt over the probe it is built from, or + when an output path is a link that cannot be resolved (a loop).""" + from dokimi.repos.reach import ReachError + + if args.format != "receipt": + return + try: + overwrite = _receipt_would_overwrite(out_dir / "reach.json", out_dir / RECEIPT_NAME) + except ReachError as exc: + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if overwrite: + print(f"✗ {out_dir / RECEIPT_NAME} is the probe itself (a link to {out_dir / 'reach.json'}): " + "remove the link", file=sys.stderr) + sys.exit(EXIT_USAGE) + + def cmd_repo_reach(args: argparse.Namespace) -> None: """L1 reach probe: which guidance files one harness/version/model loads.""" from dokimi.repos import ManifestError, RepoSource, SourceError, open_source @@ -269,7 +373,6 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: from dokimi.repos.source import SourceArgumentError from dokimi.repos.reach import ( ReachError, - format_matrix, planned, probe_files, run_reach, @@ -313,7 +416,8 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: sys.exit(_tree_read_exit_code(exc)) print(f"{len(files)} guidance files, {planned_count} scenarios, " f"at most ${args.budget:g} each", file=sys.stderr) - out_dir = Path(args.out) + out_dir = Path(args.out or "reach-out") + _refuse_receipt_over_probe(args, out_dir) try: # the harness runs: a runtime failure payload = run_reach( tree=tree, manifest=manifest, profile=profile, model=args.model, out_dir=out_dir, @@ -325,12 +429,66 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) path = save_reach(payload, out_dir) - print(format_matrix(payload)) - print(f"wrote {path}") + _print_reach_output(tree, profile, payload, out_dir, path, receipt=args.format == "receipt") if any(s["error"] for s in payload["scenarios"]): sys.exit(1) +def _print_reach_output(tree, profile, payload: dict, out_dir: Path, path: Path, receipt: bool) -> None: + """The matrix on stdout; with ``receipt``, the effective-instructions receipt on stdout and the note on stderr.""" + from dokimi.repos.reach import format_matrix + + if receipt: + _print_receipt(tree, profile, payload, out_dir) + print(f"wrote {path}", file=sys.stderr) + else: + print(format_matrix(payload)) + print(f"wrote {path}") + + +RECEIPT_NAME = "effective_instructions.json" + + +def _receipt_path(from_file: str) -> Path: + return Path(from_file).parent / RECEIPT_NAME + + +def _strict_resolve(path: Path) -> Path: + """``path`` with every link resolved, a missing or dangling last component included. Non-strict resolve() of a link + loop raises RuntimeError before Python 3.13 and returns the looping path from 3.13 on; strict resolve raises + OSError on a loop on every version.""" + try: + return path.resolve(strict=True) + except FileNotFoundError: + return path.resolve() + + +def _receipt_would_overwrite(probe: Path, receipt: Path) -> bool: + """A receipt written at the probe's path, directly or through a link, would replace the probe it is built from. + A link loop cannot be resolved (OSError, or RuntimeError before Python 3.13): a usage error, not a traceback.""" + from dokimi.repos.reach import ReachError + + try: + return _strict_resolve(probe) == _strict_resolve(receipt) + except (OSError, RuntimeError) as exc: + raise ReachError(f"cannot resolve {receipt}: {exc}") from exc + + +def _print_receipt(tree, profile, payload: dict, out_dir: Path) -> None: + """The effective-instructions receipt of a probe: budget's start-up chain, reach's observed states.""" + from dokimi.artifacts import save_artifact + from dokimi.repos.budget import budget + from dokimi.repos.receipt import build_receipt + + receipt = build_receipt(payload, budget(tree, profile.guidance_names, profile.import_max_depth), tree) + try: # a directory that is not writable, or one already named effective_instructions.json: a runtime failure + written = save_artifact("effective_instructions", receipt, out_dir / RECEIPT_NAME) + except OSError as exc: + print(f"✗ cannot write {out_dir / RECEIPT_NAME}: {exc}", file=sys.stderr) + sys.exit(1) + print(json.dumps(written, indent=2)) + + def cmd_repo_budget(args: argparse.Namespace) -> None: """What the repository puts into an agent's context at start-up, lazily and on demand.""" from dokimi.artifacts import save_artifact, stamp @@ -520,10 +678,14 @@ def register(sub, positive_int) -> None: "at --manifest (for a repository that has not added it yet)") reach_p.add_argument("--harness", default="claude-code", help="Built-in profile name or a harness_profile.v1 YAML path (default: claude-code)") - reach_p.add_argument("--model", required=True, help="Model string passed to the harness as it is") + reach_p.add_argument("--model", default=None, help="Model string passed to the harness as it is (required " + "unless --from)") + reach_p.add_argument("--from", default=None, dest="from_file", metavar="REACH_JSON", + help="Read a saved reach.json instead of probing: no harness runs, no cost. The source is " + "still read, to check the file's repository, harness and sha; the receipt goes beside the file") reach_p.add_argument("--bin", default=None, help="Harness executable (default: the profile's `bin`)") - reach_p.add_argument("--out", default="reach-out", help="Output directory: reach.json and traces/") - reach_p.add_argument("--scenarios", default="root,cwd,read", + reach_p.add_argument("--out", default=None, help="Output directory: reach.json and traces/ (default: reach-out)") + reach_p.add_argument("--scenarios", default=None, help="Comma list of scenario kinds: root, cwd, read (default: all)") reach_p.add_argument("--only", action="append", default=None, metavar="SCENARIO", help="Run only this scenario id (for example read=cpp); repeatable") @@ -531,14 +693,18 @@ def register(sub, positive_int) -> None: help="Parent of the temporary workspace (default: the system temp dir). " "No ancestor may hold guidance files.") reach_p.add_argument("--salt", default=None, help="Token salt (default: random per run)") - reach_p.add_argument("--max-turns", type=positive_int, default=4, dest="max_turns") - reach_p.add_argument("--budget", type=float, default=0.10, help="Per-scenario USD cap (default: 0.10)") - reach_p.add_argument("--timeout", type=positive_int, default=300, help="Per-scenario seconds") + reach_p.add_argument("--max-turns", type=positive_int, default=None, dest="max_turns", + help="Turn cap per scenario (default: 4)") + reach_p.add_argument("--budget", type=float, default=None, help="Per-scenario USD cap (default: 0.10)") + reach_p.add_argument("--timeout", type=positive_int, default=None, help="Per-scenario seconds (default: 300)") reach_p.add_argument("--keep", action="store_true", help="Keep the temporary workspace") + reach_p.add_argument("--format", choices=["matrix", "receipt"], default="matrix", + help="matrix: the table (default). receipt: print the effective-instructions/v0 receipt as JSON and " + "write it to effective_instructions.json in --out (docs/repo-evaluation-contract.md §7.1)") reach_p.add_argument("--dry-run", action="store_true", dest="dry_run", help="Print the files and the exact argv of every scenario; run nothing") reach_p.add_argument("--cache-dir", default=None, dest="cache_dir", help=cache_help) - reach_p.set_defaults(func=cmd_repo_reach) + reach_p.set_defaults(func=_reach_entry) budget_p = repo_sub.add_parser( "budget", diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 599b4845..98a0ea65 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -405,9 +405,34 @@ def _matrix(records: list[dict[str, Any]]) -> dict[str, dict[str, str]]: return matrix +def saved_gaps(payload: dict[str, Any], fmt: str) -> list[str]: + """The fields a saved probe must carry to be rendered as ``fmt``: the schema leaves them optional.""" + if fmt == "receipt": + return [key for key in ("salt", "generated_at") if key not in payload] + return [] if "matrix" in payload["summary"] else ["summary.matrix"] + + +_SHORT = {"loaded": "L", "truncated": "T", "fetched": "F", "not_loaded": ".", "error": "E"} # a cell -> its letter + + +def saved_matrix_problem(payload: dict[str, Any]) -> str | None: + """What keeps a saved ``summary.matrix`` from rendering: a row that is not an object, or a cell that is not one of + the classes the table shows (format_matrix would raise on either). None when every cell can be shown.""" + matrix = payload["summary"]["matrix"] + if not isinstance(matrix, dict): + return "summary.matrix is not an object" + for path, row in matrix.items(): + if not isinstance(row, dict): + return f"summary.matrix[{path!r}] is not an object" + for scenario, cell in row.items(): + if not isinstance(cell, str) or cell not in _SHORT: + return f"summary.matrix[{path!r}][{scenario!r}] is {cell!r}, not a reach class" + return None + + def format_matrix(payload: dict[str, Any]) -> str: """A plain-text table: one row per guidance file, one column per scenario.""" - short = {"loaded": "L", "truncated": "T", "fetched": "F", "not_loaded": ".", "error": "E"} + short = _SHORT ids = [s["id"] for s in payload["scenarios"]] label = {f["path"]: " = ".join([f["path"], ", ".join(f["aliases"])]) if f.get("aliases") else f["path"] for f in payload["files"]} @@ -431,7 +456,7 @@ def format_matrix(payload: dict[str, Any]) -> str: if (s.get("prose") or 0) > PROSE_LIMIT: lines.append(f"warning: {s['id']} answered with {s['prose']} words besides tokens; " "read its trace before trusting its not_loaded cells") - if s["unknown"]: + if s.get("unknown"): # schema-optional: a scenario saved without it reports no unknown tokens lines.append(f"unknown tokens in {s['id']}: {', '.join(s['unknown'])}") return "\n".join(lines) diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py new file mode 100644 index 00000000..0b60ca44 --- /dev/null +++ b/dokimi/dokimi/repos/receipt.py @@ -0,0 +1,309 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The effective-instructions receipt (``effective-instructions/v0``, agentsmd/agents.md#213) of one reach probe. + +Built from a ``guidance_reach.v1`` payload and the start-up chain of ``repo_budget.v1``. Per run (one reach +scenario) and per guidance source it gives the expected state (budget's start-up rules), the observed state (the +probe's marker tokens) and the states ``accessed``, ``selected``, ``loaded`` and ``followed``. What the probe +cannot establish is ``unknown``; ``followed`` always is. docs/repo-evaluation-contract.md §7.1 holds the mapping +and its limits. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + +from .budget import _imports, _target +from .reach import PROSE_LIMIT, ReachError +from .source import RepoTree + +SCHEMA = "effective-instructions/v0" +SCHEMA_VERSION = "effective_instructions.v1" +UNKNOWN = "unknown" +# The thread vocabulary's observation_method: no runtime event, prompt capture or hook backs the probe. +OBSERVATION_METHOD = UNKNOWN +# dokimi's own method: the model reports the marker tokens it can see; the request itself is not captured. +DOKIMI_METHOD = "marker_self_report" +# Root-level files that turn AGENTS.md off in Claude Code (budget.py): the displacing set. +CLAUDE_ROOT = ("CLAUDE.md", "CLAUDE.local.md", ".claude/CLAUDE.md") +ROOT_NESTED = (".claude/CLAUDE.md", ".claude/AGENTS.md") # project memory in .claude/ applies repo-wide +SUBTREE_NAMES = ("AGENTS.md", "CLAUDE.md", "CLAUDE.local.md") +_STATE: dict[str | None, str] = {"loaded": "loaded", "fetched": "fetched", "not_loaded": "not_loaded"} +_LOADED: dict[str | None, bool] = {"loaded": True, "not_loaded": False} +DETAIL = ("the model is asked to report the marker tokens it can see; the request sent to the provider is not " + "captured, so a reported token is evidence of loading, not a copy of the bytes it received") +NOT_PROBED = ("start-up imports and skill listings from repo_budget are not probed by reach; their observed state is " + "unknown, and skill bodies (loaded on demand) are not listed") + + +@dataclass(frozen=True) +class _Run: + row: dict[str, Any] # the reach scenario row + records: dict[tuple[str, str], dict[str, Any]] # (file, scenario id) -> record + when_of: dict[str, str] # canonical path -> budget's when + own: frozenset[str] # guidance files budget lists under their own path (not a link to another file) + digests: dict[str, str | None] + workdir: str # the directory the run works in + rules: bool # budget's rules apply (Claude Code) + + +def build_receipt(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) -> dict[str, Any]: + """The ``effective_instructions.v1`` artifact: one ``effective-instructions/v0`` receipt per reach scenario.""" + harness = reach["harness"] + rules = harness["name"] == "claude-code" # budget's start-up rules are Claude Code's + when_of = _budget_when(reach, budget) if rules else {} + own = _own_guidance(reach, budget, tree) + start_up = [p for p, when in when_of.items() if when in ("start", "off") and p not in _known(reach)] + digests = source_digests(tree, [p for p, _ in _probed(reach)] + start_up) + records = {(r["file"], r["scenario"]): r for r in reach["records"]} + runtime = {"name": harness["name"], "version": harness["version"]} + receipts = [] + for row in reach["scenarios"]: + run = _Run(row, records, when_of, own, digests, _workdir(row), rules) + receipts.append(_receipt(reach, run, runtime, start_up)) + return { + "schema_version": SCHEMA_VERSION, + "generated_at": reach["generated_at"], + "repo": {**reach["repo"], "tree": _shareable_tree(reach["repo"]["tree"])}, + "runtime": runtime, + "model": reach["model"], + "observation": {"method": OBSERVATION_METHOD, "dokimi_method": DOKIMI_METHOD, "detail": DETAIL}, + "unknowns": _global_unknowns(harness["name"], rules, pinned=bool(getattr(tree, "sha", None))), + "receipts": receipts, + } + + +_REV_SUFFIX = re.compile(r"@[0-9a-f]{7,40}$") # the pinned revision a git source's label ends with +_URL_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://") # https://, ssh://, file://, ... +_SCP_LIKE = re.compile(r"^[^/\\\s]+@[^/\\\s]+:") # git@host:path +_USERINFO = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*://)([^/]*)@") # scheme://user[:secret]@, up to the last @ before the host + + +def _shareable_tree(label: str) -> str: + """The tree label with a local directory cut to its last component, so no home path reaches a shared receipt. + A remote label (any URL scheme but file://, or git@host:path) is kept whole; any other label names a local + directory, absolute or relative, and is cut. The revision is the label's own hex suffix, so an ``@`` in a parent + directory is part of the path.""" + match = _REV_SUFFIX.search(label) + name, rev = (label[: match.start()], match.group()) if match else (label, "") + if name.lower().startswith("file://"): + local = True + else: + local = not (_URL_SCHEME.match(name) or _SCP_LIKE.match(name)) + if not local: + return _USERINFO.sub(_drop_secret, name, count=1) + rev + base = PurePosixPath(name.replace("\\", "/")).name + return (base if base not in ("", ".", "..") else "tree") + rev + + +def _drop_secret(match: re.Match[str]) -> str: + """Userinfo that can carry a credential is dropped: any ``user:secret``, and all of it over http(s), where a + token is often the user name. A bare ssh user (``ssh://git@``) names no secret and is kept.""" + scheme, user = match.groups() + if ":" in user or scheme.lower().startswith("http"): + return scheme + return match.group() + + +def check_saved_reach(reach: dict[str, Any], repo_id: str, harness_name: str, tree_sha: str | None) -> None: + """Refuse a saved reach.json probed on another repository, harness or commit than the one it is read against.""" + if reach["repo"]["id"] != repo_id: + raise ReachError(f"reach.json is for repository {reach['repo']['id']!r}, not {repo_id!r}") + if reach["harness"]["name"] != harness_name: + raise ReachError(f"reach.json was probed with harness {reach['harness']['name']!r}, not {harness_name!r}") + probed = reach["repo"].get("sha") + if probed and tree_sha and probed != tree_sha: + raise ReachError(f"reach.json was probed at {probed[:12]}, the source is at {tree_sha[:12]}") + + +def check_receipt_provenance(reach: dict[str, Any], tree_sha: str | None) -> None: + """Refuse a receipt rebuilt from a saved probe unless both sides are at a commit. The digests and expectations + are read from the source now, so they describe the probed text only when that text is pinned: a working tree + may have changed since the probe, and a probe of a working tree has no commit to check the source against.""" + if not tree_sha: + raise ReachError("--format receipt needs a pinned source: a working tree can change after the probe, so give " + "the repository at a revision (@)") + if not reach["repo"].get("sha"): + raise ReachError("reach.json was probed from a working tree (repo.sha is null), so its receipt cannot be " + "tied to a commit; probe a pinned source to get a receipt") + + +def source_digests(tree: RepoTree, paths: list[str]) -> dict[str, str | None]: + """sha256 of the text dokimi reads for each path (LF line endings: the text the probe seeds); None if unreadable.""" + out: dict[str, str | None] = {} + for path in dict.fromkeys(paths): + try: + out[path] = hashlib.sha256(tree.read_text(path).encode("utf-8")).hexdigest() + except (OSError, ValueError, RuntimeError): # SourceError and UnicodeDecodeError are ValueErrors + out[path] = None + return out + + +def _probed(reach: dict[str, Any]) -> list[tuple[str, str | None]]: + """(path, alias_of) for every probed path: each guidance file, then the links to it.""" + out: list[tuple[str, str | None]] = [] + for item in reach["files"]: + out.append((item["path"], None)) + out.extend((alias, item["path"]) for alias in item.get("aliases", [])) + return out + + +def _known(reach: dict[str, Any]) -> set[str]: + return {path for path, _ in _probed(reach)} + + +def _budget_when(reach: dict[str, Any], budget: dict[str, Any]) -> dict[str, str]: + """budget's ``when`` of each guidance file and import, keyed by the file the path resolves to.""" + target_of = {alias: item["path"] for item in reach["files"] for alias in item.get("aliases", [])} + return {target_of.get(e["path"], e["path"]): e["when"] + for e in budget["entries"] if e["kind"] in ("guidance", "import", "skill listing")} + + +def _own_guidance(reach: dict[str, Any], budget: dict[str, Any], tree: RepoTree) -> frozenset[str]: + """Guidance files budget lists under their own path, and that no other guidance file @-imports: a link or an + import is activated where its importer is, so the file has no subtree of its own.""" + aliases = {alias for item in reach["files"] for alias in item.get("aliases", [])} + guidance = [e["path"] for e in budget["entries"] if e["kind"] == "guidance"] + imported = {_target(tree, p) for g in guidance for p in _imports(tree, [g])[0]} + return frozenset(p for p in guidance if p not in aliases and p not in imported) + + +def _dir(path: str) -> str: + parent = str(PurePosixPath(path).parent) + return "" if parent == "." else parent + + +def _workdir(row: dict[str, Any]) -> str: + """The directory a run works in: the one holding the file it is told to read, else where it starts.""" + return _dir(row["read"]) if row.get("read") else row["cwd"] + + +def _receipt(reach: dict[str, Any], run: _Run, runtime: dict[str, str], start_up: list[str]) -> dict[str, Any]: + sources = [_source(path, alias_of, run) for path, alias_of in _probed(reach)] + sources += [_budget_only(path, run) for path in start_up] + off = {p for p, when in run.when_of.items() if when == "off"} + row = run.row + return { + "schema": SCHEMA, + "run_id": f"{reach['salt']}:{row['id']}", + "runtime": runtime, + "cwd": row["cwd"] or ".", + "target": row.get("read"), + "sources": sources, + "precedence_observed": _precedence(sources, off), + "unknowns": _run_unknowns(row, sources, run), + } + + +def _source(path: str, alias_of: str | None, run: _Run) -> dict[str, Any]: + target = alias_of or path + rec = run.records.get((target, run.row["id"])) + cls = rec["class"] if rec else None + return { + "path": path, + **({"alias_of": alias_of} if alias_of else {}), + "sha256": run.digests.get(path), + "scope": _scope(path, alias_of is not None), + "expected": _expected(target, run.when_of.get(target), run), + "observed": _STATE.get(cls, UNKNOWN), # a truncated cell saw one token: an observation, no verdict + "observation_method": OBSERVATION_METHOD, + "dokimi_method": DOKIMI_METHOD if rec else None, + "states": _states(cls, run.row["tools"] == "none", bool(rec and rec["read_by_tool"])), + } + + +def _budget_only(path: str, run: _Run) -> dict[str, Any]: + """A start-up import or guidance file budget found and reach did not probe: expected, never observed.""" + return {"path": path, "sha256": run.digests.get(path), "scope": _scope(path, False), + "expected": _expected(path, run.when_of[path], run), "observed": UNKNOWN, + "observation_method": OBSERVATION_METHOD, "dokimi_method": None, + "states": _states(None, False, False)} + + +def _states(cls: str | None, tools_off: bool, read_by_tool: bool) -> dict[str, Any]: + """accessed: a Read-tool read was seen (false with no tools at all); selected: the probe saw the source enter the + request, by the harness or by the agent's own read; loaded: at the harness's own load step, not through a read.""" + accessed: bool | str = True if read_by_tool else (False if tools_off else UNKNOWN) + selected: bool | str = True if cls in ("loaded", "fetched") else UNKNOWN + return {"accessed": accessed, "selected": selected, "loaded": _LOADED.get(cls, UNKNOWN), "followed": UNKNOWN} + + +def _expected(target: str, when: str | None, run: _Run) -> bool | str: + """Start-up files are expected in every run; off ones in none; a nested lazy file in a run whose directory is its + subtree (or inside it), since the harness loads it when the work reaches that directory.""" + if not run.rules or when is None: + return UNKNOWN + if when in ("start", "off"): + return when == "start" + if when == "lazy": # budget's flattened when cannot say where a link or an import is activated: only a subtree file + d = _dir(target) + if target not in run.own or not d or target.rsplit("/", 1)[-1] not in SUBTREE_NAMES: + return UNKNOWN + return run.workdir == d or run.workdir.startswith(d + "/") + return UNKNOWN + + +def _scope(path: str, alias: bool) -> str: + if alias: + return "adapter" # a link to another guidance file: its tokens are the target's + if "/" not in path or path in ROOT_NESTED: + return "repo" + return "subtree" if path.rsplit("/", 1)[-1] in SUBTREE_NAMES else UNKNOWN + + +def _precedence(sources: list[dict[str, Any]], off: set[str]) -> list[dict[str, str]]: + """A file budget says a root CLAUDE* file displaces, seen not loaded while that file was seen loaded in the run.""" + winners = sorted(s["path"] for s in sources if s["path"] in CLAUDE_ROOT and s["observed"] == "loaded") + if not winners: + return [] + return [{"winner": w, "displaced": s["path"]} + for s in sources if (s.get("alias_of") or s["path"]) in off and s["observed"] == "not_loaded" + for w in winners] + + +def _recorded_class(source: dict[str, Any], run: _Run) -> str | None: + """The reach class of the record this source's observation came from (its alias's, for a link).""" + rec = run.records.get((source.get("alias_of") or source["path"], run.row["id"])) + return rec["class"] if rec else None + + +def _run_unknowns(row: dict[str, Any], sources: list[dict[str, Any]], run: _Run) -> list[str]: + out = [] + if row.get("error"): + out.append("no usable answer from this run: every observed state is unknown") + else: + # a truncated record saw one of the two marker tokens; an error record saw neither, so it says nothing + out += [f"{s['path']}: one of its two marker tokens was reported; loaded is not established" + for s in sources if _recorded_class(s, run) == "truncated"] + out += [f"{s['path']}: the probe recorded an error for this file; no state is established" + for s in sources if _recorded_class(s, run) == "error"] + if (row.get("prose") or 0) > PROSE_LIMIT: + out.append(f"answered with {row['prose']} words besides tokens; its not_loaded states may be under-reports") + if row.get("unknown"): + out.append("reported tokens that were never seeded: " + ", ".join(row["unknown"])) + return out + + +def _global_unknowns(name: str, rules: bool, pinned: bool) -> list[str]: + out = [ + "followed is not observed: no marker shows what the model did with a loaded source", + "observation_method is unknown: no prompt is captured; dokimi_method names the marker self-report instead", + "accessed is observed only where the harness reports a read event; " + "a shell read it does not normalise is not seen", + "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", + "no load timestamps: the probe does not record when a source entered the request", + "the order in which sources entered the request is not observed; precedence_observed lists displacements only", + NOT_PROBED, + ] + if not rules: + out.append(f"budget's start-up rules are Claude Code's: no source has an expected state for {name}") + if not pinned: + out.append("the source is a working tree: sha256 and expected are read after the harness exits, so an edit " + "made while the run was in flight is not reflected in the observations; pin the source to a " + "revision for a stable receipt") + return out diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index fde7e7cf..d7340c39 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -63,6 +63,8 @@ class KindFields: counts=("cells",)), "result": KindFields(time=("generated_at",), counts=("cells",)), "repo_budget": KindFields(subject=("tree",), counts=("entries",)), + "effective_instructions": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",), + counts=("receipts",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), "need_graph": KindFields(subject=("artifact_name",), counts=("nodes", "edges")), diff --git a/dokimi/dokimi/schemas/effective_instructions.v1.json b/dokimi/dokimi/schemas/effective_instructions.v1.json new file mode 100644 index 00000000..7e79f606 --- /dev/null +++ b/dokimi/dokimi/schemas/effective_instructions.v1.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/effective_instructions.v1.json", + "title": "Effective-instructions receipt v1", + "description": "The effective-instructions/v0 receipt (agentsmd/agents.md#213) of one `repo reach` probe (docs/repo-evaluation-contract.md §7.1): one receipt per scenario, each source's expected state from budget's start-up rules and its observed state from the probe's marker tokens. `followed` is always unknown: reach never proves it.", + "type": "object", + "required": ["schema_version", "generated_at", "repo", "runtime", "model", "observation", "unknowns", "receipts"], + "properties": { + "schema_version": {"const": "effective_instructions.v1"}, + "generated_at": {"type": "string"}, + "repo": { + "type": "object", + "required": ["id", "tree"], + "properties": { + "id": {"type": "string"}, + "tree": {"type": "string"}, + "sha": {"type": ["string", "null"]}, + "manifest": {"type": "string"} + } + }, + "runtime": {"$ref": "#/$defs/runtime"}, + "model": {"type": "string"}, + "observation": { + "type": "object", + "required": ["method", "dokimi_method", "detail"], + "properties": { + "method": {"enum": ["unknown"], "description": "The thread vocabulary's observation_method: no runtime event, prompt capture or hook backs the probe."}, + "dokimi_method": {"const": "marker_self_report", "description": "dokimi's method: the model reports the marker tokens it can see; the request is not captured."}, + "detail": {"type": "string"} + } + }, + "unknowns": {"type": "array", "items": {"type": "string"}}, + "receipts": { + "type": "array", + "items": { + "type": "object", + "required": ["schema", "run_id", "runtime", "cwd", "target", "sources", "precedence_observed", "unknowns"], + "properties": { + "schema": {"const": "effective-instructions/v0"}, + "run_id": {"type": "string", "description": "The reach salt and the scenario id, so the receipt names its record in reach.json."}, + "runtime": {"$ref": "#/$defs/runtime"}, + "cwd": {"type": "string", "description": "The directory the run started in; `.` is the repository root."}, + "target": {"type": ["string", "null"], "description": "The file the run was told to read (the probe file of a read= scenario), else null."}, + "sources": {"type": "array", "items": {"$ref": "#/$defs/source"}}, + "precedence_observed": { + "type": "array", + "description": "Displacements seen in this run: a root CLAUDE* file loaded while the AGENTS.md it displaces was not.", + "items": { + "type": "object", + "required": ["winner", "displaced"], + "properties": {"winner": {"type": "string"}, "displaced": {"type": "string"}} + } + }, + "unknowns": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "$defs": { + "runtime": { + "type": "object", + "required": ["name", "version"], + "properties": {"name": {"type": "string"}, "version": {"type": "string"}} + }, + "tristate": {"enum": [true, false, "unknown"]}, + "source": { + "type": "object", + "required": ["path", "sha256", "scope", "expected", "observed", "observation_method", "dokimi_method", "states"], + "properties": { + "path": {"type": "string"}, + "alias_of": {"type": "string", "description": "Set when the path is a link to another probed file; its tokens and its observation are the target's."}, + "sha256": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$", "description": "Null when the file cannot be read as text."}, + "scope": {"enum": ["repo", "subtree", "user", "adapter", "unknown"]}, + "expected": {"$ref": "#/$defs/tristate", "description": "Budget's start-up rules: true for start, false for off, true for a lazy file whose subtree the run works in. unknown without Claude Code's rules."}, + "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, + "observation_method": {"enum": ["runtime_event", "prompt_capture", "hook", "unknown"], "description": "The thread vocabulary's method; unknown for every source, since no runtime event backs the probe."}, + "dokimi_method": {"enum": ["marker_self_report", null], "description": "Set when the probe recorded the source in this run, whether or not a marker token came back; null for a source it did not probe."}, + "states": { + "type": "object", + "required": ["accessed", "selected", "loaded", "followed"], + "properties": { + "accessed": {"$ref": "#/$defs/tristate"}, + "selected": {"$ref": "#/$defs/tristate"}, + "loaded": {"$ref": "#/$defs/tristate"}, + "followed": {"const": "unknown", "description": "Reach never proves that an instruction was followed."} + } + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json new file mode 100644 index 00000000..11a646f6 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/effective_instructions.v1.json @@ -0,0 +1,110 @@ +{ + "schema_version": "effective_instructions.v1", + "generated_at": "2026-10-31T00:00:00+00:00", + "repo": { + "id": "toy", + "tree": "toy", + "sha": null, + "manifest": "evals/repo-eval.yaml" + }, + "runtime": { + "name": "claude-code", + "version": "2.1.0" + }, + "model": "m", + "observation": { + "method": "unknown", + "dokimi_method": "marker_self_report", + "detail": "the model is asked to report the marker tokens it can see; the request sent to the provider is not captured, so a reported token is evidence of loading, not a copy of the bytes it received" + }, + "unknowns": [ + "followed is not observed: no marker shows what the model did with a loaded source", + "observation_method is unknown: no prompt is captured; dokimi_method names the marker self-report instead", + "accessed is observed only for the Read tool; a read through a shell (cat, rg, grep) is not seen", + "sha256 is over the LF-normalised UTF-8 text dokimi reads, which the probe seeds; a CRLF file's bytes differ", + "no load timestamps: the probe does not record when a source entered the request", + "the order in which sources entered the request is not observed; precedence_observed lists displacements only", + "start-up imports and skills from repo_budget are not probed by reach; their observed state is unknown" + ], + "receipts": [ + { + "schema": "effective-instructions/v0", + "run_id": "salt:cwd=.", + "runtime": { + "name": "claude-code", + "version": "2.1.0" + }, + "cwd": ".", + "target": null, + "sources": [ + { + "path": "AGENTS.md", + "sha256": "87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7", + "scope": "repo", + "expected": false, + "observed": "not_loaded", + "observation_method": "unknown", + "dokimi_method": "marker_self_report", + "states": { + "accessed": false, + "selected": "unknown", + "loaded": false, + "followed": "unknown" + } + }, + { + "path": "CLAUDE.md", + "sha256": "a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478", + "scope": "repo", + "expected": true, + "observed": "loaded", + "observation_method": "unknown", + "dokimi_method": "marker_self_report", + "states": { + "accessed": false, + "selected": true, + "loaded": true, + "followed": "unknown" + } + }, + { + "path": "sub/AGENTS.md", + "sha256": "cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556", + "scope": "subtree", + "expected": false, + "observed": "not_loaded", + "observation_method": "unknown", + "dokimi_method": "marker_self_report", + "states": { + "accessed": false, + "selected": "unknown", + "loaded": false, + "followed": "unknown" + } + }, + { + "path": "docs/style.md", + "sha256": "8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be", + "scope": "unknown", + "expected": true, + "observed": "unknown", + "observation_method": "unknown", + "dokimi_method": null, + "states": { + "accessed": "unknown", + "selected": "unknown", + "loaded": "unknown", + "followed": "unknown" + } + } + ], + "precedence_observed": [ + { + "winner": "CLAUDE.md", + "displaced": "AGENTS.md" + } + ], + "unknowns": [] + } + ] +} diff --git a/dokimi/tests/test_repo_reach_defaults.py b/dokimi/tests/test_repo_reach_defaults.py new file mode 100644 index 00000000..5291f7b5 --- /dev/null +++ b/dokimi/tests/test_repo_reach_defaults.py @@ -0,0 +1,31 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The documented defaults of `repo reach`: the parser leaves the probe flags unset, and a probe fills them in.""" + +from __future__ import annotations + +import json + +import pytest +from tests.test_repo_reach import _repo + +from dokimi.cli import build_parser + + +@pytest.mark.outcome("cjm-repo-reach") +def test_repo_reach_bare_dry_run_plans_the_documented_defaults(tmp_path, capsys): + args = build_parser().parse_args(["repo", "reach", str(_repo(tmp_path)), "--manifest", "evals/repo-eval.yaml", + "--model", "m", "--dry-run"]) + args.func(args) + runs = json.loads(capsys.readouterr().out)["runs"] + assert runs + ids = [run["scenario"] for run in runs] + assert "cwd=." in ids # the root kind + assert any(i.startswith("cwd=") and i != "cwd=." for i in ids) # the cwd kind + assert any(i.startswith("read=") for i in ids) # the read kind + for run in runs: + argv = run["argv"] + assert argv[argv.index("--max-budget-usd") + 1] == "0.1" + assert argv[argv.index("--max-turns") + 1] == "4" + assert (args.timeout, args.scenarios, args.budget, args.max_turns) == (300, "root,cwd,read", 0.1, 4) diff --git a/dokimi/tests/test_repo_receipt.py b/dokimi/tests/test_repo_receipt.py new file mode 100644 index 00000000..700935d7 --- /dev/null +++ b/dokimi/tests/test_repo_receipt.py @@ -0,0 +1,640 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The effective-instructions receipt of a reach probe (no LLM): its mapping, its schema and the CLI.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess + +import pytest + +from dokimi.artifacts import ArtifactValidationError, validate_artifact +from dokimi.cli import build_parser +from dokimi.repos import WorkTree +from dokimi.repos.receipt import build_receipt, check_receipt_provenance, check_saved_reach, source_digests +from dokimi.repos.reach import ReachError, format_matrix, load_reach +from tests.test_repo_reach import _profile, _repo + +ROOT_ROW = {"id": "cwd=.", "cwd": "", "tools": "none", "read": None, "error": None, "prose": 0, "unknown": []} +SUB_ROW = {"id": "read=sub", "cwd": "", "tools": "read", "read": "sub/REACH_PROBE.txt", "error": None, + "prose": 0, "unknown": []} +BUDGET = {"entries": [ + {"path": "CLAUDE.md", "kind": "guidance", "when": "start", "chars": 10}, + {"path": "AGENTS.md", "kind": "guidance", "when": "off", "chars": 10}, + {"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "docs/style.md", "kind": "import", "when": "start", "chars": 10}, +]} + + +def _record(file: str, scenario: str, cls: str, read_by_tool: bool = False) -> dict: + return {"file": file, "scenario": scenario, "harness": "claude-code", "harness_version": "2.1.0", + "model": "m", "head": cls in ("loaded", "fetched", "truncated"), "tail": cls in ("loaded", "fetched"), + "read_by_tool": read_by_tool, "class": cls} + + +def _reach(records: list[dict], scenarios: list[dict], harness: str = "claude-code", + files: list[dict] | None = None) -> dict: + files = files or [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + return {"generated_at": "2026-10-31T00:00:00+00:00", "repo": {"id": "toy", "tree": "toy", "sha": None, + "manifest": "evals/repo-eval.yaml"}, "harness": {"name": harness, "version": "2.1.0"}, "model": "m", + "salt": "salt", "files": files, "scenarios": scenarios, "records": records, "summary": {}} + + +def _tree(tmp_path, files: dict[str, str | bytes]) -> WorkTree: + for rel, text in files.items(): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + if isinstance(text, bytes): + (tmp_path / rel).write_bytes(text) + else: + (tmp_path / rel).write_text(text, encoding="utf-8") + return WorkTree(tmp_path) + + +def _sources(receipt: dict, run_id: str) -> dict[str, dict]: + run = next(r for r in receipt["receipts"] if r["run_id"].endswith(run_id)) + return {s["path"]: s for s in run["sources"]} + + +CLASS_MAP = { # reach class -> (observed, selected, loaded) + "loaded": ("loaded", True, True), + "fetched": ("fetched", True, "unknown"), + "not_loaded": ("not_loaded", "unknown", False), + "truncated": ("unknown", "unknown", "unknown"), + "error": ("unknown", "unknown", "unknown"), +} + + +@pytest.mark.parametrize("cls", sorted(CLASS_MAP)) +def test_each_reach_class_maps_to_its_observed_and_thread_states(tmp_path, cls): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", cls, read_by_tool=cls == "fetched")], [ROOT_ROW]) + src = _sources(build_receipt(reach, {"entries": []}, tree), "cwd=.")["AGENTS.md"] + observed, selected, loaded = CLASS_MAP[cls] + assert src["observed"] == observed + assert src["observation_method"] == "unknown" # the thread vocabulary: no runtime event backs the probe + assert src["dokimi_method"] == "marker_self_report" # the probe saw a marker record for this source + assert (src["states"]["selected"], src["states"]["loaded"]) == (selected, loaded) + assert src["states"]["followed"] == "unknown" + + +def test_a_source_the_probe_did_not_record_has_no_dokimi_method(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "docs/style.md": "d\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + receipt = build_receipt(reach, BUDGET, tree) + src = _sources(receipt, "cwd=.") + assert src["docs/style.md"]["dokimi_method"] is None and src["docs/style.md"]["observation_method"] == "unknown" + assert receipt["observation"] == {"method": "unknown", "dokimi_method": "marker_self_report", + "detail": receipt["observation"]["detail"]} + + +def test_start_up_expectations_follow_budget_and_the_run_directory(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n", "docs/style.md": "d\n"}) + records = [_record(f, s, "not_loaded") for f in ("AGENTS.md", "CLAUDE.md", "sub/AGENTS.md") + for s in ("cwd=.", "read=sub")] + reach = _reach(records, [ROOT_ROW, SUB_ROW]) + receipt = build_receipt(reach, BUDGET, tree) + root = _sources(receipt, "cwd=.") + assert root["CLAUDE.md"]["expected"] is True # start-up + assert root["AGENTS.md"]["expected"] is False # off: a root CLAUDE.md displaces it + assert root["sub/AGENTS.md"]["expected"] is False # lazy, and the run starts at the root + assert root["docs/style.md"]["expected"] is True and root["docs/style.md"]["observed"] == "unknown" + assert _sources(receipt, "read=sub")["sub/AGENTS.md"]["expected"] is True # the run reads inside its subtree + + +def test_a_displacement_seen_in_the_run_is_precedence_observed(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + records = [_record("CLAUDE.md", "cwd=.", "loaded"), _record("AGENTS.md", "cwd=.", "not_loaded")] + receipt = build_receipt(_reach(records, [ROOT_ROW]), BUDGET, tree) + run = receipt["receipts"][0] + assert run["precedence_observed"] == [{"winner": "CLAUDE.md", "displaced": "AGENTS.md"}] + + +def test_no_displacement_claim_without_the_winner_loaded(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + records = [_record("CLAUDE.md", "cwd=.", "not_loaded"), _record("AGENTS.md", "cwd=.", "not_loaded")] + assert build_receipt(_reach(records, [ROOT_ROW]), BUDGET, tree)["receipts"][0]["precedence_observed"] == [] + + +def test_other_harnesses_get_no_expectations_and_say_so(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + records = [_record("AGENTS.md", "cwd=.", "loaded")] + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + receipt = build_receipt(_reach(records, [ROOT_ROW], harness="codex", files=files), BUDGET, tree) + src = _sources(receipt, "cwd=.") + assert set(src) == {"AGENTS.md", "CLAUDE.md"} # budget's start-up chain is Claude Code's, so none is added + assert all(s["expected"] == "unknown" for s in src.values()) + assert any("no source has an expected state for codex" in u for u in receipt["unknowns"]) + + +def test_a_link_is_an_adapter_with_the_targets_observation(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "sub/AGENTS.md": "s\n"}) + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002", "aliases": ["CLAUDE.md"]}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + records = [_record("AGENTS.md", "cwd=.", "loaded")] + src = _sources(build_receipt(_reach(records, [ROOT_ROW], files=files), {"entries": []}, tree), "cwd=.") + assert src["CLAUDE.md"]["alias_of"] == "AGENTS.md" + assert src["CLAUDE.md"]["scope"] == "adapter" and src["CLAUDE.md"]["observed"] == "loaded" + assert src["sub/AGENTS.md"]["scope"] == "subtree" and src["sub/AGENTS.md"]["observed"] == "unknown" + + +def test_a_scenario_with_no_answer_is_unknown_and_says_why(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + row = dict(ROOT_ROW, error="exit 1") + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "error")], [row]), {"entries": []}, tree) + run = receipt["receipts"][0] + assert all(s["observed"] == "unknown" for s in run["sources"]) + assert any("no usable answer" in u for u in run["unknowns"]) + + +def test_a_prose_heavy_or_unseeded_answer_is_flagged(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + row = dict(ROOT_ROW, prose=40, unknown=["RC0000FF"]) + run = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "not_loaded")], [row]), {"entries": []}, tree) + text = " ".join(run["receipts"][0]["unknowns"]) + assert "under-reports" in text and "RC0000FF" in text + + +def test_digests_are_of_the_text_read_and_none_when_unreadable(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\r\nb\n", "blob.md": b"\xff\xfe\x00bin"}) + digests = source_digests(tree, ["AGENTS.md", "blob.md", "missing.md"]) + assert digests["AGENTS.md"] == hashlib.sha256("a\nb\n".encode("utf-8")).hexdigest() # LF, as dokimi reads it + assert digests["blob.md"] is None and digests["missing.md"] is None + + +@pytest.mark.parametrize("label, shared", [ + ("/home/alice/src/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("C:\\Users\\alice\\toy", "toy"), + ("~alice/toy", "toy"), + ("https://example.com/org/toy.git@4ff761e2d943", "https://example.com/org/toy.git@4ff761e2d943"), + ("toy@4ff761e2d943", "toy@4ff761e2d943"), + ("/home/alice@corp/src/toy", "toy"), + ("/home/alice@corp.com/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("C:\\Users\\alice@corp\\toy", "toy"), + ("../alice/private/toy@4ff761e2d943", "toy@4ff761e2d943"), + ("alice/private/toy", "toy"), + ("file:///home/alice/toy.git@4ff761e2d943", "toy.git@4ff761e2d943"), + ("git@github.com:org/toy.git@4ff761e2d943", "git@github.com:org/toy.git@4ff761e2d943"), + ("ssh://git@example.com/org/toy.git@4ff761e2d943", "ssh://git@example.com/org/toy.git@4ff761e2d943"), + ("https://alice:s3cret@example.com/org/toy.git@4ff761e2d943", "https://example.com/org/toy.git@4ff761e2d943"), + ("https://alice@example.com/org/toy.git", "https://example.com/org/toy.git"), + ("ssh://alice:s3cret@example.com/org/toy.git", "ssh://example.com/org/toy.git"), + ("https://alice:p@ss@example.com/org/toy.git", "https://example.com/org/toy.git"), +]) +def test_the_tree_label_keeps_no_local_directory(tmp_path, label, shared): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["tree"] = label + receipt = build_receipt(reach, {"entries": []}, tree) + assert receipt["repo"]["tree"] == shared + assert "alice" not in json.dumps(receipt) + + +def test_a_working_tree_receipt_says_its_source_is_read_after_the_run(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), {"entries": []}, tree) + assert any("working tree" in u and "pin the source" in u for u in receipt["unknowns"]) + + +def test_the_receipt_is_a_valid_artifact_and_a_schema_rejects_a_followed_claim(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + validate_artifact("effective_instructions", receipt) + receipt["receipts"][0]["sources"][0]["states"]["followed"] = "loaded" + with pytest.raises(ArtifactValidationError, match="followed"): + validate_artifact("effective_instructions", receipt) + + +def _end_to_end_args(tmp_path, monkeypatch, *extra): + monkeypatch.setenv("FAKE_MODE", "ok") + root = _repo(tmp_path) + profile = _profile(tmp_path, name="claude-code", guidance={"names": ["CLAUDE.md", "AGENTS.md"]}) + work = tmp_path / "work" + work.mkdir() + return ["repo", "reach", str(root), "--manifest", "evals/repo-eval.yaml", "--harness", str(profile), + "--model", "m", "--out", str(tmp_path / "out"), "--work-dir", str(work), "--salt", "s", + "--scenarios", "root", *extra] + + +def test_cli_receipt_format_prints_and_writes_a_valid_receipt(tmp_path, monkeypatch, capsys): + args = build_parser().parse_args(_end_to_end_args(tmp_path, monkeypatch, "--format", "receipt")) + args.func(args) + out = capsys.readouterr().out + data = json.loads(out) + validate_artifact("effective_instructions", data) + assert any("working tree" in u for u in data["unknowns"]) # live on a working tree, unpinned + assert json.loads((tmp_path / "out" / "effective_instructions.json").read_text(encoding="utf-8")) == data + run = data["receipts"][0] + assert run["run_id"] == "s:cwd=." and run["runtime"] == {"name": "claude-code", "version": "9.8.7"} + states = {s["path"]: s["observed"] for s in run["sources"]} + assert states["CLAUDE.md"] == "loaded" and states["AGENTS.md"] == "not_loaded" + # budget turns off every AGENTS.md while a root CLAUDE.md exists, nested ones included + assert sorted(p["displaced"] for p in run["precedence_observed"]) == ["AGENTS.md", "sub/AGENTS.md"] + assert all(p["winner"] == "CLAUDE.md" for p in run["precedence_observed"]) + assert out.lstrip().startswith("{") and "wrote" not in out # the matrix is not printed in receipt mode + + +def test_default_format_still_prints_the_matrix(tmp_path, monkeypatch, capsys): + args = build_parser().parse_args(_end_to_end_args(tmp_path, monkeypatch)) + args.func(args) + out = capsys.readouterr().out + assert "wrote" in out and not (tmp_path / "out" / "effective_instructions.json").exists() + + +def _from_argv(live: list[str], *extra: str) -> list[str]: + """The argv that reads a saved reach.json against the same source and harness as a live run's argv.""" + return ["repo", "reach", live[2], "--manifest", "evals/repo-eval.yaml", "--harness", + live[live.index("--harness") + 1], *extra] + + +def _saved(tmp_path) -> str: + return str(tmp_path / "out" / "reach.json") + + +def _pinned(live: list[str]) -> list[str]: + """The same argv with its source committed and pinned to HEAD: a receipt needs a commit on both sides.""" + root = live[2] + for args in (["init", "-q"], ["add", "-A"], + ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--no-gpg-sign", "-m", "x"]): + subprocess.run(["git", "-C", root, *args], check=True, capture_output=True) + return [live[0], live[1], f"{root}@HEAD", *live[3:]] + + +def test_from_a_saved_reach_gives_the_receipt_of_the_live_run(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch, "--format", "receipt")) + args = build_parser().parse_args(live) + args.func(args) + live_receipt = json.loads(capsys.readouterr().out) + assert not any("working tree" in u for u in live_receipt["unknowns"]) # pinned: nothing is read after the run + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + args.func(args) + assert json.loads(capsys.readouterr().out) == live_receipt + assert (tmp_path / "out" / "effective_instructions.json").exists() # beside the saved file + + +def test_from_prints_the_matrix_of_the_saved_probe_and_runs_nothing(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(live) + args.func(args) + capsys.readouterr() + + def no_probe(**_kwargs): + raise AssertionError("a saved probe must not run the harness") + + monkeypatch.setattr("dokimi.repos.reach.run_reach", no_probe) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path))) + args.func(args) + assert capsys.readouterr().out == format_matrix(load_reach(tmp_path / "out" / "reach.json")) + "\n" + + +def test_from_a_missing_file_is_a_usage_error(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(_from_argv(live, "--from", str(tmp_path / "nope.json"))) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "nope.json" in capsys.readouterr().err + + +@pytest.mark.parametrize("text", ["{not json", "[1, 2]", '{"schema": "guidance_reach.v1"}']) +def test_from_a_malformed_or_wrong_shaped_file_is_a_usage_error(tmp_path, monkeypatch, capsys, text): + live = _end_to_end_args(tmp_path, monkeypatch) + bad = tmp_path / "bad.json" + bad.write_text(text, encoding="utf-8") + args = build_parser().parse_args(_from_argv(live, "--from", str(bad))) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and capsys.readouterr().err.startswith("\u2717") + + +def test_from_refuses_a_saved_probe_of_another_repository_or_harness(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + for key, value, text in (("repo", {**data["repo"], "id": "other"}, "for repository 'other'"), + ("harness", {**data["harness"], "name": "codex"}, "harness 'codex'")): + other = tmp_path / f"{key}.json" + other.write_text(json.dumps({**data, key: value}), encoding="utf-8") + args = build_parser().parse_args(_from_argv(live, "--from", str(other))) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and text in capsys.readouterr().err + + +def test_from_refuses_the_probe_only_flags(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--budget", "0.5", "--model", "m")) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "--model, --budget" in err # both named, in one message + + +def test_a_probe_needs_a_model(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + no_model = live[:live.index("--model")] + live[live.index("--model") + 2:] + args = build_parser().parse_args(no_model) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "--model is required" in capsys.readouterr().err + + +def test_a_saved_probe_must_match_the_commit_it_is_read_against(tmp_path): + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["sha"] = "a" * 40 + check_saved_reach(reach, "toy", "claude-code", "a" * 40) # the same commit is accepted + check_saved_reach(reach, "toy", "claude-code", None) # a tree without a commit cannot be compared + with pytest.raises(ReachError, match="probed at aaaaaaaaaaaa, the source is at bbbbbbbbbbbb"): + check_saved_reach(reach, "toy", "claude-code", "b" * 40) + + +def test_a_receipt_needs_a_pinned_probe_and_a_pinned_source(tmp_path): + reach = _reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]) + reach["repo"]["sha"] = "a" * 40 + check_receipt_provenance(reach, "a" * 40) # both at the same commit: the digests describe the probed text + with pytest.raises(ReachError, match="pinned"): # a working tree can change after the probe + check_receipt_provenance(reach, None) + reach["repo"]["sha"] = None # probed from a working tree: its observations have no commit to check against + with pytest.raises(ReachError, match="pinned"): + check_receipt_provenance(reach, "a" * 40) + + +def test_from_a_receipt_refuses_a_pinned_probe_read_against_a_working_tree(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) # probed at HEAD: repo.sha is set + capsys.readouterr() + working = _from_argv(live, "--from", _saved(tmp_path), "--format", "receipt") + working[2] = str(tmp_path / "repo") # the same directory, read as a working tree (no revision) + args = build_parser().parse_args(working) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "--format receipt needs a pinned source" in capsys.readouterr().err + + +def test_from_a_receipt_refuses_an_unpinned_source_where_the_matrix_runs(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) # an unpinned probe: repo.sha is null + capsys.readouterr() + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "--format receipt needs a pinned source" in err + assert not (tmp_path / "out" / "effective_instructions.json").exists() + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path))) # the matrix needs no commit + args.func(args) + assert capsys.readouterr().out == format_matrix(load_reach(tmp_path / "out" / "reach.json")) + "\n" + + +def test_from_a_receipt_refuses_an_unpinned_probe_read_against_a_pinned_source(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) # probed from a working tree: repo.sha null + capsys.readouterr() + pinned = _pinned(live) # the same repository, committed, now read at HEAD + args = build_parser().parse_args(_from_argv(pinned, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "probed from a working tree (repo.sha is null)" in err + + +def test_a_lazy_guidance_is_expected_only_in_the_subtree_it_activates(tmp_path): + tree = _tree(tmp_path, {"shared.md": "x\n", "sub/AGENTS.md": "s\n"}) + records = [_record(f, s, "not_loaded") for f in ("shared.md", "sub/AGENTS.md") for s in ("cwd=.", "read=sub")] + files = [{"path": "shared.md", "head": "RC000001", "tail": "RC000002", "aliases": ["sub/CLAUDE.md"]}, + {"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + budget = {"entries": [ # sub/CLAUDE.md is a link to shared.md: budget's lazy is the link's activation, not shared.md's + {"path": "sub/CLAUDE.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW], files=files), budget, tree) + root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") + assert root["sub/CLAUDE.md"]["expected"] == "unknown" and sub["sub/CLAUDE.md"]["expected"] == "unknown" + assert root["shared.md"]["expected"] == "unknown" and sub["shared.md"]["expected"] == "unknown" + assert root["sub/AGENTS.md"]["expected"] is False and sub["sub/AGENTS.md"]["expected"] is True + + +def test_a_start_up_skill_listing_is_a_source_and_its_body_is_not(tmp_path): + tree = _tree(tmp_path, {"CLAUDE.md": "c\n", ".claude/skills/fmt/SKILL.md": "---\nname: fmt\n---\n"}) + budget = {"entries": [ + {"path": "CLAUDE.md", "kind": "guidance", "when": "start", "chars": 10}, + {"path": ".claude/skills/fmt/SKILL.md", "kind": "skill listing", "when": "start", "chars": 10}, + {"path": ".claude/skills/fmt/SKILL.md", "kind": "skill body", "when": "on demand", "chars": 10}]} + files = [{"path": "CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + receipt = build_receipt(_reach([_record("CLAUDE.md", "cwd=.", "loaded")], [ROOT_ROW], files=files), budget, tree) + sources = receipt["receipts"][0]["sources"] + skill = [s for s in sources if s["path"] == ".claude/skills/fmt/SKILL.md"] + assert len(skill) == 1 and skill[0]["expected"] is True and skill[0]["observed"] == "unknown" + assert skill[0]["sha256"] is not None + + +@pytest.mark.parametrize("fmt, drop, key", [ + ("receipt", "salt", "salt"), + ("receipt", "generated_at", "generated_at"), + ("matrix", "matrix", "summary.matrix"), +]) +def test_a_schema_valid_probe_without_what_its_format_needs_is_a_usage_error(tmp_path, monkeypatch, capsys, + fmt, drop, key): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + if drop == "matrix": + del data["summary"]["matrix"] + else: + del data[drop] + bad = tmp_path / "partial.json" + bad.write_text(json.dumps(data), encoding="utf-8") + argv = _from_argv(live, "--from", str(bad), *(["--format", "receipt"] if fmt == "receipt" else [])) + args = build_parser().parse_args(argv) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and key in err and "Traceback" not in err + + +@pytest.mark.parametrize("link", [False, True], ids=["renamed", "symlink"]) +def test_from_never_writes_the_receipt_over_the_probe_it_reads(tmp_path, monkeypatch, capsys, link): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + capsys.readouterr() + saved = tmp_path / "out" / "reach.json" + before = saved.read_text(encoding="utf-8") + receipt = tmp_path / "out" / "effective_instructions.json" + if link: + receipt.symlink_to(saved) + else: + saved.rename(receipt) + saved = receipt + args = build_parser().parse_args(_from_argv(live, "--from", str(saved), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "Traceback" not in capsys.readouterr().err + assert receipt.resolve().read_text(encoding="utf-8") == before + + +def test_a_receipt_that_cannot_be_written_is_a_runtime_failure_not_a_traceback(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + capsys.readouterr() + (tmp_path / "out" / "effective_instructions.json").mkdir() # a directory where the receipt goes + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 1 and "cannot write" in err and "Traceback" not in err + + +def test_a_live_receipt_that_cannot_be_written_is_a_runtime_failure(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch, "--format", "receipt") + (tmp_path / "out").mkdir() + (tmp_path / "out" / "effective_instructions.json").mkdir() + args = build_parser().parse_args(live) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 and "cannot write" in capsys.readouterr().err + + +def test_a_matrix_of_a_probe_saved_without_unknown_tokens_renders(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + for scenario in data["scenarios"]: + scenario.pop("unknown", None) # schema-optional + (tmp_path / "out" / "bare.json").write_text(json.dumps(data), encoding="utf-8") + capsys.readouterr() + args = build_parser().parse_args(_from_argv(live, "--from", str(tmp_path / "out" / "bare.json"))) + args.func(args) + assert capsys.readouterr().out == format_matrix(data) + "\n" + + +def test_a_lazy_guidance_imported_by_a_nested_file_is_expected_only_where_that_file_is(tmp_path): + tree = _tree(tmp_path, {"sub/CLAUDE.md": "@../other/AGENTS.md\n", "other/AGENTS.md": "o\n"}) + records = [_record(f, s, "not_loaded") for f in ("other/AGENTS.md", "sub/CLAUDE.md") for s in ("cwd=.", "read=sub")] + files = [{"path": "other/AGENTS.md", "head": "RC000001", "tail": "RC000002"}, + {"path": "sub/CLAUDE.md", "head": "RC000003", "tail": "RC000004"}] + budget = {"entries": [ # the import activates other/AGENTS.md in sub/, not where the file lies + {"path": "sub/CLAUDE.md", "kind": "guidance", "when": "lazy", "chars": 10}, + {"path": "other/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW], files=files), budget, tree) + root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") + assert root["other/AGENTS.md"]["expected"] == "unknown" and sub["other/AGENTS.md"]["expected"] == "unknown" + assert root["sub/CLAUDE.md"]["expected"] is False and sub["sub/CLAUDE.md"]["expected"] is True + + +def test_accessed_is_false_without_tools_unknown_when_a_read_is_unseen_and_true_when_seen(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + seen = dict(SUB_ROW, id="read=seen") + records = [_record("AGENTS.md", "cwd=.", "not_loaded"), _record("AGENTS.md", "read=sub", "not_loaded"), + _record("AGENTS.md", "read=seen", "not_loaded", read_by_tool=True)] + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW, seen]), {"entries": []}, tree) + accessed = {r["run_id"].split(":")[1]: _sources(receipt, r["run_id"].split(":")[1])["AGENTS.md"]["states"] + ["accessed"] for r in receipt["receipts"]} + assert accessed == {"cwd=.": False, "read=sub": "unknown", "read=seen": True} + + +def test_a_lazy_subtree_file_is_expected_in_its_directory_and_below_not_in_a_sibling(tmp_path): + tree = _tree(tmp_path, {"sub/AGENTS.md": "s\n"}) + deep = dict(SUB_ROW, id="read=sub/deeper", read="sub/deeper/REACH_PROBE.txt") + sibling = dict(SUB_ROW, id="read=sub2", read="sub2/REACH_PROBE.txt") + rows = [ROOT_ROW, SUB_ROW, deep, sibling] + records = [_record("sub/AGENTS.md", row["id"], "not_loaded") for row in rows] + files = [{"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006"}] + budget = {"entries": [{"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, rows, files=files), budget, tree) + expected = {row["id"]: _sources(receipt, row["id"])["sub/AGENTS.md"]["expected"] for row in rows} + assert expected == {"cwd=.": False, "read=sub": True, "read=sub/deeper": True, "read=sub2": False} + + +def test_a_nested_link_has_the_expectation_of_the_file_it_links_to(tmp_path): + tree = _tree(tmp_path, {"sub/AGENTS.md": "s\n"}) + records = [_record("sub/AGENTS.md", r, "not_loaded") for r in ("cwd=.", "read=sub")] + files = [{"path": "sub/AGENTS.md", "head": "RC000005", "tail": "RC000006", "aliases": ["sub/CLAUDE.md"]}] + budget = {"entries": [{"path": "sub/AGENTS.md", "kind": "guidance", "when": "lazy", "chars": 10}]} + receipt = build_receipt(_reach(records, [ROOT_ROW, SUB_ROW], files=files), budget, tree) + root, sub = _sources(receipt, "cwd=."), _sources(receipt, "read=sub") + assert root["sub/CLAUDE.md"]["scope"] == "adapter" and sub["sub/CLAUDE.md"]["scope"] == "adapter" + assert root["sub/CLAUDE.md"]["expected"] is False and sub["sub/CLAUDE.md"]["expected"] is True + + +@pytest.mark.parametrize("path, scope", [(".claude/CLAUDE.md", "repo"), (".claude/AGENTS.md", "repo"), + ("docs/CLAUDE.md", "subtree"), ("README.md", "repo")]) +def test_project_memory_in_dot_claude_applies_to_the_repo_not_a_subtree(tmp_path, path, scope): + tree = _tree(tmp_path, {path: "x\n"}) + files = [{"path": path, "head": "RC000001", "tail": "RC000002"}] + receipt = build_receipt(_reach([_record(path, "cwd=.", "loaded")], [ROOT_ROW], files=files), {"entries": []}, tree) + assert _sources(receipt, "cwd=.")[path]["scope"] == scope + + +@pytest.mark.parametrize("bad", [{"CLAUDE.md": None}, {"CLAUDE.md": {"cwd=.": "bogus"}}], + ids=["row-not-an-object", "cell-not-a-class"]) +def test_a_saved_matrix_with_a_malformed_row_or_cell_is_a_usage_error(tmp_path, monkeypatch, capsys, bad): + live = _end_to_end_args(tmp_path, monkeypatch) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + data = json.loads((tmp_path / "out" / "reach.json").read_text(encoding="utf-8")) + data["summary"]["matrix"] = bad + (tmp_path / "out" / "malformed.json").write_text(json.dumps(data), encoding="utf-8") + capsys.readouterr() + args = build_parser().parse_args(_from_argv(live, "--from", str(tmp_path / "out" / "malformed.json"))) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "cannot be shown as a matrix" in err and "Traceback" not in err + + +def test_a_receipt_output_link_that_loops_is_a_usage_error_before_the_harness_runs(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch, "--format", "receipt") + (tmp_path / "out").mkdir() + loop = tmp_path / "out" / "effective_instructions.json" + loop.symlink_to(loop) # a link to itself: it cannot be resolved + args = build_parser().parse_args(live) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "cannot resolve" in err and "Traceback" not in err + assert not (tmp_path / "out" / "reach.json").exists() # the probe never ran + + +def test_from_a_receipt_output_link_that_loops_is_a_usage_error_not_a_traceback(tmp_path, monkeypatch, capsys): + live = _pinned(_end_to_end_args(tmp_path, monkeypatch)) + build_parser().parse_args(live).func(build_parser().parse_args(live)) + capsys.readouterr() + loop = tmp_path / "out" / "effective_instructions.json" + loop.symlink_to(loop) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), "--format", "receipt")) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and "cannot resolve" in err and "Traceback" not in err + + +@pytest.mark.parametrize("cls", ["truncated", "error"]) +def test_the_unknowns_name_a_truncated_record_and_an_error_record_apart(tmp_path, cls): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + reach = _reach([_record("AGENTS.md", "cwd=.", cls)], [ROOT_ROW]) + run = next(r for r in build_receipt(reach, {"entries": []}, tree)["receipts"] if r["run_id"].endswith("cwd=.")) + text = " | ".join(run["unknowns"]) + assert ("AGENTS.md: one of its two marker tokens was reported" in text) == (cls == "truncated") + assert ("AGENTS.md: the probe recorded an error for this file" in text) == (cls == "error") + + +@pytest.mark.parametrize("flags", [["--budget", "0.10"], ["--max-turns", "4"], ["--scenarios", "root,cwd,read"], + ["--timeout", "300"]]) +def test_from_refuses_a_probe_flag_given_at_its_default_value(tmp_path, monkeypatch, capsys, flags): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args(_from_argv(live, "--from", _saved(tmp_path), *flags)) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and flags[0] in capsys.readouterr().err + + +def test_from_an_empty_file_name_is_a_usage_error_and_runs_no_probe(tmp_path, monkeypatch, capsys): + live = _end_to_end_args(tmp_path, monkeypatch) + args = build_parser().parse_args([*live, "--from", ""]) # a shell variable that holds no file name + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "--from needs a file name" in capsys.readouterr().err + assert not (tmp_path / "out" / "reach.json").exists() diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index b5bd54ee..ea59f097 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -46,6 +46,7 @@ "plan.v1": ({}, "skill:onetbb", "claude-sonnet-4-6", "2026-10-21T00:00:00Z"), "result.v1": ({}, None, None, "2026-10-21T01:00:00Z"), "repo_budget.v1": ({}, "toy", None, None), + "effective_instructions.v1": ({}, "toy", "m", "2026-10-31T00:00:00+00:00"), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), "need_graph.v1": ({}, "dpnp-io", None, None), diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index de76bb8d..b1aa0c6e 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -32,6 +32,7 @@ "plugin_delta.v1": {"arms": 2}, "questions.v2": {"questions": 3}, "repo_budget.v1": {"entries": 9}, + "effective_instructions.v1": {"receipts": 1}, "repo_environment.v1": {"instructions": 4, "findings": 13}, "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6},