From 83863a948fa3b2a14ea612c651b3f79a1e191e66 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:03:34 -0700 Subject: [PATCH 01/14] feat(repo): guidance:overlay arm and overlay reach check A harness profile names its agent with an `overlay` key (claude-code: claude, codex, copilot). The guidance:overlay arm keeps the base AGENTS.md and the AGENTS..md of that agent, and removes every other AGENTS..md. The arm is refused without a profile overlay and when the base has no overlay for the agent. The reach probe now probes every overlay, even where the manifest globs miss it, and reports a foreign overlay that a harness loaded as a cross_loaded finding. own_loaded says whether the harness loads its own overlay at all. Refs agentsmd/agents.md#185 Signed-off-by: Nikolay Petrov --- .../20261031_120000_napetrov_ce-overlay.md | 8 + dokimi/docs/preparing-a-repository.md | 7 +- dokimi/docs/repo-evaluation-contract.md | 19 +- dokimi/dokimi/commands/repo_run.py | 3 +- dokimi/dokimi/repos/harness_profile.py | 9 + dokimi/dokimi/repos/overlays.py | 37 +++ dokimi/dokimi/repos/profiles/claude-code.yaml | 1 + .../dokimi/repos/profiles/codex-review.yaml | 1 + dokimi/dokimi/repos/profiles/codex.yaml | 1 + .../dokimi/repos/profiles/copilot-review.yaml | 1 + dokimi/dokimi/repos/profiles/copilot.yaml | 1 + dokimi/dokimi/repos/reach.py | 35 ++- dokimi/dokimi/repos/workspace.py | 52 +++- dokimi/dokimi/schemas/guidance_reach.v1.json | 12 + dokimi/dokimi/schemas/harness_profile.v1.json | 5 + dokimi/tests/test_repo_overlay.py | 250 ++++++++++++++++++ 16 files changed, 432 insertions(+), 10 deletions(-) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_ce-overlay.md create mode 100644 dokimi/dokimi/repos/overlays.py create mode 100644 dokimi/tests/test_repo_overlay.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_ce-overlay.md b/dokimi/changelog.d/20261031_120000_napetrov_ce-overlay.md new file mode 100644 index 000000000..778a6539b --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_ce-overlay.md @@ -0,0 +1,8 @@ + + +### Added + +- **`guidance:overlay` arm for per-agent AGENTS.md overlays** — `dokimi repo run --arm guidance:overlay` gives a cell the base `AGENTS.md` and the `AGENTS..md` of the harness under test, and removes the overlays of other agents (agentsmd/agents.md#185). A harness profile names its agent with the new `overlay` key (`claude` for Claude Code, `codex`, `copilot`). The arm refuses to run without that key, or when the base has no overlay for the agent. `dokimi repo reach` now probes every `AGENTS..md`, and reports a foreign overlay that the harness loaded as a `finding:` line and in the `overlays` block of `guidance_reach.v1`. Its `own_loaded` field says whether the harness loads its own overlay at all. diff --git a/dokimi/docs/preparing-a-repository.md b/dokimi/docs/preparing-a-repository.md index e52910f23..98642c6ff 100644 --- a/dokimi/docs/preparing-a-repository.md +++ b/dokimi/docs/preparing-a-repository.md @@ -6,7 +6,8 @@ SPDX-License-Identifier: Apache-2.0 # Preparing a repository for coding agents > **Status: recommendations.** §1–§7 apply today. The measurement loop in §8 runs -> today for guidance arms (`guidance:none`, `guidance:raw`, `guidance:bridge`). +> today for guidance arms (`guidance:none`, `guidance:raw`, `guidance:bridge`, +> `guidance:overlay`). > `guidance:bridge` measures the `CLAUDE.md` import (§7). The `skill:` and > `skill-catalog:` arms, which install skills, run today too > ([skill arms](benchmarking-and-comparison.md#skill-arms-in-repository-mode)). Arms @@ -249,7 +250,9 @@ Treat each item above as a hypothesis for your repository. 2. Run the baseline and the arm with `dokimi repo run`, on the current models you use. Today an arm can remove or keep the guidance (`guidance:none`, `guidance:raw`), or inject a file (`guidance:bridge`, which writes a `CLAUDE.md` - containing `@AGENTS.md` next to each `AGENTS.md` for profiles with a bridge key). + containing `@AGENTS.md` next to each `AGENTS.md` for profiles with a bridge key), or + keep only the base and the harness's own `AGENTS..md` (`guidance:overlay`, for + profiles with an `overlay` key). Testing an added tool, other than a skill (`skill:`, `skill-catalog:`), needs planned arms. Today, test one added guidance line with two task directories that differ only in a `setup_patches` entry (same prompt, same graders, distinct ids), or in their `base` commit (see diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c2b873922..4eb13cb21 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -293,7 +293,7 @@ belongs to the repository's grader and is passed through unread, except `grader` 3. `git init` and commit (no history, so fixes cannot leak) 4. apply `setup_patches` 5. apply the arm: `guidance:none`, `guidance:raw`, `guidance:bridge`, - `guidance:rev:` or `patch:` + `guidance:overlay`, `guidance:rev:` or `patch:` 6. amend the commit 7. apply `review_patch` with `git am` under a neutral author 8. expire the reflog and prune unreachable objects: the @@ -313,6 +313,15 @@ content: depends on its version: Claude Code 2.1.286 loads `AGENTS.md` itself, but only when no `CLAUDE.md` is in its start-up chain, so a repository-side bridge arm would mean different things on different versions. +- `overlay` exists only for a profile with an `overlay` key (the agent name, for + Claude Code `claude`). It is the per-agent overlay of agentsmd/agents.md#185: + `AGENTS..md` beside an `AGENTS.md`, at any depth. The workspace keeps + the base `AGENTS.md` and the `AGENTS..md` files and removes every + other `AGENTS..md` (not the harness's own names such as + `AGENTS.override.md`, nor files under `.claude`). The files are recognised by + name, not by `guidance.globs`. A base with no `AGENTS..md` is a + preparation error, so a cell cannot be the base alone by accident. The arm + does not make a harness load its overlay: the reach probe reports that. - `rev:` keeps the base's code and replaces its guidance with that of another revision of the source: the base's matched files are deleted and the files the globs match at `` (outside `eval_dir`) are written. It answers @@ -864,6 +873,7 @@ guidance: # DECLARED; verified by the reach probe, ne resolution: ancestors-of-cwd + lazy-on-subtree-access imports: "@path" bridge: {for: AGENTS.md, write: CLAUDE.md, content: "@AGENTS.md"} +overlay: claude # the AGENTS.claude.md the guidance:overlay arm keeps (agentsmd/agents.md#185) args_if_present: # argv added after the "{tools}" element when the workspace file exists and the tool mode is `all` .mcp.json: ["--mcp-config", ".mcp.json"] ``` @@ -1045,6 +1055,13 @@ per scenario). - `unknown`: tokens the model reported that were never seeded; - `prose`: words besides tokens. A high count usually means an explanation or a refusal, so read that trace before trusting its `not_loaded` cells. +- Overlays (`overlays` in `guidance_reach.v1`, the design note of §3): every + `AGENTS..md` is probed, even where the globs miss it, so a foreign overlay + is tested rather than assumed absent. The expectation is that foreign overlays are + `not_loaded`. `cross_loaded` lists each foreign overlay some scenario reported + as `loaded`, `truncated` or `fetched`; the text report prints it as a `finding:` + line. `own_loaded` lists the harness's own overlays that loaded in some scenario; + when none did, the text report says so. `own` is the profile's `overlay`. - If payload capture is available, a `payload` field records whether each token was in the request bytes. This settles cells where the model under-reports, which is how a small model produces false `truncated` cells. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 8c81f0859..6a86851c6 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -460,7 +460,8 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: "result.json per cell, already finished cells skipped") _common(run_p, source_help, cache_help, positive_int) run_p.add_argument("--arm", action="append", default=None, required=True, - help="guidance:none | guidance:raw | guidance:bridge | guidance:rev:REV (the base with " + help="guidance:none | guidance:raw | guidance:bridge | guidance:overlay (the base with the AGENTS..md " + "of the harness only) | guidance:rev:REV (the base with " "the guidance files of revision REV of the source; the arm is named by REV's full SHA, " "resolved once per batch) | patch:FILE (a unified diff on this host, applied with git " "apply on top of guidance:raw; named patch:@, path and " diff --git a/dokimi/dokimi/repos/harness_profile.py b/dokimi/dokimi/repos/harness_profile.py index 07f401058..c9723a1d7 100644 --- a/dokimi/dokimi/repos/harness_profile.py +++ b/dokimi/dokimi/repos/harness_profile.py @@ -54,6 +54,7 @@ class HarnessProfile: endpoint_env: str | None = None guidance: dict[str, Any] = field(default_factory=dict) bridge: dict[str, str] | None = None + overlay: str | None = None # the agent name of its AGENTS..md (guidance:overlay, repos/overlays.py) prompt_suffix: str = "" config_files: dict[str, str] = field(default_factory=dict) bin_companions: tuple[str, ...] = () # files beside the executable that it needs; mounted next to it @@ -213,6 +214,12 @@ def _check_tools_element(raw: dict[str, Any], path: str) -> None: raise ProfileError(f"{path}: invoke must have exactly one {TOOLS_ELEMENT} element") +def _check_overlay(raw: dict[str, Any], path: str) -> None: + overlay = raw.get("overlay") + if overlay is not None and not re.fullmatch(r"[a-z0-9][a-z0-9-]*", str(overlay)): # fullmatch: `$` takes "\n" + raise ProfileError(f"{path}: overlay {overlay!r} is not an agent name (lower case, digits, dashes)") + + def parse_profile(raw: Any, path: str = "") -> HarnessProfile: errors = schema_errors("harness_profile.v1", raw) if errors: @@ -237,6 +244,7 @@ def parse_profile(raw: Any, path: str = "") -> HarnessProfile: raise ProfileError(f"{path}: invoke never passes {{prompt}} (set native_prompt for a harness that " "uses its own)") _check_tools_element(raw, path) + _check_overlay(raw, path) return HarnessProfile( name=raw["name"], bin=raw["bin"], @@ -251,6 +259,7 @@ def parse_profile(raw: Any, path: str = "") -> HarnessProfile: endpoint_env=raw.get("endpoint_env"), guidance=dict(raw.get("guidance") or {}), bridge=raw.get("bridge"), + overlay=raw.get("overlay"), prompt_suffix=raw.get("prompt_suffix") or "", config_files=dict(raw.get("config_files") or {}), bin_companions=tuple(raw.get("bin_companions") or ()), diff --git a/dokimi/dokimi/repos/overlays.py b/dokimi/dokimi/repos/overlays.py new file mode 100644 index 000000000..6e6fc2ad4 --- /dev/null +++ b/dokimi/dokimi/repos/overlays.py @@ -0,0 +1,37 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Per-agent overlays of AGENTS.md (agentsmd/agents.md#185): ``AGENTS..md`` beside an ``AGENTS.md``. + +An overlay is named by the agent it is for, in lower case (``AGENTS.claude.md``). A harness profile names +its overlay with its ``overlay`` key. The ``guidance:overlay`` arm keeps the overlay of that name and removes +the others, and the reach probe reports any foreign overlay a harness loaded anyway. +""" + +from __future__ import annotations + +import functools +import re +from pathlib import PurePosixPath + +from .harness_profile import builtin_profiles, load_profile + +OVERLAY_RE = re.compile(r"AGENTS\.([A-Za-z0-9][A-Za-z0-9-]*)\.md") + + +@functools.lru_cache(maxsize=1) +def native_names() -> frozenset[str]: + """Guidance names a builtin harness loads by its own convention (``AGENTS.override.md``): never an overlay.""" + return frozenset(n for name in builtin_profiles() for n in load_profile(name).guidance_names) + + +def overlay_of(path: str) -> str | None: + """The agent the guidance file ``path`` is an overlay for, ``None`` when it is not one. + + A file under ``.claude`` is skill content, not guidance, so it is never an overlay. + """ + parts = PurePosixPath(path).parts + if ".claude" in parts or parts[-1] in native_names(): + return None + match = OVERLAY_RE.fullmatch(parts[-1]) + return match.group(1).lower() if match else None diff --git a/dokimi/dokimi/repos/profiles/claude-code.yaml b/dokimi/dokimi/repos/profiles/claude-code.yaml index 54fdf12b2..7f19689dd 100644 --- a/dokimi/dokimi/repos/profiles/claude-code.yaml +++ b/dokimi/dokimi/repos/profiles/claude-code.yaml @@ -94,3 +94,4 @@ bridge: for: AGENTS.md write: CLAUDE.md content: "@AGENTS.md" +overlay: claude # AGENTS.claude.md, the guidance:overlay arm (agentsmd/agents.md#185) diff --git a/dokimi/dokimi/repos/profiles/codex-review.yaml b/dokimi/dokimi/repos/profiles/codex-review.yaml index 05dc3c8f7..a6231f192 100644 --- a/dokimi/dokimi/repos/profiles/codex-review.yaml +++ b/dokimi/dokimi/repos/profiles/codex-review.yaml @@ -57,3 +57,4 @@ guidance: it opened AGENTS.md but reported only defects its P0-P3 rubric keeps; convention rules moved it only when the guidance called a violation merge-blocking (P1). It reports zero token usage in this mode. +overlay: codex # AGENTS.codex.md, the guidance:overlay arm (agentsmd/agents.md#185) diff --git a/dokimi/dokimi/repos/profiles/codex.yaml b/dokimi/dokimi/repos/profiles/codex.yaml index d35539e05..c559a43c6 100644 --- a/dokimi/dokimi/repos/profiles/codex.yaml +++ b/dokimi/dokimi/repos/profiles/codex.yaml @@ -61,3 +61,4 @@ guidance: Measured on 0.160 with gpt-6-luna (oneDAL review task): root AGENTS.md applied, subtree AGENTS.md applied only after the agent printed them with a shell command; AGENTS.override.md hid the root AGENTS.md. +overlay: codex # AGENTS.codex.md, the guidance:overlay arm (agentsmd/agents.md#185) diff --git a/dokimi/dokimi/repos/profiles/copilot-review.yaml b/dokimi/dokimi/repos/profiles/copilot-review.yaml index 0427aa917..8bb8da14d 100644 --- a/dokimi/dokimi/repos/profiles/copilot-review.yaml +++ b/dokimi/dokimi/repos/profiles/copilot-review.yaml @@ -76,3 +76,4 @@ guidance: REVIEW.md included. Measured with /review: the same files reached it, and the existing oneDAL guidance did not change its recall until the review rules sat in the root AGENTS.md. +overlay: copilot # AGENTS.copilot.md, the guidance:overlay arm (agentsmd/agents.md#185) diff --git a/dokimi/dokimi/repos/profiles/copilot.yaml b/dokimi/dokimi/repos/profiles/copilot.yaml index b9bcce53c..121bd4a56 100644 --- a/dokimi/dokimi/repos/profiles/copilot.yaml +++ b/dokimi/dokimi/repos/profiles/copilot.yaml @@ -73,3 +73,4 @@ guidance: .github/instructions files are probed through the manifest's globs. The hosted Copilot code review (on a pull request) reads a different set, REVIEW.md included. +overlay: copilot # AGENTS.copilot.md, the guidance:overlay arm (agentsmd/agents.md#185) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 599b48451..058921d77 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -51,6 +51,7 @@ from .transcript import Transcript from .harness_profile import HarnessProfile from .manifest import RepoManifest, check_repo +from .overlays import overlay_of from .runner import SUPPORT_HOST from .source import RepoTree, SourceError @@ -58,6 +59,7 @@ PROBE_FILE = "REACH_PROBE.txt" PROSE_LIMIT = 8 CLASSES = ("loaded", "truncated", "fetched", "not_loaded", "error") +SEEN = ("loaded", "truncated", "fetched") # the classes that mean the harness had (some of) the file # Names some harness loads by convention. Any of them in an ancestor of the # workspace would be read by the harness and pollute every scenario. ANCESTOR_GUARD = ("CLAUDE.md", "CLAUDE.local.md", ".claude/CLAUDE.md", "AGENTS.md", "AGENT.md", @@ -113,7 +115,31 @@ def probe_files(tree: RepoTree, manifest: RepoManifest, profile: HarnessProfile) declared = set(check_repo(tree, manifest).guidance_files) names = set(profile.guidance_names) native = {p for p in tree.files() if PurePosixPath(p).name in names} - return sorted(p for p in declared | native if not manifest.in_eval_dir(p)) + # Every overlay, even where the globs miss it: whether a foreign one loads is the question (repos/overlays.py) + overlays = {p for p in tree.files() if overlay_of(p) is not None} + return sorted(p for p in declared | native | overlays if not manifest.in_eval_dir(p)) + + +def overlay_report(records: list[dict[str, Any]], own: str | None, + links: dict[str, tuple[str, ...]]) -> dict[str, Any]: + """The overlay part of the payload: which of the ``own`` overlay files loaded, and which foreign overlays did + (``cross_loaded``, a finding: the harness read guidance written for another agent). ``links`` maps each probed + file to the links that name it: a file reached through an overlay-named link is an overlay under that name.""" + names = {target: (target, *aliases) for target, aliases in links.items()} + own_files = {n for ns in names.values() for n in ns if own and overlay_of(n) == own} + foreign = {n for ns in names.values() for n in ns if overlay_of(n) not in (None, own)} + cross: dict[str, dict[str, str]] = {} + own_loaded: set[str] = set() + for r in records: + if r["class"] not in SEEN: + continue + for n in names.get(r["file"], ()): + if n in foreign: + cross.setdefault(n, {})[r["scenario"]] = r["class"] + if n in own_files: + own_loaded.add(n) + return {"own": own, "own_files": sorted(own_files), "own_loaded": sorted(own_loaded), + "foreign_files": sorted(foreign), "cross_loaded": dict(sorted(cross.items()))} def scenarios(files: list[str], kinds: tuple[str, ...] = ("root", "cwd", "read"), @@ -357,6 +383,7 @@ def run_reach( "unresolved": unresolved, "scenarios": scenario_rows, "records": records, + "overlays": overlay_report(records, profile.overlay, {s.path: s.aliases for s in seeded}), "summary": { "cost_usd": round(sum(r.get("cost_usd") or 0 for r in scenario_rows), 6), "by_class": by_class, @@ -420,6 +447,12 @@ def format_matrix(payload: dict[str, Any]) -> str: row = payload["summary"]["matrix"].get(f["path"], {}) lines.append(f"{label[f['path']]:<{width}} " + " ".join(f"{short[row.get(s, 'error')]:>2}" for s in ids)) lines.append("L loaded T truncated F fetched (after a tool read) . not loaded E error") + overlays = payload.get("overlays") or {} + for path, seen in (overlays.get("cross_loaded") or {}).items(): + where = ", ".join(f"{sid} ({cls})" for sid, cls in seen.items()) + lines.append(f"finding: {path}, an overlay for another agent, was loaded by {payload['harness']['name']} in {where}") + if overlays.get("own_files") and not overlays.get("own_loaded"): + lines.append(f"note: no {overlays['own']} overlay ({', '.join(overlays['own_files'])}) was loaded in any scenario") if any(f.get("alias_kind") == "copy" for f in payload["files"]): lines.append("note: the paths after = are copies, not links (this OS refused a symlink); " "a harness that skips symlinks would still load them") diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index b5e0dba74..d6d6fe8b9 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -12,7 +12,7 @@ 3. ``git init`` and commit (no history, so an upstream fix cannot leak) 4. apply ``setup_patches`` 5. apply the arm (``guidance:none`` | ``guidance:raw`` | ``guidance:bridge`` | - ``guidance:rev:`` | ``patch:@``, see :class:`Arm`), then its + ``guidance:overlay`` | ``guidance:rev:`` | ``patch:@``, see :class:`Arm`), then its skills into ``.claude/skills/`` (:mod:`.arms`) 6. amend the commit 7. apply ``review_patch`` with ``git am`` under a neutral author @@ -53,10 +53,12 @@ from .errors import PrepError from .harness_profile import HarnessProfile from .manifest import RepoManifest, _is_within, match_glob +from .overlays import overlay_of from .source import GitTree, RepoSource, RepoTree, SourceError, cache_path, default_cache_dir, resolve_rev from .uptake import MCP_CONFIG, base_servers, mcp_servers -ARMS = ("guidance:none", "guidance:raw", "guidance:bridge") # the guidance parts +ARMS = ("guidance:none", "guidance:raw", "guidance:bridge", "guidance:overlay") # the guidance parts +OVERLAY_ARM = "guidance:overlay" # base + the profile's AGENTS..md, other overlays removed (repos/overlays.py) REV_ARM, PATCH_ARM = "guidance:rev:", "patch:" #: Every form ``repo run --arm`` takes ARM_FORMS = (*ARMS, f"{REV_ARM}", f"{PATCH_ARM}", "skill:", "skill-catalog:") @@ -364,6 +366,10 @@ def prepare_workspace(git_dir: Path, manifest: RepoManifest, task: Task, arm: st if arm.base == "guidance:bridge" and not bridge: name = profile.name if profile is not None else "no harness" raise PrepError(f"guidance:bridge needs a harness profile with a `bridge` key; {name} has none") + overlay = profile.overlay if profile is not None else None + if arm.base == OVERLAY_ARM and not overlay: + name = profile.name if profile is not None else "no harness" + raise PrepError(f"guidance:overlay needs a harness profile with an `overlay` key; {name} has none") try: base_sha = resolve_rev(git_dir, task.base) except SourceError as exc: @@ -382,7 +388,7 @@ def prepare_workspace(git_dir: Path, manifest: RepoManifest, task: Task, arm: st _git(ws, env, "commit", "-q", "--no-gpg-sign", "--allow-empty", "--no-verify", "-m", "snapshot") patches = _apply_setup_patches(ws, env, task) # 4 try: - arm_files, arm_meta = _apply_arm(ws, env, git_dir, manifest, arm, bridge) # 5 + arm_files, arm_meta = _apply_arm(ws, env, git_dir, manifest, arm, bridge, overlay) # 5 except PrepError as exc: raise PrepError(f"task {task.id}: {exc}") from exc installed = _install_skills(ws, task, skills) @@ -446,14 +452,16 @@ def _forget(ws: Path, env: dict[str, str]) -> None: def _apply_arm(ws: Path, env: dict[str, str], git_dir: Path, manifest: RepoManifest, arm: Arm, - bridge: dict[str, str] | None) -> tuple[list[str], dict[str, Any]]: + bridge: dict[str, str] | None, overlay: str | None = None) -> tuple[list[str], dict[str, Any]]: """The files the arm removed (none), wrote (bridge, rev) or changed (patch), and its other - ``meta.json`` fields: bridge targets left alone, the revision, the patch file.""" + ``meta.json`` fields: bridge targets left alone, the revision, the patch file, the overlay kept.""" if arm.base == "guidance:raw": return [], {} if arm.patch is not None: return _apply_patch(ws, env, manifest, arm) tracked = [p for p in _git(ws, env, "ls-files", "-z").split("\0") if p] + if arm.base == OVERLAY_ARM: + return _apply_overlay(ws, tracked, overlay or "") matched = sorted(p for p in tracked if any(match_glob(g, p) for g in manifest.guidance_globs)) if arm.base == "guidance:none" or arm.rev_sha is not None: for rel in matched: @@ -522,6 +530,40 @@ def _apply_bridge(ws: Path, matched: list[str], bridge: dict[str, str]) -> tuple return written, ({"bridge_skipped": skipped} if skipped else {}) +def _canon(path: Path) -> Path: + """``path`` with its directories resolved and its final name kept, so a link is compared by its own name.""" + return Path(os.path.realpath(path.parent)) / path.name + + +def _link_hits(link: Path, gone: set[Path]) -> bool: + """Whether the symbolic link ``link`` names one of the canonical paths ``gone``, hop by hop: a shared final + target is no hit, a cycle ends the walk.""" + hop, seen = _canon(link), set() + while hop.is_symlink() and hop not in seen: + seen.add(hop) + hop = _canon(hop.parent / os.readlink(hop)) + if hop in gone: + return True + return False + + +def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], dict[str, Any]]: + """Remove the overlays of other agents, keeping ``AGENTS.md`` and the ``own`` overlay; refused without one.""" + kept = sorted(p for p in tracked if overlay_of(p) == own) + if not kept: + raise PrepError(f"guidance:overlay: the base has no AGENTS.{own}.md to keep") + removed = sorted(p for p in tracked if overlay_of(p) not in (None, own)) + gone = {_canon(ws / rel) for rel in removed} + for rel in tracked: + if rel not in removed and _link_hits(ws / rel, gone): + raise PrepError(f"guidance:overlay: {rel} links to an overlay the arm removes; the base would dangle") + for rel in removed: + if (ws / rel).is_dir() and not (ws / rel).is_symlink(): + raise PrepError(f"guidance:overlay: {rel} is a directory, not an overlay file") + (ws / rel).unlink() + return removed, {"arm_overlay": {"name": own, "kept": kept}} + + def _guidance_at(git_dir: Path, manifest: RepoManifest, sha: str) -> tuple[list[str], dict[str, str]]: """The files of commit ``sha`` the guidance globs match, outside ``eval_dir`` (step 2 deletes it), and for each one that is a symbolic link the file it resolves to in that commit. A link out of the tree, into ``eval_dir`` diff --git a/dokimi/dokimi/schemas/guidance_reach.v1.json b/dokimi/dokimi/schemas/guidance_reach.v1.json index 35f84509b..f6372dbcc 100644 --- a/dokimi/dokimi/schemas/guidance_reach.v1.json +++ b/dokimi/dokimi/schemas/guidance_reach.v1.json @@ -46,6 +46,18 @@ } } }, + "overlays": { + "type": "object", + "description": "agentsmd/agents.md#185 overlays. own: the harness's overlay name (null: none); own_files: AGENTS..md files probed; own_loaded: those loaded in some scenario; foreign_files: AGENTS..md files probed; cross_loaded: foreign file -> scenario -> loaded, truncated or fetched. Any cross_loaded entry is a finding.", + "required": ["own", "own_files", "own_loaded", "foreign_files", "cross_loaded"], + "properties": { + "own": {"type": ["string", "null"]}, + "own_files": {"type": "array", "items": {"type": "string"}}, + "own_loaded": {"type": "array", "items": {"type": "string"}}, + "foreign_files": {"type": "array", "items": {"type": "string"}}, + "cross_loaded": {"type": "object", "additionalProperties": {"type": "object", "additionalProperties": {"enum": ["loaded", "truncated", "fetched"]}}} + } + }, "unresolved": { "type": "array", "description": "Probed paths that are symlinks the probe cannot follow inside the tree (escaping, dangling, looping or naming a directory). They are not seeded and have no records.", diff --git a/dokimi/dokimi/schemas/harness_profile.v1.json b/dokimi/dokimi/schemas/harness_profile.v1.json index 956b85b2d..a6ee8f26f 100644 --- a/dokimi/dokimi/schemas/harness_profile.v1.json +++ b/dokimi/dokimi/schemas/harness_profile.v1.json @@ -76,6 +76,11 @@ "write": {"type": "string"}, "content": {"type": "string"} } + }, + "overlay": { + "type": ["string", "null"], + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "The agent name of the AGENTS..md the guidance:overlay arm keeps for this harness (agentsmd/agents.md#185). Absent or null: the arm cannot run with this profile." } } } diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py new file mode 100644 index 000000000..1f61f3dae --- /dev/null +++ b/dokimi/tests/test_repo_overlay.py @@ -0,0 +1,250 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The guidance:overlay arm and the overlay reach check (agentsmd/agents.md#185); offline, no LLM.""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest +import yaml + +from dokimi.repos import RepoSource, load_manifest, open_source +from dokimi.repos.harness_profile import PROFILE_DIR, ProfileError, load_profile, parse_profile +from dokimi.repos.overlays import overlay_of +from dokimi.repos.reach import format_matrix, overlay_report, probe_files +from dokimi.repos.workspace import ( + ARM_FORMS, + PrepError, + _apply_overlay, + export_eval_dir, + load_tasks, + parse_arm, + prepare_workspace, + source_git_dir, +) +from repo_eval_fixture import git, make_repo, write + +OVERLAY_FILES = { + "AGENTS.md": "# guidance\n", + "AGENTS.claude.md": "# for claude\n", + "AGENTS.codex.md": "# for codex\n", + "sub/AGENTS.md": "# sub guidance\n", + "sub/AGENTS.copilot.md": "# sub, for copilot\n", +} + + +def _overlay_base(root: Path, base: str) -> str: + """A commit on ``base`` that adds the overlay files; the tasks move onto it, and HEAD holds the overlays too.""" + wt = root.parent / "overlay-wt" + git(root, "worktree", "add", "-q", "-b", "overlays", str(wt), base) + for rel, text in OVERLAY_FILES.items(): + write(wt, rel, text) + git(wt, "add", "-A") + git(wt, "commit", "-qm", "overlays") + sha = git(wt, "rev-parse", "HEAD") + git(root, "worktree", "remove", "--force", str(wt)) + for task in ("fix_add", "review_add"): + path = root / ".agent-eval" / "tasks" / task / "task.json" + data = json.loads(path.read_text(encoding="utf-8")) + data["base"] = sha + path.write_text(json.dumps(data), encoding="utf-8") + for rel, text in OVERLAY_FILES.items(): + write(root, rel, text) + git(root, "add", "-A") + git(root, "commit", "-qm", "tasks on the overlay base") + return sha + + +@pytest.fixture +def repo(tmp_path): + root, base = make_repo(tmp_path) + _overlay_base(root, base) + source = RepoSource.parse(f"{root}@HEAD") + tree = open_source(source) + manifest = load_manifest(tree, ".agent-eval/repo-eval.yaml") + content = export_eval_dir(tree, manifest, tmp_path / "content") + tasks = {t.id: t for t in load_tasks(content, manifest)} + return {"root": root, "tree": tree, "manifest": manifest, "tasks": tasks, + "git_dir": source_git_dir(source, tree), "tmp": tmp_path} + + +def _prep(repo, arm: str, profile, name: str = "rd"): + meta = prepare_workspace(repo["git_dir"], repo["manifest"], repo["tasks"]["fix_add"], arm, + repo["tmp"] / name, profile) + return meta, repo["tmp"] / name / "workspace" + + +@pytest.mark.parametrize("path, name", [ + ("AGENTS.claude.md", "claude"), + ("sub/AGENTS.CODEX.md", "codex"), + ("docs/AGENTS.copilot.md", "copilot"), + ("AGENTS.md", None), + ("AGENTS.override.md", None), # codex's own file, loaded by its own convention + ("CLAUDE.md", None), + ("AGENTS.a.b.md", None), + ("AGENTS.md.bak", None), + (".claude/skills/lint/AGENTS.codex.md", None), # skill content, not guidance +]) +def test_overlay_names_are_agent_files_beside_agents_md(path, name): + assert overlay_of(path) == name + + +def test_overlay_is_a_guidance_part_and_takes_skills(): + assert "guidance:overlay" in ARM_FORMS + assert parse_arm("guidance:overlay").base == "guidance:overlay" + arm = parse_arm("guidance:overlay+skill:skills/lint") + assert str(arm) == "guidance:overlay+skill:lint" and arm.base == "guidance:overlay" + with pytest.raises(PrepError, match="more than one guidance part"): + parse_arm("guidance:overlay+guidance:none+skill:skills/lint") + + +@pytest.mark.parametrize("profile, overlay", [("claude-code", "claude"), ("codex", "codex"), + ("codex-review", "codex"), ("copilot", "copilot"), + ("copilot-review", "copilot")]) +def test_builtin_profiles_name_the_overlay_of_their_agent(profile, overlay): + assert load_profile(profile).overlay == overlay + + +def test_overlay_key_is_a_lowercase_agent_name(): + raw = yaml.safe_load((PROFILE_DIR / "codex.yaml").read_text(encoding="utf-8")) + assert parse_profile(raw).overlay == "codex" + with pytest.raises(ProfileError): + parse_profile({**raw, "overlay": "Codex/../x"}) + with pytest.raises(ProfileError): # the schema's `$` takes a trailing newline; the parser must not + parse_profile({**raw, "overlay": "codex\n"}) + + +def test_overlay_arm_keeps_base_and_the_harness_overlay(repo): + meta, ws = _prep(repo, "guidance:overlay", load_profile("claude-code")) + assert meta["arm"] == "guidance:overlay" + assert (ws / "AGENTS.md").read_text() == "# guidance\n" + assert (ws / "AGENTS.claude.md").read_text() == "# for claude\n" + assert (ws / "sub/AGENTS.md").is_file() + assert not (ws / "AGENTS.codex.md").exists() and not (ws / "sub/AGENTS.copilot.md").exists() + assert meta["arm_files"] == ["AGENTS.codex.md", "sub/AGENTS.copilot.md"] + assert meta["arm_overlay"] == {"name": "claude", "kept": ["AGENTS.claude.md"]} + + +def test_overlay_arm_follows_the_profile_not_the_arm_name(repo): + meta, ws = _prep(repo, "guidance:overlay", load_profile("codex"), name="rd-codex") + assert (ws / "AGENTS.codex.md").is_file() and not (ws / "AGENTS.claude.md").exists() + assert meta["arm_files"] == ["AGENTS.claude.md", "sub/AGENTS.copilot.md"] + assert meta["arm_overlay"] == {"name": "codex", "kept": ["AGENTS.codex.md"]} + + +def test_raw_arm_keeps_every_overlay(repo): + meta, ws = _prep(repo, "guidance:raw", load_profile("claude-code"), name="rd-raw") + assert (ws / "AGENTS.codex.md").is_file() and (ws / "AGENTS.claude.md").is_file() + assert "arm_overlay" not in meta + + +def test_overlay_arm_needs_a_profile_with_an_overlay_key(repo): + with pytest.raises(PrepError, match="needs a harness profile with an `overlay` key"): + _prep(repo, "guidance:overlay", None, name="rd-none") + with pytest.raises(PrepError, match="needs a harness profile with an `overlay` key"): + _prep(repo, "guidance:overlay", dataclasses.replace(load_profile("claude-code"), overlay=None), + name="rd-no-key") + + +def test_overlay_arm_refuses_a_base_with_no_overlay_for_its_agent(repo): + with pytest.raises(PrepError, match="has no AGENTS.gemini.md to keep"): + _prep(repo, "guidance:overlay", dataclasses.replace(load_profile("claude-code"), overlay="gemini"), + name="rd-gemini") + + +def test_probe_files_include_every_overlay_even_outside_the_globs(repo): + """The manifest's globs are ``**/AGENTS.md`` and the like: an overlay is probed for its own reach.""" + tree = repo["tree"] + files = probe_files(tree, load_manifest(tree, ".agent-eval/repo-eval.yaml"), load_profile("claude-code")) + assert {"AGENTS.claude.md", "AGENTS.codex.md", "sub/AGENTS.copilot.md"} <= set(files) + assert all(not f.startswith(".agent-eval/") for f in files) + + +def test_overlay_report_flags_a_foreign_overlay_that_loads(): + records = [ + {"file": "AGENTS.md", "scenario": "cwd=.", "class": "loaded"}, + {"file": "AGENTS.claude.md", "scenario": "cwd=.", "class": "loaded"}, + {"file": "AGENTS.codex.md", "scenario": "cwd=.", "class": "not_loaded"}, + {"file": "sub/AGENTS.copilot.md", "scenario": "cwd=sub", "class": "fetched"}, + {"file": "sub/AGENTS.copilot.md", "scenario": "cwd=.", "class": "error"}, + ] + paths = ["AGENTS.md", "AGENTS.claude.md", "AGENTS.codex.md", "sub/AGENTS.copilot.md"] + assert overlay_report(records, "claude", {p: () for p in paths}) == { + "own": "claude", "own_files": ["AGENTS.claude.md"], "own_loaded": ["AGENTS.claude.md"], + "foreign_files": ["AGENTS.codex.md", "sub/AGENTS.copilot.md"], + "cross_loaded": {"sub/AGENTS.copilot.md": {"cwd=sub": "fetched"}}, + } + + +def test_a_harness_without_an_overlay_has_only_foreign_overlays(): + records = [{"file": "AGENTS.codex.md", "scenario": "cwd=.", "class": "truncated"}] + report = overlay_report(records, None, {"AGENTS.md": (), "AGENTS.codex.md": ()}) + assert report["own_files"] == [] and report["cross_loaded"] == {"AGENTS.codex.md": {"cwd=.": "truncated"}} + + +def test_overlay_report_flags_a_foreign_overlay_reached_through_a_link(): + """The probe seeds the file the link names; the harness reads the link, so the report keys on the link name.""" + records = [{"file": "docs/codex.md", "scenario": "cwd=.", "class": "loaded"}] + report = overlay_report(records, "claude", {"docs/codex.md": ("AGENTS.codex.md",)}) + assert report["foreign_files"] == ["AGENTS.codex.md"] + assert report["cross_loaded"] == {"AGENTS.codex.md": {"cwd=.": "loaded"}} + + +def test_overlay_arm_refuses_a_base_link_into_an_overlay_it_removes(tmp_path): + (tmp_path / "AGENTS.md").symlink_to("AGENTS.codex.md") + (tmp_path / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + with pytest.raises(PrepError, match="the base would dangle"): + _apply_overlay(tmp_path, ["AGENTS.md", "AGENTS.codex.md", "AGENTS.claude.md"], "claude") + assert (tmp_path / "AGENTS.codex.md").exists() # refused before anything is removed + + +def test_overlay_arm_keeps_a_base_link_whose_target_a_removed_link_also_names(tmp_path): + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text("shared\n", encoding="utf-8") + (tmp_path / "AGENTS.md").symlink_to("docs/shared.md") + (tmp_path / "AGENTS.codex.md").symlink_to("docs/shared.md") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + removed, _ = _apply_overlay(tmp_path, ["AGENTS.md", "AGENTS.codex.md", "AGENTS.claude.md", "docs/shared.md"], + "claude") + assert removed == ["AGENTS.codex.md"] + assert (tmp_path / "AGENTS.md").read_text(encoding="utf-8") == "shared\n" + + +def test_overlay_arm_refuses_a_base_link_through_a_directory_link(tmp_path): + (tmp_path / "real").mkdir() + (tmp_path / "real" / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + (tmp_path / "dir").symlink_to("real") + (tmp_path / "AGENTS.md").symlink_to("dir/AGENTS.codex.md") + with pytest.raises(PrepError, match="the base would dangle"): + _apply_overlay(tmp_path, ["AGENTS.md", "dir", "real/AGENTS.codex.md", "AGENTS.claude.md"], "claude") + assert (tmp_path / "real" / "AGENTS.codex.md").exists() + + +def test_overlay_arm_does_not_hang_on_a_symlink_loop_in_the_base(tmp_path): + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "a").symlink_to("b") + (tmp_path / "docs" / "b").symlink_to("a") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + (tmp_path / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + removed, _ = _apply_overlay(tmp_path, ["AGENTS.codex.md", "AGENTS.claude.md", "docs/a", "docs/b"], "claude") + assert removed == ["AGENTS.codex.md"] + + +def test_text_report_states_the_cross_load_as_a_finding(): + payload = { + "harness": {"name": "claude-code", "version": "2.1.0"}, "model": "m", + "summary": {"cost_usd": 0.0, "matrix": {}}, "files": [], "unresolved": [], + "scenarios": [{"id": "cwd=.", "error": None, "prose": 0, "unknown": []}], + "overlays": {"own": "claude", "own_files": ["AGENTS.claude.md"], "own_loaded": [], + "foreign_files": ["AGENTS.codex.md"], "cross_loaded": {"AGENTS.codex.md": {"cwd=.": "loaded"}}}, + } + text = format_matrix(payload) + assert ("finding: AGENTS.codex.md, an overlay for another agent, was loaded by claude-code in cwd=. (loaded)" + in text) + assert "note: no claude overlay (AGENTS.claude.md) was loaded in any scenario" in text From ee38237f8a9b01e31c205a9dfbfdb5223d7c5625 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:20:42 -0700 Subject: [PATCH 02/14] Address review: overlay arm link walk, kept overlays, setup-patch files The arm walks each path component before resolving it, so a removed directory link in the middle of a base link is a hit; an own overlay must resolve to a file inside the workspace; the arm lists the workspace (index and untracked files), so a setup patch applied with git apply is seen. Thread: PRRT_kwDOVBg7g86yWjv2 Thread: PRRT_kwDOVBg7g86yWjwG Thread: PRRT_kwDOVBg7g86yWj4D Thread: PRRT_kwDOVBg7g86yWj4E Thread: PRRT_kwDOVBg7g86yWj4G Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/workspace.py | 52 +++++++++++++++++++++++++------ dokimi/tests/test_repo_overlay.py | 46 +++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index d6d6fe8b9..7fd530b7e 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -459,9 +459,9 @@ def _apply_arm(ws: Path, env: dict[str, str], git_dir: Path, manifest: RepoManif return [], {} if arm.patch is not None: return _apply_patch(ws, env, manifest, arm) - tracked = [p for p in _git(ws, env, "ls-files", "-z").split("\0") if p] if arm.base == OVERLAY_ARM: - return _apply_overlay(ws, tracked, overlay or "") + return _apply_overlay(ws, _workspace_files(ws, env), overlay or "") + tracked = [p for p in _git(ws, env, "ls-files", "-z").split("\0") if p] matched = sorted(p for p in tracked if any(match_glob(g, p) for g in manifest.guidance_globs)) if arm.base == "guidance:none" or arm.rev_sha is not None: for rel in matched: @@ -535,15 +535,45 @@ def _canon(path: Path) -> Path: return Path(os.path.realpath(path.parent)) / path.name +_MAX_HOPS = 64 # symbolic links followed by one walk; a cycle runs out + + +def _workspace_files(ws: Path, env: dict[str, str]) -> list[str]: + """The paths the workspace holds now: the indexed ones and the untracked ones (ignored ones too), since a setup + patch applied with ``git apply`` changes the tree and not the index. A path that is gone is no path.""" + listed = _git(ws, env, "ls-files", "-z", "--cached", "--others").split("\0") + return sorted({p for p in listed if p and os.path.lexists(ws / p)}) + + def _link_hits(link: Path, gone: set[Path]) -> bool: - """Whether the symbolic link ``link`` names one of the canonical paths ``gone``, hop by hop: a shared final - target is no hit, a cycle ends the walk.""" - hop, seen = _canon(link), set() - while hop.is_symlink() and hop not in seen: - seen.add(hop) - hop = _canon(hop.parent / os.readlink(hop)) - if hop in gone: + """Whether resolving the symbolic link ``link`` passes through one of the canonical paths ``gone``. Each path + component is checked before it is resolved, so a removed directory link named in the middle of the path is a + hit as well; a link is followed hop by hop, and a cycle ends the walk.""" + resolved = Path(os.path.realpath(link.parent)) + todo = [link.name] + hops = 0 + while todo: + part = todo.pop(0) + if part in ("", "."): + continue + if part == "..": + resolved = resolved.parent + continue + cand = resolved / part + if cand in gone: return True + if not cand.is_symlink(): + resolved = cand + continue + hops += 1 + if hops > _MAX_HOPS: + return False + target = PurePosixPath(os.readlink(cand)) + if target.is_absolute(): + resolved, parts = Path(target.anchor), target.parts[1:] + else: + parts = target.parts + todo[:0] = list(parts) return False @@ -552,6 +582,10 @@ def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], d kept = sorted(p for p in tracked if overlay_of(p) == own) if not kept: raise PrepError(f"guidance:overlay: the base has no AGENTS.{own}.md to keep") + for rel in kept: # the arm keeps it, so it must supply a file: a dangling, cyclic or directory link does not + real = Path(os.path.realpath(ws / rel)) + if not real.is_file() or not real.is_relative_to(ws.resolve()): + raise PrepError(f"guidance:overlay: {rel} does not resolve to a usable file in the workspace") removed = sorted(p for p in tracked if overlay_of(p) not in (None, own)) gone = {_canon(ws / rel) for rel in removed} for rel in tracked: diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 1f61f3dae..214dff16a 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -18,8 +18,12 @@ from dokimi.repos.reach import format_matrix, overlay_report, probe_files from dokimi.repos.workspace import ( ARM_FORMS, + OVERLAY_ARM, + SNAPSHOT_AUTHOR, PrepError, + _apply_arm, _apply_overlay, + _git_env, export_eval_dir, load_tasks, parse_arm, @@ -248,3 +252,45 @@ def test_text_report_states_the_cross_load_as_a_finding(): assert ("finding: AGENTS.codex.md, an overlay for another agent, was loaded by claude-code in cwd=. (loaded)" in text) assert "note: no claude overlay (AGENTS.claude.md) was loaded in any scenario" in text + + +def test_overlay_arm_refuses_a_base_link_through_a_removed_directory_link(tmp_path): + """A removed overlay that is a directory link: the base link resolves through it, so it would dangle.""" + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "base.md").write_text("base\n", encoding="utf-8") + (tmp_path / "AGENTS.codex.md").symlink_to("docs") + (tmp_path / "AGENTS.md").symlink_to("AGENTS.codex.md/base.md") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + with pytest.raises(PrepError, match="the base would dangle"): + _apply_overlay(tmp_path, ["AGENTS.md", "AGENTS.codex.md", "AGENTS.claude.md", "docs/base.md"], "claude") + assert (tmp_path / "AGENTS.codex.md").is_symlink() # refused before anything is removed + + +@pytest.mark.parametrize("own_target", ["missing.md", "AGENTS.claude.md", "docs"]) +def test_overlay_arm_refuses_an_own_overlay_that_supplies_no_file(tmp_path, own_target): + (tmp_path / "docs").mkdir() + (tmp_path / "AGENTS.md").write_text("base\n", encoding="utf-8") + (tmp_path / "AGENTS.claude.md").symlink_to(own_target) # dangling, a cycle, a directory + (tmp_path / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + with pytest.raises(PrepError, match="does not resolve to a usable file"): + _apply_overlay(tmp_path, ["AGENTS.md", "AGENTS.claude.md", "AGENTS.codex.md", "docs"], "claude") + assert (tmp_path / "AGENTS.codex.md").exists() + + +def test_overlay_arm_reads_the_workspace_after_setup_patches_not_the_index(tmp_path): + """A setup patch applied with git apply changes the tree and not the index: the arm sees an added foreign + overlay (removes it) and a deleted one (skips it), instead of failing on a stale index entry.""" + ws = tmp_path / "ws" + ws.mkdir() + (ws / "AGENTS.md").write_text("base\n", encoding="utf-8") + (ws / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + (ws / "AGENTS.copilot.md").write_text("copilot\n", encoding="utf-8") + git(ws, "init", "-q") + git(ws, "add", "-A") + git(ws, "commit", "-q", "-m", "base") + (ws / "AGENTS.copilot.md").unlink() # a setup patch deleted it + (ws / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") # and added this one, unstaged + env = _git_env(SNAPSHOT_AUTHOR) + removed, _ = _apply_arm(ws, env, tmp_path, None, parse_arm(OVERLAY_ARM), None, "claude") + assert removed == ["AGENTS.codex.md"] + assert not (ws / "AGENTS.codex.md").exists() and (ws / "AGENTS.claude.md").exists() From a52e675531f870ec6cf44f93d917ea8fcb61a1e6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:20:42 -0700 Subject: [PATCH 03/14] Address review: leave a shared-target load unattributed in the overlay report A harness that loads a file of a native name, or a file that an own and a foreign overlay both name, cannot be pinned on one alias: those aliases are listed as ambiguous, not as own or cross loads. Thread: PRRT_kwDOVBg7g86yWj4A Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/reach.py | 59 ++++++++++++++++++++++++------- dokimi/tests/test_repo_overlay.py | 18 ++++++++++ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 058921d77..6a713648d 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -120,16 +120,29 @@ def probe_files(tree: RepoTree, manifest: RepoManifest, profile: HarnessProfile) return sorted(p for p in declared | native | overlays if not manifest.in_eval_dir(p)) -def overlay_report(records: list[dict[str, Any]], own: str | None, - links: dict[str, tuple[str, ...]]) -> dict[str, Any]: - """The overlay part of the payload: which of the ``own`` overlay files loaded, and which foreign overlays did - (``cross_loaded``, a finding: the harness read guidance written for another agent). ``links`` maps each probed - file to the links that name it: a file reached through an overlay-named link is an overlay under that name.""" - names = {target: (target, *aliases) for target, aliases in links.items()} - own_files = {n for ns in names.values() for n in ns if own and overlay_of(n) == own} - foreign = {n for ns in names.values() for n in ns if overlay_of(n) not in (None, own)} +def _pinned_names(links: dict[str, tuple[str, ...]], own: str | None, + native: set[str] | frozenset[str]) -> tuple[dict[str, tuple[str, ...]], set[str]]: + """Each probed file with the names a load of it can be pinned on, and the aliases that cannot be pinned.""" + names: dict[str, tuple[str, ...]] = {} + ambiguous: set[str] = set() + for target, aliases in links.items(): + kinds = {"own" if overlay_of(n) == own else "foreign" for n in aliases if overlay_of(n) is not None} + if aliases and (PurePosixPath(target).name in native or {"own", "foreign"} <= kinds): + ambiguous.update(aliases) + names[target] = (target,) + else: + names[target] = (target, *aliases) + return names, ambiguous + + +def _loads(records: list[dict[str, Any]], names: dict[str, tuple[str, ...]], links: dict[str, tuple[str, ...]], + ambiguous: set[str], foreign: set[str], own_files: set[str]) -> tuple[dict[str, dict[str, str]], set[str], + set[str]]: + """The foreign overlays and own overlay files that loaded, by the names they can be pinned on, and the + aliases whose file loaded without a way to tell which of them was read.""" cross: dict[str, dict[str, str]] = {} own_loaded: set[str] = set() + unpinned: set[str] = set() for r in records: if r["class"] not in SEEN: continue @@ -138,8 +151,29 @@ def overlay_report(records: list[dict[str, Any]], own: str | None, cross.setdefault(n, {})[r["scenario"]] = r["class"] if n in own_files: own_loaded.add(n) - return {"own": own, "own_files": sorted(own_files), "own_loaded": sorted(own_loaded), - "foreign_files": sorted(foreign), "cross_loaded": dict(sorted(cross.items()))} + unpinned.update(n for n in links.get(r["file"], ()) if n in ambiguous) + return cross, own_loaded, unpinned + + +def overlay_report(records: list[dict[str, Any]], own: str | None, + links: dict[str, tuple[str, ...]], native: set[str] | frozenset[str] = frozenset()) -> dict[str, Any]: + """The overlay part of the payload: which of the ``own`` overlay files loaded, and which foreign overlays did + (``cross_loaded``, a finding: the harness read guidance written for another agent). ``links`` maps each probed + file to the links that name it: a file reached through an overlay-named link is an overlay under that name. + + A load is pinned on a link only when no other name of its file could have been read: the harness reads a + file of a ``native`` name by itself, and an own and a foreign overlay can name one file. Those links are + ``ambiguous`` when the file loaded, and are neither own nor cross loads.""" + every = {n for target, aliases in links.items() for n in (target, *aliases)} + own_files = {n for n in every if own and overlay_of(n) == own} + foreign = {n for n in every if overlay_of(n) not in (None, own)} + names, ambiguous = _pinned_names(links, own, native) + cross, own_loaded, unpinned = _loads(records, names, links, ambiguous, foreign, own_files) + report: dict[str, Any] = {"own": own, "own_files": sorted(own_files), "own_loaded": sorted(own_loaded), + "foreign_files": sorted(foreign), "cross_loaded": dict(sorted(cross.items()))} + if unpinned: + report["ambiguous"] = sorted(unpinned) + return report def scenarios(files: list[str], kinds: tuple[str, ...] = ("root", "cwd", "read"), @@ -383,7 +417,8 @@ def run_reach( "unresolved": unresolved, "scenarios": scenario_rows, "records": records, - "overlays": overlay_report(records, profile.overlay, {s.path: s.aliases for s in seeded}), + "overlays": overlay_report(records, profile.overlay, {s.path: s.aliases for s in seeded}, + {PurePosixPath(n).name for n in profile.guidance_names}), "summary": { "cost_usd": round(sum(r.get("cost_usd") or 0 for r in scenario_rows), 6), "by_class": by_class, @@ -451,7 +486,7 @@ def format_matrix(payload: dict[str, Any]) -> str: for path, seen in (overlays.get("cross_loaded") or {}).items(): where = ", ".join(f"{sid} ({cls})" for sid, cls in seen.items()) lines.append(f"finding: {path}, an overlay for another agent, was loaded by {payload['harness']['name']} in {where}") - if overlays.get("own_files") and not overlays.get("own_loaded"): + if overlays.get("own_files") and not overlays.get("own_loaded") and not overlays.get("ambiguous"): lines.append(f"note: no {overlays['own']} overlay ({', '.join(overlays['own_files'])}) was loaded in any scenario") if any(f.get("alias_kind") == "copy" for f in payload["files"]): lines.append("note: the paths after = are copies, not links (this OS refused a symlink); " diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 214dff16a..5c77407d5 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -294,3 +294,21 @@ def test_overlay_arm_reads_the_workspace_after_setup_patches_not_the_index(tmp_p removed, _ = _apply_arm(ws, env, tmp_path, None, parse_arm(OVERLAY_ARM), None, "claude") assert removed == ["AGENTS.codex.md"] assert not (ws / "AGENTS.codex.md").exists() and (ws / "AGENTS.claude.md").exists() + + +def test_overlay_report_leaves_a_load_of_a_shared_target_unattributed(): + """A harness that loads AGENTS.md reads the file its AGENTS.claude.md and AGENTS.codex.md aliases name too: + neither alias is pinned as loaded, and the load is reported as ambiguous.""" + records = [{"file": "AGENTS.md", "scenario": "cwd=.", "class": "loaded"}] + links = {"AGENTS.md": ("AGENTS.claude.md", "AGENTS.codex.md")} + report = overlay_report(records, "claude", links, native={"AGENTS.md"}) + assert report["own_loaded"] == [] and report["cross_loaded"] == {} + assert report["ambiguous"] == ["AGENTS.claude.md", "AGENTS.codex.md"] + assert report["own_files"] == ["AGENTS.claude.md"] and report["foreign_files"] == ["AGENTS.codex.md"] + + +def test_overlay_report_leaves_a_load_of_an_own_and_a_foreign_alias_unattributed(): + records = [{"file": "docs/x.md", "scenario": "cwd=.", "class": "loaded"}] + report = overlay_report(records, "claude", {"docs/x.md": ("AGENTS.claude.md", "AGENTS.codex.md")}) + assert report["own_loaded"] == [] and report["cross_loaded"] == {} + assert report["ambiguous"] == ["AGENTS.claude.md", "AGENTS.codex.md"] From f442d5de8396b85a9dbe3631215daf94dd32bb20 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:20:42 -0700 Subject: [PATCH 04/14] Address review: an overlay cell is identified by its overlay A resumed batch reused an overlay cell that kept another agent's overlay, since the execution id did not carry the overlay. The id now includes the profile's overlay for cells of the overlay arm only; other cells keep their ids. Thread: PRRT_kwDOVBg7g86yWj4C Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/batch_ledger.py | 13 +++++++---- dokimi/dokimi/repos/runner.py | 18 +++++++++------ dokimi/tests/test_repo_overlay.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/dokimi/dokimi/repos/batch_ledger.py b/dokimi/dokimi/repos/batch_ledger.py index 1c7102d51..5eaeaabb9 100644 --- a/dokimi/dokimi/repos/batch_ledger.py +++ b/dokimi/dokimi/repos/batch_ledger.py @@ -70,9 +70,11 @@ def harness_sha256(path: Path) -> str: def execution_id(profile: HarnessProfile, tools: str, delegation: str = "native", harness_sha256: str | None = None, - harness_version: str | None = None, companion_sha256: dict[str, str | None] | None = None) -> str: + harness_version: str | None = None, companion_sha256: dict[str, str | None] | None = None, + overlay: str | None = None) -> str: """Short hash of how the agent is started: the profile's ``invoke`` template with its tool arguments, - environment, config, prompt suffix, egress, bridge, trace, answer format and companions. + environment, config, prompt suffix, egress, bridge, trace, answer format and companions, and for a cell of + the overlay arm the overlay it keeps (``overlay``; the other cells carry none, so their ids do not change). Two runs with the same id ran the same command line up to the per-cell values (prompt, model, caps); ``repo run`` reuses a finished cell only when its id matches. Includes the harness @@ -95,14 +97,17 @@ def execution_id(profile: HarnessProfile, tools: str, delegation: str = "native" if profile.bin_companions: key["bin_companions"] = (list(profile.bin_companions) if companion_sha256 is None else {n: companion_sha256.get(n) for n in profile.bin_companions}) + if overlay is not None: + key["overlay"] = overlay return hashlib.sha256(json.dumps(key, sort_keys=True).encode("utf-8")).hexdigest()[:12] def execution_record(profile: HarnessProfile, tools: str, delegation: str, harness_sha256: str, harness_version: str | None, - companion_sha256: dict[str, str | None] | None = None) -> dict[str, str]: + companion_sha256: dict[str, str | None] | None = None, + overlay: str | None = None) -> dict[str, str]: """How a cell's agent was started, as ``meta.json`` and ``result.json`` record it.""" - return {"id": execution_id(profile, tools, delegation, harness_sha256, harness_version, companion_sha256), + return {"id": execution_id(profile, tools, delegation, harness_sha256, harness_version, companion_sha256, overlay), "tools": tools, "delegation": profile.effective_delegation(tools, delegation)} diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a0a122128..7a838990c 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -76,7 +76,7 @@ from .host import probe_host, unmet, validate_tasks from .manifest import match_glob, scratch_name from .process_metrics import process_metrics -from .workspace import PrepError, Task, prepare_workspace +from .workspace import OVERLAY_ARM, PrepError, Task, parse_arm, prepare_workspace SUPPORT_HOST = Path(__file__).resolve().parent / "support" CONFIG_DIR = f"{RUN_DIR}/config" @@ -99,22 +99,26 @@ class AgentOptions: budget_usd: float = 2.0 -def execution_record(opts: AgentOptions) -> dict[str, str]: +def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, str]: """How the cell's agent was started, as ``meta.json`` and ``result.json`` record it. The executable is digested for each cell, not once per batch: a cell records the bytes it actually ran, so a - binary replaced mid-batch shows up as cells of two executions, which the next run refuses.""" + binary replaced mid-batch shows up as cells of two executions, which the next run refuses. A cell of the + overlay arm also records the profile's overlay: the arm keeps that overlay's file, so another one is another run.""" beside = opts.harness_bin.resolve().parent # where harness_mounts takes the companions from companions = {n: harness_sha256(beside / n) if (beside / n).is_file() else None for n in opts.profile.bin_companions} + overlay = opts.profile.overlay if arm is not None and parse_arm(arm).base == OVERLAY_ARM else None return batch_ledger.execution_record(opts.profile, opts.tools, opts.delegation, harness_sha256(opts.harness_bin), - opts.harness_version, companions) + opts.harness_version, companions, overlay) def check_batch_executions(batch_dir: Path, options: list[AgentOptions], replacing: set[str] | None = None) -> None: """Refuse, before anything runs, a batch dir whose finished cells ran with another execution than ``options`` - (see :func:`batch_ledger.check_batch_executions`).""" - batch_ledger.check_batch_executions(batch_dir, {execution_record(o)["id"] for o in options}, replacing) + (see :func:`batch_ledger.check_batch_executions`). Each option's plain cells and its overlay cells are both + this execution, so a batch may hold either kind of cell of one profile.""" + wanted = {execution_record(o, a)["id"] for o in options for a in (None, OVERLAY_ARM)} + batch_ledger.check_batch_executions(batch_dir, wanted, replacing) def agent_network(opts: AgentOptions, ctx: EvalContext, task: Task) -> str: @@ -200,7 +204,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, ``host`` (the probed facts of :func:`.host.probe_host`) is recorded in ``meta.json``.""" rd = batch_dir / cell_name(task.id, arm, opts.model, rep) - execution = execution_record(opts) + execution = execution_record(opts, arm) done = batch_ledger.reused_result(rd, execution, force) if done is not None: return {**done, "skipped": True} diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 5c77407d5..680ec4d3b 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -312,3 +312,39 @@ def test_overlay_report_leaves_a_load_of_an_own_and_a_foreign_alias_unattributed report = overlay_report(records, "claude", {"docs/x.md": ("AGENTS.claude.md", "AGENTS.codex.md")}) assert report["own_loaded"] == [] and report["cross_loaded"] == {} assert report["ambiguous"] == ["AGENTS.claude.md", "AGENTS.codex.md"] + + +def test_an_overlay_cell_is_identified_by_the_overlay_it_keeps(tmp_path): + """A resumed batch must not reuse an overlay cell that kept another agent's overlay; other cells keep their ids.""" + from dokimi.repos.runner import AgentOptions, ExecutionMismatch, check_batch_executions, execution_record + + harness = tmp_path / "harness" + harness.write_bytes(b"harness\n") + claude = load_profile("claude-code") + codex = dataclasses.replace(claude, overlay="codex") + + def options(profile): + return AgentOptions(profile=profile, model="m", harness_bin=harness) + + def cell_id(profile, arm): + return execution_record(options(profile), arm)["id"] + + assert cell_id(claude, "guidance:overlay") != cell_id(codex, "guidance:overlay") + assert cell_id(claude, "guidance:none") == cell_id(codex, "guidance:none") + assert cell_id(claude, "guidance:overlay") == execution_record(options(claude), "guidance:overlay")["id"] + + batch = tmp_path / "batch" + cell = batch / "task__guidance-overlay__m__r1" + cell.mkdir(parents=True) + (cell / "result.json").write_text( + json.dumps({"execution": {"id": cell_id(claude, "guidance:overlay"), "tools": "all", "delegation": "native"}}), + encoding="utf-8") + check_batch_executions(batch, [options(claude)]) # the same execution: no refusal + with pytest.raises(ExecutionMismatch, match="another execution"): + check_batch_executions(batch, [options(codex)]) + plain = batch / "task__guidance-none__m__r1" + plain.mkdir() + (plain / "result.json").write_text( + json.dumps({"execution": {"id": cell_id(claude, "guidance:none"), "tools": "all", "delegation": "native"}}), + encoding="utf-8") + check_batch_executions(batch, [options(claude)]) # a plain cell beside an overlay cell of one profile From 7d4fd3c5fbb32a86128dfe3c21a5c5b055c62c5c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:25:27 -0700 Subject: [PATCH 05/14] Address review: document ambiguous overlays in the reach schema The overlay report's ambiguous key was not in the guidance_reach schema. Thread: PRRT_kwDOVBg7g86yWj4A Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/workspace.py | 3 ++- dokimi/dokimi/schemas/guidance_reach.v1.json | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 7fd530b7e..bf18589c8 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -588,8 +588,9 @@ def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], d raise PrepError(f"guidance:overlay: {rel} does not resolve to a usable file in the workspace") removed = sorted(p for p in tracked if overlay_of(p) not in (None, own)) gone = {_canon(ws / rel) for rel in removed} + removed_set = set(removed) for rel in tracked: - if rel not in removed and _link_hits(ws / rel, gone): + if rel not in removed_set and (ws / rel).is_symlink() and _link_hits(ws / rel, gone): raise PrepError(f"guidance:overlay: {rel} links to an overlay the arm removes; the base would dangle") for rel in removed: if (ws / rel).is_dir() and not (ws / rel).is_symlink(): diff --git a/dokimi/dokimi/schemas/guidance_reach.v1.json b/dokimi/dokimi/schemas/guidance_reach.v1.json index f6372dbcc..c4bf69f69 100644 --- a/dokimi/dokimi/schemas/guidance_reach.v1.json +++ b/dokimi/dokimi/schemas/guidance_reach.v1.json @@ -48,14 +48,15 @@ }, "overlays": { "type": "object", - "description": "agentsmd/agents.md#185 overlays. own: the harness's overlay name (null: none); own_files: AGENTS..md files probed; own_loaded: those loaded in some scenario; foreign_files: AGENTS..md files probed; cross_loaded: foreign file -> scenario -> loaded, truncated or fetched. Any cross_loaded entry is a finding.", + "description": "agentsmd/agents.md#185 overlays. own: the harness's overlay name (null: none); own_files: AGENTS..md files probed; own_loaded: those loaded in some scenario; foreign_files: AGENTS..md files probed; cross_loaded: foreign file -> scenario -> loaded, truncated or fetched. Any cross_loaded entry is a finding. ambiguous (only when some): overlay aliases whose file loaded but could not be told apart from the name it shares (a native name, or an own and a foreign overlay); they are neither own nor cross loads.", "required": ["own", "own_files", "own_loaded", "foreign_files", "cross_loaded"], "properties": { "own": {"type": ["string", "null"]}, "own_files": {"type": "array", "items": {"type": "string"}}, "own_loaded": {"type": "array", "items": {"type": "string"}}, "foreign_files": {"type": "array", "items": {"type": "string"}}, - "cross_loaded": {"type": "object", "additionalProperties": {"type": "object", "additionalProperties": {"enum": ["loaded", "truncated", "fetched"]}}} + "cross_loaded": {"type": "object", "additionalProperties": {"type": "object", "additionalProperties": {"enum": ["loaded", "truncated", "fetched"]}}}, + "ambiguous": {"type": "array", "items": {"type": "string"}} } }, "unresolved": { From 4f33efca15fdadc6c40b2ca339897d095b8fc3cd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:28:35 -0700 Subject: [PATCH 06/14] Address review: pin no load on an ambiguous overlay link A foreign target with an own alias was counted as an own load and a cross load of the target. The ambiguity check now looks at the target too, and an ambiguous link pins no name, as the docstring says. Thread: PRRT_kwDOVBg7g86yWj4A Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/reach.py | 4 ++-- dokimi/tests/test_repo_overlay.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 6a713648d..85934a05c 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -126,10 +126,10 @@ def _pinned_names(links: dict[str, tuple[str, ...]], own: str | None, names: dict[str, tuple[str, ...]] = {} ambiguous: set[str] = set() for target, aliases in links.items(): - kinds = {"own" if overlay_of(n) == own else "foreign" for n in aliases if overlay_of(n) is not None} + kinds = {"own" if overlay_of(n) == own else "foreign" for n in (target, *aliases) if overlay_of(n) is not None} if aliases and (PurePosixPath(target).name in native or {"own", "foreign"} <= kinds): ambiguous.update(aliases) - names[target] = (target,) + names[target] = () else: names[target] = (target, *aliases) return names, ambiguous diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 680ec4d3b..89a44665e 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -348,3 +348,10 @@ def cell_id(profile, arm): json.dumps({"execution": {"id": cell_id(claude, "guidance:none"), "tools": "all", "delegation": "native"}}), encoding="utf-8") check_batch_executions(batch, [options(claude)]) # a plain cell beside an overlay cell of one profile + + +def test_overlay_report_leaves_a_load_of_a_foreign_target_with_an_own_alias_unattributed(): + records = [{"file": "AGENTS.codex.md", "scenario": "cwd=.", "class": "loaded"}] + report = overlay_report(records, "claude", {"AGENTS.codex.md": ("AGENTS.claude.md",)}) + assert report["cross_loaded"] == {} and report["own_loaded"] == [] + assert report["ambiguous"] == ["AGENTS.claude.md"] From 3d7334b99c62cf8a92bbf88453b5ae6cef83fdb6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 23:40:42 -0700 Subject: [PATCH 07/14] fix(overlay): a native name in a link group pins nothing; readlink targets use native separators A shared target reached through a native alias (CLAUDE.md) is not pinned to the foreign alias. A Windows link target with backslashes splits into names when read as a native path. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/reach.py | 3 ++- dokimi/dokimi/repos/workspace.py | 2 +- dokimi/tests/test_repo_overlay.py | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 85934a05c..5ea371229 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -127,7 +127,8 @@ def _pinned_names(links: dict[str, tuple[str, ...]], own: str | None, ambiguous: set[str] = set() for target, aliases in links.items(): kinds = {"own" if overlay_of(n) == own else "foreign" for n in (target, *aliases) if overlay_of(n) is not None} - if aliases and (PurePosixPath(target).name in native or {"own", "foreign"} <= kinds): + # a native name anywhere in the group (the target or an alias) cannot tell which file a harness read + if aliases and (any(PurePosixPath(n).name in native for n in (target, *aliases)) or {"own", "foreign"} <= kinds): ambiguous.update(aliases) names[target] = () else: diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index bf18589c8..52e37b394 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -568,7 +568,7 @@ def _link_hits(link: Path, gone: set[Path]) -> bool: hops += 1 if hops > _MAX_HOPS: return False - target = PurePosixPath(os.readlink(cand)) + target = Path(os.readlink(cand)) # the native separators: a Windows target splits on backslashes if target.is_absolute(): resolved, parts = Path(target.anchor), target.parts[1:] else: diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 89a44665e..72ee779a4 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -7,6 +7,7 @@ import dataclasses import json +import os from pathlib import Path import pytest @@ -219,6 +220,17 @@ def test_overlay_arm_keeps_a_base_link_whose_target_a_removed_link_also_names(tm assert (tmp_path / "AGENTS.md").read_text(encoding="utf-8") == "shared\n" +@pytest.mark.skipif(os.name != "nt", reason="a backslash in a link target separates names on Windows only") +def test_overlay_arm_refuses_a_backslash_base_link_through_a_removed_directory_link(tmp_path): + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "base.md").write_text("base\n", encoding="utf-8") + (tmp_path / "AGENTS.codex.md").symlink_to("docs", target_is_directory=True) + (tmp_path / "AGENTS.md").symlink_to("AGENTS.codex.md\\base.md") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + with pytest.raises(PrepError, match="the base would dangle"): + _apply_overlay(tmp_path, ["AGENTS.md", "AGENTS.codex.md", "AGENTS.claude.md"], "claude") + + def test_overlay_arm_refuses_a_base_link_through_a_directory_link(tmp_path): (tmp_path / "real").mkdir() (tmp_path / "real" / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") @@ -307,6 +319,15 @@ def test_overlay_report_leaves_a_load_of_a_shared_target_unattributed(): assert report["own_files"] == ["AGENTS.claude.md"] and report["foreign_files"] == ["AGENTS.codex.md"] +def test_overlay_report_does_not_pin_a_shared_target_read_through_a_native_alias(): + """CLAUDE.md and AGENTS.codex.md both link to docs/shared.md: a Claude load of it is CLAUDE.md, not the foreign alias.""" + records = [{"file": "docs/shared.md", "scenario": "cwd=.", "class": "loaded"}] + links = {"docs/shared.md": ("CLAUDE.md", "AGENTS.codex.md")} + report = overlay_report(records, "claude", links, native={"CLAUDE.md"}) + assert report["cross_loaded"] == {} + assert "AGENTS.codex.md" in report["ambiguous"] + + def test_overlay_report_leaves_a_load_of_an_own_and_a_foreign_alias_unattributed(): records = [{"file": "docs/x.md", "scenario": "cwd=.", "class": "loaded"}] report = overlay_report(records, "claude", {"docs/x.md": ("AGENTS.claude.md", "AGENTS.codex.md")}) From b880a3ef2daee1265298cb9ff80af7047618d4e8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:37:59 -0800 Subject: [PATCH 08/14] fix(overlay): pin both names of an overlay link group; say when a missing own overlay is unknown A native name in a link group pins nothing. Two overlay names in one group (own or foreign) are ambiguous, and the ambiguous set now includes the target. A load of a link target counts as unpinned too. The 'no overlay was loaded' note needs a usable negative record and no error, otherwise the note says it could not be determined. Adds the base-link refusal cases and moves the native-alias test to test_repo_reach_links.py to keep test_repo_reach.py under the length cap. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 9 ++- dokimi/dokimi/repos/reach.py | 27 ++++++-- dokimi/tests/test_repo_overlay.py | 88 ++++++++++++++++++++++++- dokimi/tests/test_repo_reach_links.py | 29 ++++++++ 4 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 dokimi/tests/test_repo_reach_links.py diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 4eb13cb21..ebb4b6852 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1061,7 +1061,14 @@ per scenario). `not_loaded`. `cross_loaded` lists each foreign overlay some scenario reported as `loaded`, `truncated` or `fetched`; the text report prints it as a `finding:` line. `own_loaded` lists the harness's own overlays that loaded in some scenario; - when none did, the text report says so. `own` is the profile's `overlay`. + when none did, the text report says so, but only if some own-overlay scenario + read as `not_loaded` and none errored; otherwise it says loading could not be + determined. `own` is the profile's `overlay`. + `ambiguous` lists the names of a load that could have been read under another + name: a file reached through an alias group that holds a native guidance name + or two overlay names. Those loads are neither own nor cross loads. The text + report prints no own-overlay note for them, and a foreign target with an own + alias is listed there too, not reported as a cross load. - If payload capture is available, a `payload` field records whether each token was in the request bytes. This settles cells where the model under-reports, which is how a small model produces false `truncated` cells. diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 5ea371229..c316a4bcd 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -126,13 +126,15 @@ def _pinned_names(links: dict[str, tuple[str, ...]], own: str | None, names: dict[str, tuple[str, ...]] = {} ambiguous: set[str] = set() for target, aliases in links.items(): - kinds = {"own" if overlay_of(n) == own else "foreign" for n in (target, *aliases) if overlay_of(n) is not None} - # a native name anywhere in the group (the target or an alias) cannot tell which file a harness read - if aliases and (any(PurePosixPath(n).name in native for n in (target, *aliases)) or {"own", "foreign"} <= kinds): - ambiguous.update(aliases) + group = (target, *aliases) + overlay_names = [n for n in group if overlay_of(n) is not None] + # a native name anywhere in the group cannot tell which file a harness read, and neither can two overlay + # names (own or foreign): a harness that reads one of them reads the same file under every other one + if aliases and (any(PurePosixPath(n).name in native for n in group) or len(overlay_names) > 1): + ambiguous.update((*aliases, *overlay_names)) names[target] = () else: - names[target] = (target, *aliases) + names[target] = group return names, ambiguous @@ -152,7 +154,7 @@ def _loads(records: list[dict[str, Any]], names: dict[str, tuple[str, ...]], lin cross.setdefault(n, {})[r["scenario"]] = r["class"] if n in own_files: own_loaded.add(n) - unpinned.update(n for n in links.get(r["file"], ()) if n in ambiguous) + unpinned.update(n for n in (r["file"], *links.get(r["file"], ())) if n in ambiguous) return cross, own_loaded, unpinned @@ -468,6 +470,17 @@ def _matrix(records: list[dict[str, Any]]) -> dict[str, dict[str, str]]: return matrix +def _own_overlay_note(payload: dict[str, Any], overlays: dict[str, Any]) -> str: + """A "not loaded" note needs a usable negative record and no error: an errored scenario says nothing about loading.""" + group = {f["path"]: (f["path"], *f.get("aliases", ())) for f in payload["files"]} + own = set(overlays["own_files"]) + seen = {r["class"] for r in payload["records"] if own & set(group.get(r["file"], (r["file"],)))} + files = ", ".join(overlays["own_files"]) + if "not_loaded" in seen and "error" not in seen: + return f"note: no {overlays['own']} overlay ({files}) was loaded in any scenario" + return f"note: could not determine whether the {overlays['own']} overlay ({files}) was loaded" + + def format_matrix(payload: dict[str, Any]) -> str: """A plain-text table: one row per guidance file, one column per scenario.""" short = {"loaded": "L", "truncated": "T", "fetched": "F", "not_loaded": ".", "error": "E"} @@ -488,7 +501,7 @@ def format_matrix(payload: dict[str, Any]) -> str: where = ", ".join(f"{sid} ({cls})" for sid, cls in seen.items()) lines.append(f"finding: {path}, an overlay for another agent, was loaded by {payload['harness']['name']} in {where}") if overlays.get("own_files") and not overlays.get("own_loaded") and not overlays.get("ambiguous"): - lines.append(f"note: no {overlays['own']} overlay ({', '.join(overlays['own_files'])}) was loaded in any scenario") + lines.append(_own_overlay_note(payload, overlays)) if any(f.get("alias_kind") == "copy" for f in payload["files"]): lines.append("note: the paths after = are copies, not links (this OS refused a symlink); " "a harness that skips symlinks would still load them") diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 72ee779a4..d038abf5b 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -257,6 +257,7 @@ def test_text_report_states_the_cross_load_as_a_finding(): "harness": {"name": "claude-code", "version": "2.1.0"}, "model": "m", "summary": {"cost_usd": 0.0, "matrix": {}}, "files": [], "unresolved": [], "scenarios": [{"id": "cwd=.", "error": None, "prose": 0, "unknown": []}], + "records": [{"file": "AGENTS.claude.md", "scenario": "cwd=.", "class": "not_loaded"}], "overlays": {"own": "claude", "own_files": ["AGENTS.claude.md"], "own_loaded": [], "foreign_files": ["AGENTS.codex.md"], "cross_loaded": {"AGENTS.codex.md": {"cwd=.": "loaded"}}}, } @@ -266,6 +267,91 @@ def test_text_report_states_the_cross_load_as_a_finding(): assert "note: no claude overlay (AGENTS.claude.md) was loaded in any scenario" in text +def _payload(records, own, links, native=frozenset(), scenarios=("cwd=.",)): + """A text-report payload with the overlay part computed from ``records`` the way run_reach computes it.""" + files = [{"path": t, **({"aliases": list(a)} if a else {})} for t, a in links.items()] + return { + "harness": {"name": "claude-code", "version": "2.1.0"}, "model": "m", + "summary": {"cost_usd": 0.0, "matrix": {}}, "files": files, "unresolved": [], + "scenarios": [{"id": s, "error": None, "prose": 0, "unknown": []} for s in scenarios], + "records": records, "overlays": overlay_report(records, own, links, native), + } + + +@pytest.mark.parametrize("cls, note", [ + ("not_loaded", "note: no claude overlay (AGENTS.claude.md) was loaded in any scenario"), + ("error", "note: could not determine whether the claude overlay (AGENTS.claude.md) was loaded"), +]) +def test_text_report_says_not_loaded_only_for_a_usable_negative_record(cls, note): + records = [{"file": "AGENTS.claude.md", "scenario": "cwd=.", "class": cls}] + text = format_matrix(_payload(records, "claude", {"AGENTS.claude.md": ()})) + assert note in text + assert ("was loaded in any scenario" in text) == (cls == "not_loaded") + + +def test_text_report_does_not_claim_not_loaded_for_an_all_error_run(): + records = [{"file": "AGENTS.claude.md", "scenario": s, "class": "error"} for s in ("cwd=.", "read=sub")] + text = format_matrix(_payload(records, "claude", {"AGENTS.claude.md": ()}, scenarios=("cwd=.", "read=sub"))) + assert "was loaded in any scenario" not in text + assert "could not determine whether the claude overlay (AGENTS.claude.md) was loaded" in text + + +def test_text_report_keeps_the_ambiguous_load_out_of_the_own_overlay_note(): + """A loaded file that two own/foreign names could name is ambiguous: no own-overlay note is printed for it.""" + records = [{"file": "docs/x.md", "scenario": "cwd=.", "class": "loaded"}] + payload = _payload(records, "claude", {"docs/x.md": ("AGENTS.claude.md", "AGENTS.codex.md")}) + assert payload["overlays"]["ambiguous"] == ["AGENTS.claude.md", "AGENTS.codex.md"] + assert "note:" not in format_matrix(payload) + + +def test_overlay_report_leaves_a_load_of_two_foreign_aliases_unattributed(): + """A harness reading one of two foreign alias names of docs/shared.md reads the same file under both.""" + records = [{"file": "docs/shared.md", "scenario": "cwd=.", "class": "loaded"}] + links = {"docs/shared.md": ("AGENTS.codex.md", "AGENTS.copilot.md")} + report = overlay_report(records, "claude", links, native={"CLAUDE.md", "AGENTS.md"}) + assert report["cross_loaded"] == {} + assert report["ambiguous"] == ["AGENTS.codex.md", "AGENTS.copilot.md"] + + +def test_overlay_report_leaves_a_load_of_a_foreign_target_with_a_foreign_alias_unattributed(): + records = [{"file": "AGENTS.codex.md", "scenario": "cwd=.", "class": "loaded"}] + report = overlay_report(records, "claude", {"AGENTS.codex.md": ("AGENTS.copilot.md",)}) + assert report["cross_loaded"] == {} + assert report["ambiguous"] == ["AGENTS.codex.md", "AGENTS.copilot.md"] + + +@pytest.mark.parametrize("via", ["absolute", "dotdot", "dot-then-up", "chain"]) +def test_overlay_arm_refuses_a_base_link_to_a_removed_overlay_however_it_is_written(tmp_path, via): + (tmp_path / "sub").mkdir() + (tmp_path / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + files = ["AGENTS.md", "AGENTS.codex.md", "AGENTS.claude.md"] + if via == "chain": + (tmp_path / "mid.md").symlink_to("AGENTS.codex.md") + (tmp_path / "AGENTS.md").symlink_to("mid.md") + files.append("mid.md") + else: + target = {"absolute": str(tmp_path / "AGENTS.codex.md"), + "dotdot": f"../{tmp_path.name}/AGENTS.codex.md", + "dot-then-up": "./sub/../AGENTS.codex.md"}[via] + (tmp_path / "AGENTS.md").symlink_to(target) + with pytest.raises(PrepError, match="the base would dangle"): + _apply_overlay(tmp_path, files, "claude") + assert (tmp_path / "AGENTS.codex.md").exists() + + +def test_overlay_arm_refuses_an_own_overlay_that_links_outside_the_workspace(tmp_path): + ws = tmp_path / "ws" + ws.mkdir() + (tmp_path / "guide.md").write_text("outside\n", encoding="utf-8") + (ws / "AGENTS.md").write_text("base\n", encoding="utf-8") + (ws / "AGENTS.claude.md").symlink_to("../guide.md") + (ws / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + with pytest.raises(PrepError, match="does not resolve to a usable file"): + _apply_overlay(ws, ["AGENTS.md", "AGENTS.claude.md", "AGENTS.codex.md"], "claude") + assert (ws / "AGENTS.codex.md").exists() + + def test_overlay_arm_refuses_a_base_link_through_a_removed_directory_link(tmp_path): """A removed overlay that is a directory link: the base link resolves through it, so it would dangle.""" (tmp_path / "docs").mkdir() @@ -375,4 +461,4 @@ def test_overlay_report_leaves_a_load_of_a_foreign_target_with_an_own_alias_unat records = [{"file": "AGENTS.codex.md", "scenario": "cwd=.", "class": "loaded"}] report = overlay_report(records, "claude", {"AGENTS.codex.md": ("AGENTS.claude.md",)}) assert report["cross_loaded"] == {} and report["own_loaded"] == [] - assert report["ambiguous"] == ["AGENTS.claude.md"] + assert report["ambiguous"] == ["AGENTS.claude.md", "AGENTS.codex.md"] diff --git a/dokimi/tests/test_repo_reach_links.py b/dokimi/tests/test_repo_reach_links.py new file mode 100644 index 000000000..2a149f447 --- /dev/null +++ b/dokimi/tests/test_repo_reach_links.py @@ -0,0 +1,29 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Overlay links in the L1 reach probe: which names a harness is read as, and what that pins (no LLM).""" + +from __future__ import annotations + +from dokimi.artifacts import validate_artifact +from dokimi.repos import load_manifest +from dokimi.repos.harness_profile import load_profile +from dokimi.repos.reach import run_reach +from test_repo_reach import _commit_links, _profile, _repo, _write + + +def test_reach_payload_leaves_a_native_alias_of_a_foreign_overlay_unpinned(tmp_path, monkeypatch): + """run_reach gives overlay_report the profile's native names: CLAUDE.md -> AGENTS.codex.md is read as the + native file, so a load of it is not a load of the foreign overlay.""" + monkeypatch.setenv("FAKE_MODE", "ok") + root = _repo(tmp_path) + _write(root, "AGENTS.codex.md", "# codex overlay\n") + (root / "CLAUDE.md").unlink() + (root / "sub/CLAUDE.md").unlink() + tree = _commit_links(root, {"CLAUDE.md": "AGENTS.codex.md"}) + payload = run_reach(tree=tree, manifest=load_manifest(tree, "evals/repo-eval.yaml"), + profile=load_profile(_profile(tmp_path, overlay="claude")), model="m", + out_dir=tmp_path / "out", work_dir=tmp_path, salt="s", kinds=("root",)) + validate_artifact("guidance_reach", {**payload, "schema_version": "guidance_reach.v1"}) + assert payload["overlays"]["cross_loaded"] == {} + assert payload["overlays"]["ambiguous"] == ["AGENTS.codex.md", "CLAUDE.md"] From e761468042cfef93f680bddf9995d7571825a545 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:05:37 -0800 Subject: [PATCH 09/14] fix(overlay): drop the unused own parameter of _pinned_names Review: the helper never reads own; attribution depends only on the alias group and the native names. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/reach.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index c316a4bcd..9c250e461 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -120,7 +120,7 @@ def probe_files(tree: RepoTree, manifest: RepoManifest, profile: HarnessProfile) return sorted(p for p in declared | native | overlays if not manifest.in_eval_dir(p)) -def _pinned_names(links: dict[str, tuple[str, ...]], own: str | None, +def _pinned_names(links: dict[str, tuple[str, ...]], native: set[str] | frozenset[str]) -> tuple[dict[str, tuple[str, ...]], set[str]]: """Each probed file with the names a load of it can be pinned on, and the aliases that cannot be pinned.""" names: dict[str, tuple[str, ...]] = {} @@ -170,7 +170,7 @@ def overlay_report(records: list[dict[str, Any]], own: str | None, every = {n for target, aliases in links.items() for n in (target, *aliases)} own_files = {n for n in every if own and overlay_of(n) == own} foreign = {n for n in every if overlay_of(n) not in (None, own)} - names, ambiguous = _pinned_names(links, own, native) + names, ambiguous = _pinned_names(links, native) cross, own_loaded, unpinned = _loads(records, names, links, ambiguous, foreign, own_files) report: dict[str, Any] = {"own": own, "own_files": sorted(own_files), "own_loaded": sorted(own_loaded), "foreign_files": sorted(foreign), "cross_loaded": dict(sorted(cross.items()))} From 6232e4d9450921859cd182f84d8edb8ac8af6229 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:05:37 -0800 Subject: [PATCH 10/14] fix(overlay): refuse to unlink an overlay under a path a setup patch linked out A setup patch can turn a tracked directory into a link to a host directory. The index still lists the overlay under it and the file now resolves outside the workspace, so the arm unlinked the host file. Refuse before anything is removed, and keep the removal of the final component itself as it was. Red before the fix: DID NOT RAISE, host file removed. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/workspace.py | 3 +++ dokimi/tests/test_repo_overlay.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 52e37b394..1b3079ea1 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -587,6 +587,9 @@ def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], d if not real.is_file() or not real.is_relative_to(ws.resolve()): raise PrepError(f"guidance:overlay: {rel} does not resolve to a usable file in the workspace") removed = sorted(p for p in tracked if overlay_of(p) not in (None, own)) + for rel in removed: # a setup patch may have made a directory of the index a link out of the workspace + if not _canon(ws / rel).parent.is_relative_to(ws.resolve()): + raise PrepError(f"guidance:overlay: {rel} is under a path that resolves outside the workspace") gone = {_canon(ws / rel) for rel in removed} removed_set = set(removed) for rel in tracked: diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index d038abf5b..7d96364c1 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -8,6 +8,7 @@ import dataclasses import json import os +import shutil from pathlib import Path import pytest @@ -394,6 +395,29 @@ def test_overlay_arm_reads_the_workspace_after_setup_patches_not_the_index(tmp_p assert not (ws / "AGENTS.codex.md").exists() and (ws / "AGENTS.claude.md").exists() +def test_overlay_arm_does_not_unlink_through_a_setup_patch_that_made_a_directory_a_link(tmp_path): + """A setup patch replaces the tracked directory sub/ with a link to a host directory. The index still lists + sub/AGENTS.codex.md and it now resolves outside the workspace: the arm must refuse, not delete the host file.""" + ws = tmp_path / "ws" + ws.mkdir() + host = tmp_path / "host" + host.mkdir() + (host / "AGENTS.codex.md").write_text("host\n", encoding="utf-8") + (ws / "AGENTS.md").write_text("base\n", encoding="utf-8") + (ws / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + (ws / "sub").mkdir() + (ws / "sub" / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + git(ws, "init", "-q") + git(ws, "add", "-A") + git(ws, "commit", "-q", "-m", "base") + shutil.rmtree(ws / "sub") # the setup patch, applied without the index + (ws / "sub").symlink_to(host, target_is_directory=True) + env = _git_env(SNAPSHOT_AUTHOR) + with pytest.raises(PrepError, match="resolves outside the workspace"): + _apply_arm(ws, env, tmp_path, None, parse_arm(OVERLAY_ARM), None, "claude") + assert (host / "AGENTS.codex.md").read_text(encoding="utf-8") == "host\n" + + def test_overlay_report_leaves_a_load_of_a_shared_target_unattributed(): """A harness that loads AGENTS.md reads the file its AGENTS.claude.md and AGENTS.codex.md aliases name too: neither alias is pinned as loaded, and the load is reported as ambiguous.""" From e9a486f63c8ba86e776fc953ba620c4564e0efd0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:02:01 -0800 Subject: [PATCH 11/14] fix(overlay): refuse a kept path under a removed link; unlink a file once A setup patch can turn an indexed directory AGENTS.codex.md/ into a link to docs/. The stale kept path AGENTS.codex.md/AGENTS.claude.md then ends up dangling once the link is removed. The check only looked at symlink names, so it was not refused. The walk now runs over every non-removed path from the workspace root. A directory alias (sub -> real) lists one file under two names; the removal unlinked it twice and raised FileNotFoundError. The removed list is now one name per file. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/workspace.py | 37 ++++++++++++++++++++----------- dokimi/tests/test_repo_overlay.py | 28 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 1b3079ea1..db685a880 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -545,12 +545,13 @@ def _workspace_files(ws: Path, env: dict[str, str]) -> list[str]: return sorted({p for p in listed if p and os.path.lexists(ws / p)}) -def _link_hits(link: Path, gone: set[Path]) -> bool: - """Whether resolving the symbolic link ``link`` passes through one of the canonical paths ``gone``. Each path - component is checked before it is resolved, so a removed directory link named in the middle of the path is a - hit as well; a link is followed hop by hop, and a cycle ends the walk.""" - resolved = Path(os.path.realpath(link.parent)) - todo = [link.name] +def _path_hits(ws: Path, rel: str, gone: set[Path]) -> bool: + """Whether the workspace path ``rel`` runs through one of the canonical paths ``gone``, a link or a plain name. + Walked from the workspace root: each name is checked before it is resolved, so a removed directory link named in + the middle of the path is a hit (the path would not exist once it is removed); a link is followed hop by hop, + and a cycle ends the walk.""" + resolved = Path(os.path.realpath(ws)) + todo = rel.split("/") hops = 0 while todo: part = todo.pop(0) @@ -577,6 +578,17 @@ def _link_hits(link: Path, gone: set[Path]) -> bool: return False +def _overlay_targets(ws: Path, listed: list[str]) -> list[str]: + """The overlays of other agents to remove, one name per file: a directory the index lists that a setup patch made + a link to another directory names the same file twice, and the first name in order stands for it.""" + names: dict[Path, str] = {} + for rel in listed: # a setup patch may have made a directory of the index a link out of the workspace + if not _canon(ws / rel).parent.is_relative_to(ws.resolve()): + raise PrepError(f"guidance:overlay: {rel} is under a path that resolves outside the workspace") + names.setdefault(_canon(ws / rel), rel) + return sorted(names.values()) + + def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], dict[str, Any]]: """Remove the overlays of other agents, keeping ``AGENTS.md`` and the ``own`` overlay; refused without one.""" kept = sorted(p for p in tracked if overlay_of(p) == own) @@ -586,15 +598,14 @@ def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], d real = Path(os.path.realpath(ws / rel)) if not real.is_file() or not real.is_relative_to(ws.resolve()): raise PrepError(f"guidance:overlay: {rel} does not resolve to a usable file in the workspace") - removed = sorted(p for p in tracked if overlay_of(p) not in (None, own)) - for rel in removed: # a setup patch may have made a directory of the index a link out of the workspace - if not _canon(ws / rel).parent.is_relative_to(ws.resolve()): - raise PrepError(f"guidance:overlay: {rel} is under a path that resolves outside the workspace") + listed = [p for p in tracked if overlay_of(p) not in (None, own)] + removed = _overlay_targets(ws, listed) gone = {_canon(ws / rel) for rel in removed} - removed_set = set(removed) + listed_set = set(listed) for rel in tracked: - if rel not in removed_set and (ws / rel).is_symlink() and _link_hits(ws / rel, gone): - raise PrepError(f"guidance:overlay: {rel} links to an overlay the arm removes; the base would dangle") + if rel not in listed_set and _path_hits(ws, rel, gone): + raise PrepError(f"guidance:overlay: {rel} resolves through an overlay the arm removes; " + "the base would dangle") for rel in removed: if (ws / rel).is_dir() and not (ws / rel).is_symlink(): raise PrepError(f"guidance:overlay: {rel} is a directory, not an overlay file") diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index 7d96364c1..ab5a98813 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -418,6 +418,34 @@ def test_overlay_arm_does_not_unlink_through_a_setup_patch_that_made_a_directory assert (host / "AGENTS.codex.md").read_text(encoding="utf-8") == "host\n" +def test_overlay_arm_refuses_a_kept_overlay_under_a_removed_directory_link(tmp_path): + """A setup patch replaces the indexed directory AGENTS.codex.md/ with a link to docs/. The stale index entry + AGENTS.codex.md/AGENTS.claude.md still resolves to a file, but once the removed link goes it no longer exists.""" + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + (tmp_path / "AGENTS.codex.md").symlink_to("docs", target_is_directory=True) + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + tracked = ["AGENTS.md", "AGENTS.codex.md", "AGENTS.claude.md", "AGENTS.codex.md/AGENTS.claude.md", + "docs/AGENTS.claude.md"] + with pytest.raises(PrepError, match="the base would dangle"): + _apply_overlay(tmp_path, tracked, "claude") + assert (tmp_path / "AGENTS.codex.md").is_symlink() # refused before anything is removed + + +@pytest.mark.parametrize("alias", ["sub", "aa"]) # the alias sorts after the real name, or before it +def test_overlay_arm_removes_a_file_a_directory_alias_lists_twice_once(tmp_path, alias): + """A setup patch replaces the indexed directory `alias` with a link to real/: the index still lists + alias/AGENTS.codex.md next to real/AGENTS.codex.md, which is one file. It is removed once, not twice.""" + (tmp_path / "real").mkdir() + (tmp_path / "real" / "AGENTS.codex.md").write_text("codex\n", encoding="utf-8") + (tmp_path / alias).symlink_to("real", target_is_directory=True) + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + tracked = sorted(["AGENTS.claude.md", "real/AGENTS.codex.md", f"{alias}/AGENTS.codex.md"]) + removed, _ = _apply_overlay(tmp_path, tracked, "claude") + assert len(removed) == 1 + assert not (tmp_path / "real" / "AGENTS.codex.md").exists() + + def test_overlay_report_leaves_a_load_of_a_shared_target_unattributed(): """A harness that loads AGENTS.md reads the file its AGENTS.claude.md and AGENTS.codex.md aliases name too: neither alias is pinned as loaded, and the load is reported as ambiguous.""" From ba6dbfd5d1c94874fa76bd2221f89322099df1e6 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 22:01:40 +0000 Subject: [PATCH 12/14] fix(overlay): load builtin profiles by path, not name Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/overlays.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/overlays.py b/dokimi/dokimi/repos/overlays.py index 6e6fc2ad4..71ec4350a 100644 --- a/dokimi/dokimi/repos/overlays.py +++ b/dokimi/dokimi/repos/overlays.py @@ -14,7 +14,7 @@ import re from pathlib import PurePosixPath -from .harness_profile import builtin_profiles, load_profile +from .harness_profile import PROFILE_DIR, builtin_profiles, load_profile OVERLAY_RE = re.compile(r"AGENTS\.([A-Za-z0-9][A-Za-z0-9-]*)\.md") @@ -22,7 +22,11 @@ @functools.lru_cache(maxsize=1) def native_names() -> frozenset[str]: """Guidance names a builtin harness loads by its own convention (``AGENTS.override.md``): never an overlay.""" - return frozenset(n for name in builtin_profiles() for n in load_profile(name).guidance_names) + return frozenset( + n + for name in builtin_profiles() + for n in load_profile(str(PROFILE_DIR / f"{name}.yaml")).guidance_names + ) def overlay_of(path: str) -> str | None: From 7e46426b4077540f1451cbdad42b6813090cdab4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 17:15:28 -0800 Subject: [PATCH 13/14] fix(overlay): unlink under a removed alias first, attribute linked loads honestly - workspace: remove the removed overlay paths children-first, so a file under a directory alias is unlinked before the alias itself. - reach: a load of a link target is pinned to an overlay alias only when no native file in the probe can import it, and two overlay names stay unpinned as before. - reach: a refusal or an errored scenario gives no usable not_loaded record for the own-overlay note; the note is printed when only an ambiguous foreign file is unrelated to the own overlay. - Tests: nested alias order, a directly imported non-native target, an own-overlay note beside an unrelated ambiguous load, a refusal, and a same-named profile file in the working directory. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/reach.py | 15 +++++--- dokimi/dokimi/repos/workspace.py | 2 +- dokimi/tests/test_repo_overlay.py | 59 ++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index c81bad1eb..0fb9c89e5 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -128,9 +128,9 @@ def _pinned_names(links: dict[str, tuple[str, ...]], for target, aliases in links.items(): group = (target, *aliases) overlay_names = [n for n in group if overlay_of(n) is not None] - # a native name anywhere in the group cannot tell which file a harness read, and neither can two overlay - # names (own or foreign): a harness that reads one of them reads the same file under every other one - if aliases and (any(PurePosixPath(n).name in native for n in group) or len(overlay_names) > 1): + # a native file in the probe can import the target (CLAUDE.md with @docs/shared.md): the load is not the alias's; + # two overlay names name one file, so either could have been read + if aliases and (native or len(overlay_names) > 1): ambiguous.update((*aliases, *overlay_names)) names[target] = () else: @@ -475,8 +475,12 @@ def _own_overlay_note(payload: dict[str, Any], overlays: dict[str, Any]) -> str: group = {f["path"]: (f["path"], *f.get("aliases", ())) for f in payload["files"]} own = set(overlays["own_files"]) seen = {r["class"] for r in payload["records"] if own & set(group.get(r["file"], (r["file"],)))} + # a scenario that errored, or answered with prose (a refusal), says nothing about what it loaded + usable = {s["id"] for s in payload["scenarios"] if not s.get("error") and (s.get("prose") or 0) <= PROSE_LIMIT} + negative = {r["class"] for r in payload["records"] + if r["scenario"] in usable and own & set(group.get(r["file"], (r["file"],)))} files = ", ".join(overlays["own_files"]) - if "not_loaded" in seen and "error" not in seen: + if "not_loaded" in negative and "error" not in seen: return f"note: no {overlays['own']} overlay ({files}) was loaded in any scenario" return f"note: could not determine whether the {overlays['own']} overlay ({files}) was loaded" @@ -525,7 +529,8 @@ def format_matrix(payload: dict[str, Any]) -> str: for path, seen in (overlays.get("cross_loaded") or {}).items(): where = ", ".join(f"{sid} ({cls})" for sid, cls in seen.items()) lines.append(f"finding: {path}, an overlay for another agent, was loaded by {payload['harness']['name']} in {where}") - if overlays.get("own_files") and not overlays.get("own_loaded") and not overlays.get("ambiguous"): + own_ambiguous = set(overlays.get("own_files") or ()) & set(overlays.get("ambiguous") or ()) + if overlays.get("own_files") and not overlays.get("own_loaded") and not own_ambiguous: lines.append(_own_overlay_note(payload, overlays)) if any(f.get("alias_kind") == "copy" for f in payload["files"]): lines.append("note: the paths after = are copies, not links (this OS refused a symlink); " diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index db685a880..c31006e0d 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -606,7 +606,7 @@ def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], d if rel not in listed_set and _path_hits(ws, rel, gone): raise PrepError(f"guidance:overlay: {rel} resolves through an overlay the arm removes; " "the base would dangle") - for rel in removed: + for rel in reversed(removed): # a name under a link sorts after the link: unlink it first, or its path is gone if (ws / rel).is_dir() and not (ws / rel).is_symlink(): raise PrepError(f"guidance:overlay: {rel} is a directory, not an overlay file") (ws / rel).unlink() diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index ab5a98813..aadddd403 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -16,7 +16,7 @@ from dokimi.repos import RepoSource, load_manifest, open_source from dokimi.repos.harness_profile import PROFILE_DIR, ProfileError, load_profile, parse_profile -from dokimi.repos.overlays import overlay_of +from dokimi.repos.overlays import native_names, overlay_of from dokimi.repos.reach import format_matrix, overlay_report, probe_files from dokimi.repos.workspace import ( ARM_FORMS, @@ -305,6 +305,46 @@ def test_text_report_keeps_the_ambiguous_load_out_of_the_own_overlay_note(): assert "note:" not in format_matrix(payload) +def test_overlay_of_reads_the_bundled_profiles_not_a_same_named_file_in_the_cwd(tmp_path, monkeypatch): + """A file named like a bundled profile, in the working directory, is not parsed as that profile.""" + (tmp_path / "codex").write_text("not: [a profile\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + native_names.cache_clear() + try: + assert overlay_of("AGENTS.claude.md") == "claude" + assert native_names() + finally: + native_names.cache_clear() + + +def test_overlay_report_leaves_a_link_load_unattributed_when_a_native_file_may_import_it(): + """CLAUDE.md can import docs/shared.md, so a load of that file is not a load of its AGENTS.codex.md alias.""" + records = [{"file": "docs/shared.md", "scenario": "cwd=.", "class": "loaded"}] + links = {"CLAUDE.md": (), "docs/shared.md": ("AGENTS.codex.md",)} + report = overlay_report(records, "claude", links, native={"CLAUDE.md"}) + assert report["cross_loaded"] == {} + assert report["ambiguous"] == ["AGENTS.codex.md"] + + +def test_text_report_still_says_not_loaded_for_an_own_overlay_next_to_an_unrelated_ambiguous_load(): + records = [{"file": "AGENTS.claude.md", "scenario": "cwd=.", "class": "not_loaded"}, + {"file": "AGENTS.codex.md", "scenario": "cwd=.", "class": "loaded"}] + payload = _payload(records, "claude", {"AGENTS.claude.md": (), "AGENTS.codex.md": ("CLAUDE.md",)}, + native={"CLAUDE.md"}) + assert payload["overlays"]["ambiguous"] == ["AGENTS.codex.md", "CLAUDE.md"] + assert "note: no claude overlay (AGENTS.claude.md) was loaded in any scenario" in format_matrix(payload) + + +def test_text_report_does_not_claim_not_loaded_from_a_refusal(): + """A scenario that answered with prose (a refusal) gives no usable negative record.""" + records = [{"file": "AGENTS.claude.md", "scenario": "read=sub", "class": "not_loaded"}] + payload = _payload(records, "claude", {"AGENTS.claude.md": ()}, scenarios=("read=sub",)) + payload["scenarios"][0]["prose"] = 40 + text = format_matrix(payload) + assert "was loaded in any scenario" not in text + assert "could not determine whether the claude overlay (AGENTS.claude.md) was loaded" in text + + def test_overlay_report_leaves_a_load_of_two_foreign_aliases_unattributed(): """A harness reading one of two foreign alias names of docs/shared.md reads the same file under both.""" records = [{"file": "docs/shared.md", "scenario": "cwd=.", "class": "loaded"}] @@ -446,6 +486,23 @@ def test_overlay_arm_removes_a_file_a_directory_alias_lists_twice_once(tmp_path, assert not (tmp_path / "real" / "AGENTS.codex.md").exists() +@pytest.mark.parametrize("listed", [ + ["AGENTS.claude.md", "AGENTS.codex.md", "AGENTS.codex.md/AGENTS.copilot.md", "real/AGENTS.copilot.md"], + ["AGENTS.claude.md", "AGENTS.codex.md", "real/AGENTS.copilot.md"], +]) +def test_overlay_arm_removes_a_file_under_a_removed_directory_alias_before_the_alias(tmp_path, listed): + """AGENTS.codex.md is a link to real/ and the index also lists a file under it: the file goes before the link, + or its unlink finds no path. Either listing names the same file, so it is removed once.""" + (tmp_path / "real").mkdir() + (tmp_path / "real" / "AGENTS.copilot.md").write_text("copilot\n", encoding="utf-8") + (tmp_path / "AGENTS.codex.md").symlink_to("real", target_is_directory=True) + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + _apply_overlay(tmp_path, sorted(listed), "claude") + assert not os.path.lexists(tmp_path / "AGENTS.codex.md") + assert not (tmp_path / "real" / "AGENTS.copilot.md").exists() + assert (tmp_path / "AGENTS.claude.md").is_file() + + def test_overlay_report_leaves_a_load_of_a_shared_target_unattributed(): """A harness that loads AGENTS.md reads the file its AGENTS.claude.md and AGENTS.codex.md aliases name too: neither alias is pinned as loaded, and the load is reported as ambiguous.""" From d70dd57df8e467a3f22e17637220c96485339fb3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 20:23:24 -0800 Subject: [PATCH 14/14] fix(overlay): unlink removed overlays by canonical path; native names count only when probed A chained directory alias (AGENTS.a.md -> AGENTS.z.md -> real/) failed the removal when AGENTS.z.md went first. Each removed overlay is now unlinked by the canonical path captured before any unlink, after every check. A configured harness name (CLAUDE.md) with no file in the probe no longer makes a link group ambiguous: native_in_probe keeps only the names present. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/reach.py | 10 +++++++++- dokimi/dokimi/repos/workspace.py | 24 ++++++++++++++---------- dokimi/tests/test_repo_overlay.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 0fb9c89e5..61438a480 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -39,6 +39,7 @@ import re import shutil import tempfile +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -158,6 +159,13 @@ def _loads(records: list[dict[str, Any]], names: dict[str, tuple[str, ...]], lin return cross, own_loaded, unpinned +def native_in_probe(seeded: list[Any], guidance_names: Iterable[str]) -> set[str]: + """The harness-native names (the profile's guidance names) that a probed file or link carries. A native file in + the probe can import a link's target; a configured name with no file in the probe cannot.""" + probed = {PurePosixPath(p).name for s in seeded for p in (s.path, *s.aliases)} + return probed & {PurePosixPath(n).name for n in guidance_names} + + def overlay_report(records: list[dict[str, Any]], own: str | None, links: dict[str, tuple[str, ...]], native: set[str] | frozenset[str] = frozenset()) -> dict[str, Any]: """The overlay part of the payload: which of the ``own`` overlay files loaded, and which foreign overlays did @@ -421,7 +429,7 @@ def run_reach( "scenarios": scenario_rows, "records": records, "overlays": overlay_report(records, profile.overlay, {s.path: s.aliases for s in seeded}, - {PurePosixPath(n).name for n in profile.guidance_names}), + native_in_probe(seeded, profile.guidance_names)), "summary": { "cost_usd": round(sum(r.get("cost_usd") or 0 for r in scenario_rows), 6), "by_class": by_class, diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index c31006e0d..02ef127ba 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -578,15 +578,16 @@ def _path_hits(ws: Path, rel: str, gone: set[Path]) -> bool: return False -def _overlay_targets(ws: Path, listed: list[str]) -> list[str]: - """The overlays of other agents to remove, one name per file: a directory the index lists that a setup patch made - a link to another directory names the same file twice, and the first name in order stands for it.""" +def _overlay_targets(ws: Path, listed: list[str]) -> dict[Path, str]: + """The overlays of other agents to remove, by canonical path (the directories resolved, so a name under a link + is the file it reaches), one name per file: a directory the index lists that a setup patch made a link to another + directory names the same file twice, and the first name in order stands for it.""" names: dict[Path, str] = {} for rel in listed: # a setup patch may have made a directory of the index a link out of the workspace if not _canon(ws / rel).parent.is_relative_to(ws.resolve()): raise PrepError(f"guidance:overlay: {rel} is under a path that resolves outside the workspace") names.setdefault(_canon(ws / rel), rel) - return sorted(names.values()) + return names def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], dict[str, Any]]: @@ -599,18 +600,21 @@ def _apply_overlay(ws: Path, tracked: list[str], own: str) -> tuple[list[str], d if not real.is_file() or not real.is_relative_to(ws.resolve()): raise PrepError(f"guidance:overlay: {rel} does not resolve to a usable file in the workspace") listed = [p for p in tracked if overlay_of(p) not in (None, own)] - removed = _overlay_targets(ws, listed) - gone = {_canon(ws / rel) for rel in removed} + targets = _overlay_targets(ws, listed) + gone = set(targets) listed_set = set(listed) for rel in tracked: if rel not in listed_set and _path_hits(ws, rel, gone): raise PrepError(f"guidance:overlay: {rel} resolves through an overlay the arm removes; " "the base would dangle") - for rel in reversed(removed): # a name under a link sorts after the link: unlink it first, or its path is gone - if (ws / rel).is_dir() and not (ws / rel).is_symlink(): + for canon, rel in targets.items(): # all checks before the first unlink: a refusal leaves the workspace as it was + if canon.is_dir() and not canon.is_symlink(): raise PrepError(f"guidance:overlay: {rel} is a directory, not an overlay file") - (ws / rel).unlink() - return removed, {"arm_overlay": {"name": own, "kept": kept}} + # Each canonical path has real directories only as its parents, so removing one leaves the others valid, whatever + # order the aliases sort in (a name under a link is unlinked by its canonical path, not through the link) + for canon in targets: + canon.unlink() + return sorted(targets.values()), {"arm_overlay": {"name": own, "kept": kept}} def _guidance_at(git_dir: Path, manifest: RepoManifest, sha: str) -> tuple[list[str], dict[str, str]]: diff --git a/dokimi/tests/test_repo_overlay.py b/dokimi/tests/test_repo_overlay.py index aadddd403..1d314f29f 100644 --- a/dokimi/tests/test_repo_overlay.py +++ b/dokimi/tests/test_repo_overlay.py @@ -10,6 +10,7 @@ import os import shutil from pathlib import Path +from types import SimpleNamespace import pytest import yaml @@ -17,7 +18,7 @@ from dokimi.repos import RepoSource, load_manifest, open_source from dokimi.repos.harness_profile import PROFILE_DIR, ProfileError, load_profile, parse_profile from dokimi.repos.overlays import native_names, overlay_of -from dokimi.repos.reach import format_matrix, overlay_report, probe_files +from dokimi.repos.reach import format_matrix, native_in_probe, overlay_report, probe_files from dokimi.repos.workspace import ( ARM_FORMS, OVERLAY_ARM, @@ -503,6 +504,33 @@ def test_overlay_arm_removes_a_file_under_a_removed_directory_alias_before_the_a assert (tmp_path / "AGENTS.claude.md").is_file() +def test_overlay_arm_removes_a_chained_directory_alias_listing_without_failing(tmp_path): + """AGENTS.a.md links to AGENTS.z.md, which links to real/: the listed AGENTS.a.md/AGENTS.copilot.md is the same + file as real/AGENTS.copilot.md. Removing AGENTS.z.md first would leave AGENTS.a.md dangling before the nested + name is unlinked; each file is removed by its canonical path, so the order of the names does not matter.""" + (tmp_path / "real").mkdir() + (tmp_path / "real" / "AGENTS.copilot.md").write_text("copilot\n", encoding="utf-8") + (tmp_path / "AGENTS.z.md").symlink_to("real", target_is_directory=True) + (tmp_path / "AGENTS.a.md").symlink_to("AGENTS.z.md", target_is_directory=True) + (tmp_path / "AGENTS.claude.md").write_text("claude\n", encoding="utf-8") + listed = ["AGENTS.a.md", "AGENTS.z.md", "AGENTS.a.md/AGENTS.copilot.md", "real/AGENTS.copilot.md", "AGENTS.claude.md"] + removed, _ = _apply_overlay(tmp_path, sorted(listed), "claude") + assert removed == ["AGENTS.a.md", "AGENTS.a.md/AGENTS.copilot.md", "AGENTS.z.md"] + assert not os.path.lexists(tmp_path / "AGENTS.a.md") + assert not os.path.lexists(tmp_path / "AGENTS.z.md") + assert not (tmp_path / "real" / "AGENTS.copilot.md").exists() + assert (tmp_path / "AGENTS.claude.md").is_file() + + +def test_native_in_probe_counts_only_the_native_names_a_probed_file_or_link_carries(): + """The profile names CLAUDE.md, but a probe with no CLAUDE.md file or link has no native name to import a target: + the configured name alone does not make an alias ambiguous.""" + shared = SimpleNamespace(path="docs/shared.md", aliases=("AGENTS.codex.md",)) + assert native_in_probe([shared], ["CLAUDE.md"]) == set() + assert native_in_probe([shared, SimpleNamespace(path="sub/CLAUDE.md", aliases=())], ["CLAUDE.md"]) == {"CLAUDE.md"} + assert native_in_probe([SimpleNamespace(path="CLAUDE.md", aliases=("AGENTS.codex.md",))], ["CLAUDE.md"]) == {"CLAUDE.md"} + + def test_overlay_report_leaves_a_load_of_a_shared_target_unattributed(): """A harness that loads AGENTS.md reads the file its AGENTS.claude.md and AGENTS.codex.md aliases name too: neither alias is pinned as loaded, and the load is reported as ambiguous."""