diff --git a/dokimi/changelog.d/20261101_120000_napetrov_fidelity-arm-leaks.md b/dokimi/changelog.d/20261101_120000_napetrov_fidelity-arm-leaks.md new file mode 100644 index 000000000..5fa746180 --- /dev/null +++ b/dokimi/changelog.d/20261101_120000_napetrov_fidelity-arm-leaks.md @@ -0,0 +1,8 @@ + + +### Changed + +- **docker-claude arms no longer show dokimi's own names to the agent** — the host workspace is a plain `mktemp` directory instead of `dokimi-run-*`, the in-container skill plugin is mounted at `/opt/claude-plugins` instead of `/opt/abskill`, and the prompt is the task instruction with only the treatment notice in front of it, not the "Solve this coding task…" wrapper. Skills still load through `--plugin-dir`: a skill in `$CLAUDE_CONFIG_DIR/skills` is not offered under the launch's `--setting-sources ""`. The agent binary and its files live under `/opt/agent` instead of `/opt/dokimi-agent`. `tests/test_payload_audit.py` asserts the arms' full command (executable and argv, docker and host) and cwd carry no marker. Runs measured before this change are not comparable in their prompt text or command line. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index f673acf55..f6cc1ae9f 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -331,7 +331,7 @@ The result is in `metrics.isolation_probe` and `isolation_probe.json`: by `docker cp`: the `claude` binary, the files named by the file-valued variables below (CA bundles, `AWS_SHARED_CREDENTIALS_FILE` with every profile in it, `AWS_CONFIG_FILE`, `GOOGLE_APPLICATION_CREDENTIALS`), and - the skill arm's plugin at `/opt/abskill` (built with symlinks followed, as + the skill arm's plugin at `/opt/claude-plugins` (built with symlinks followed, as for the host arm). - **Image needs.** A glibc image (the host's `claude` binary is copied in; a musl image fails the `--version` check), and a working tree in `/app` that @@ -450,7 +450,7 @@ The `-host` variants run the agent on the host instead. The host cannot hide an absolute path or close the network, so these harnesses state what they seal in every run's `metrics.isolation`. What they do: -- solve in a fresh `dokimi-run-*` temp tree outside any checkout; +- solve in a fresh temp tree (a plain `mkdtemp` name) outside any checkout; - point `HOME` and `CLAUDE_CONFIG_DIR` at empty scratch dirs; - pass only an allowlist of variables; - reach `task_files`, `treatment`, `operator_config`, `network` and `env` by @@ -914,7 +914,7 @@ python cli.py tasks run \ The arms differ by the skill and one notice, and by nothing else: - **Delivery.** The skill directory is wrapped in a one-skill plugin and passed - with `--plugin-dir`, at `/opt/abskill` in the container (outside `/app`, and + with `--plugin-dir`, at `/opt/claude-plugins` in the container (outside `/app`, and removed before the verifier runs) or beside the scratch work tree on the host. The agent loads it with its own `Skill` tool. The skill's top-level `evals/` (its prompts and expected outputs) is not copied. Symlinks are diff --git a/dokimi/dokimi/harnesses/agent_sandbox.py b/dokimi/dokimi/harnesses/agent_sandbox.py index 2e4ccda1f..045e6acba 100644 --- a/dokimi/dokimi/harnesses/agent_sandbox.py +++ b/dokimi/dokimi/harnesses/agent_sandbox.py @@ -158,7 +158,10 @@ def solver_cli_flags() -> list[str]: @dataclass class AgentWorkspace: - """A scratch root ``/dokimi-run-*/{work,home,config}`` for one solve. + """A scratch root ``//{work,home,config}`` for one solve. + + The name is the one ``mktemp`` gives any tool: the agent's cwd must not say it + is under a harness (see :mod:`payload_audit`). ``forbidden`` lists host paths the agent should not be able to read (the task's grader files, a skill the arm does not offer, the operator's @@ -170,7 +173,7 @@ class AgentWorkspace: @classmethod def create(cls, *, forbidden: Optional[dict[str, list[Path]]] = None) -> "AgentWorkspace": - root = Path(tempfile.mkdtemp(prefix="dokimi-run-")) + root = Path(tempfile.mkdtemp()) for sub in ("work", "home", "config"): (root / sub).mkdir() return cls(root=root, forbidden=forbidden or {}) diff --git a/dokimi/dokimi/harnesses/container_agent.py b/dokimi/dokimi/harnesses/container_agent.py index c0242c50b..076d5fc3a 100644 --- a/dokimi/dokimi/harnesses/container_agent.py +++ b/dokimi/dokimi/harnesses/container_agent.py @@ -77,8 +77,8 @@ AGENT_UID = "10001:10001" # HOME is its own tmpfs, so the image's /tmp stays the image's (the verifier sees it too). AGENT_HOME = "/run/abhome" -AGENT_BIN = "/opt/dokimi-agent/claude" -FILES_DIR = "/opt/dokimi-agent/files" +AGENT_BIN = "/opt/agent/claude" +FILES_DIR = "/opt/agent/files" LABEL = "dokimi.run" # on the network, sidecar and task container: find leftovers with it # File-valued variables whose file is copied into the container and re-pointed there. _FILE_VARS = ("NODE_EXTRA_CA_CERTS", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "AWS_CA_BUNDLE", diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index 8b8560654..6eff31d52 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -97,7 +97,9 @@ # C tasks (missing-restrict) use gcc, C++ tasks use g++. _RECOMPILE_RE = re.compile(r"(g\+\+|gcc) [^`\n]*-o /app/[^ `\n]*") _DEFAULT_SOLVE_TIMEOUT = 600.0 -_CONTAINER_PLUGIN = "/opt/abskill" +# Root-owned, outside /app and the agent's HOME. The skill is loaded with --plugin-dir: a +# skill in $CLAUDE_CONFIG_DIR/skills is not offered under the launch's --setting-sources "". +_CONTAINER_PLUGIN = "/opt/claude-plugins" # The only tree a docker-* agent edits: the host agent's snapshot and copy-back, # the in-container agent's chown and `-w`. Anything else (an exported # SkillsBench task works in WORKDIR /root) would hand it root's home: .profile @@ -509,14 +511,14 @@ def _prepare_workspace(self, task: TaskSpec, container: str, log, treatment = build_plugin(skill_dir, workspace.root / "plugin") if extra_metrics is not None: self._record_treatment(treatment, skill_dir, extra_metrics, log) - except BaseException: # also Ctrl-C: don't leave the dokimi-run-* snapshot behind + except BaseException: # also Ctrl-C: don't leave the snapshot behind workspace.cleanup() raise return workspace, treatment def _install_plugin_in_container(self, container: str, output_dir: Path, log, extra_metrics: dict[str, Any]) -> Optional[Treatment]: - """Copy the arm's skill, as a plugin, to ``/opt/abskill`` (outside ``/app``).""" + """Copy the arm's skill, as a plugin, to ``_CONTAINER_PLUGIN`` (outside ``/app``).""" skill_dir = self._resolve_skill_dir() if skill_dir is None: return None @@ -556,12 +558,8 @@ def _claude_args(self, task: TaskSpec, treatment: Optional[Treatment], plugin_dir: Optional[str], model: Optional[str], output_dir: Path) -> list[str]: """``claude`` argv after the executable; the prompt is also saved as ``prompt.txt``.""" - prompt = ( - f"{treatment_notice(treatment) if self.skill_notice else ''}" - "Solve this coding task. Edit files in the current directory only. " - "When done, leave the optimized source and any required outputs in place.\n\n" - f"{task.instruction}" - ) + # The task instruction as the user wrote it, behind the treatment notice only. + prompt = f"{treatment_notice(treatment) if self.skill_notice else ''}{task.instruction}" (output_dir / "prompt.txt").write_text(prompt, encoding="utf-8") args = ["-p", prompt, "--output-format", "stream-json", "--verbose", *solver_cli_flags()] args += ["--plugin-dir", plugin_dir] if plugin_dir else [] diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 328078be1..21336b846 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -66,7 +66,7 @@ class ProbeConfig: #: Prefix of each run's temp root. Claude Code shows the model its working directory, -#: so the name must not say what the run is (``AgentWorkspace.create`` uses ``dokimi-run-``). +#: so the name must not say what the run is (``AgentWorkspace.create`` uses a plain ``mkdtemp`` name). WORKSPACE_PREFIX = "project-" diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d751c4643..3808f0c36 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -2,7 +2,7 @@ "files": { "dokimi/commands/novelty.py": {"lines": 1189, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, - "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/harnesses/docker_solver.py": {"lines": 897, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1200, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_agent_sandbox.py b/dokimi/tests/test_agent_sandbox.py index 0ee50a5ad..485b5dcf4 100644 --- a/dokimi/tests/test_agent_sandbox.py +++ b/dokimi/tests/test_agent_sandbox.py @@ -392,9 +392,8 @@ def fake_claude(call): prompt = seen["cmd"][seen["cmd"].index("-p") + 1] # The discovery arm installs the same plugin and only drops the notice. notice = arm == "skill" - assert prompt.startswith("A skill named `demo:demo` is installed." if notice - else "Solve this coding task.") - assert ("demo" in prompt.split("Solve this coding task.")[0]) is notice + assert prompt.startswith("A skill named `demo:demo` is installed.") is notice + assert ("A skill named" in prompt) is notice assert result.metrics["skill_notice"] is notice assert seen["cmd"][seen["cmd"].index("--output-format") + 1] == "stream-json" # stream-json under -p exits 1 with "requires --verbose" without it. diff --git a/dokimi/tests/test_cleanup_timeouts.py b/dokimi/tests/test_cleanup_timeouts.py index 396cfa83a..1296eb8fd 100644 --- a/dokimi/tests/test_cleanup_timeouts.py +++ b/dokimi/tests/test_cleanup_timeouts.py @@ -277,7 +277,7 @@ def test_plugin_removal_timeout_aborts_cell(tmp_path): """A plugin-directory removal that times out aborts the cell before verification.""" task = load_task(_fixture_task(tmp_path)) cli = _pipeline(reward=1.0, answer=lambda call: reply(timed_out=True) if ( - call.argv[1] == "exec" and "-rf" in call.argv and "abskill" in call.argv[-1]) else None) + call.argv[1] == "exec" and "-rf" in call.argv and "claude-plugins" in call.argv[-1]) else None) result = create_harness("docker-claude", runner=cli).run(task, output_dir=tmp_path / "out") diff --git a/dokimi/tests/test_container_agent.py b/dokimi/tests/test_container_agent.py index 0f86102d7..53478deb0 100644 --- a/dokimi/tests/test_container_agent.py +++ b/dokimi/tests/test_container_agent.py @@ -342,9 +342,9 @@ def test_container_solve_lifecycle(tmp_path, cli, docker_calls, monkeypatch): assert connect[2:4] == ["--alias", "egress"] run_call = next(c for c in docker_calls if c[0] == "run") assert run_call[run_call.index("--network") + 1].startswith("dokimi-net-") - assert any(c[0] == "cp" and c[-1].endswith(":/opt/dokimi-agent/claude") for c in docker_calls) + assert any(c[0] == "cp" and c[-1].endswith(":/opt/agent/claude") for c in docker_calls) assert seen["argv"][:4] == ["docker", "exec", "-u", AGENT_UID] - assert "/opt/dokimi-agent/claude" in seen["argv"] and "--tools" in seen["argv"] + assert "/opt/agent/claude" in seen["argv"] and "--tools" in seen["argv"] # Whatever the agent backgrounded is killed before the snapshot and the verifier's /tests. reap = next(i for i, c in enumerate(docker_calls) if c[:3] == ["exec", "-u", AGENT_UID] and "kill -9 -1" in c[-1]) @@ -404,10 +404,10 @@ def test_container_skill_arm_installs_a_plugin_outside_app(tmp_path, cli, docker def at(pred): return next(i for i, c in enumerate(docker_calls) if pred(c)) - mkdir = at(lambda c: c[-3:] == ["mkdir", "-p", "/opt/abskill"]) - cp_plugin = at(lambda c: c[0] == "cp" and c[-1].endswith(":/opt/abskill")) - chmod = at(lambda c: c[-4:] == ["chmod", "-R", "u=rwX,go=rX", "/opt/abskill"]) - rm_plugin = at(lambda c: c[-3:] == ["rm", "-rf", "/opt/abskill"]) + mkdir = at(lambda c: c[-3:] == ["mkdir", "-p", "/opt/claude-plugins"]) + cp_plugin = at(lambda c: c[0] == "cp" and c[-1].endswith(":/opt/claude-plugins")) + chmod = at(lambda c: c[-4:] == ["chmod", "-R", "u=rwX,go=rX", "/opt/claude-plugins"]) + rm_plugin = at(lambda c: c[-3:] == ["rm", "-rf", "/opt/claude-plugins"]) # Copy the plugin's contents into a target that exists, so it never nests. assert docker_calls[cp_plugin][1].endswith("/plugin/.") assert mkdir < cp_plugin < chmod < rm_plugin @@ -415,7 +415,7 @@ def at(pred): tests_in = [i for i, c in enumerate(docker_calls) if c[0] == "cp" and c[-1].endswith(":/tests")] assert all(rm_plugin < i for i in tests_in) argv = seen["argv"] - assert argv[argv.index("--plugin-dir") + 1] == "/opt/abskill" + assert argv[argv.index("--plugin-dir") + 1] == "/opt/claude-plugins" prompt = argv[argv.index("-p") + 1] assert prompt.startswith("A skill named `demo-skill:demo-skill` is installed.") assert "/app/.skill" not in " ".join(" ".join(c) for c in docker_calls) @@ -476,15 +476,15 @@ def test_container_discover_arm_is_the_skill_arm_without_the_notice( runs[arm] = (seen["argv"], result.metrics) (skill_argv, skill_m), (disc_argv, disc_m) = runs["skill"], runs["skill-discover"] prompt = disc_argv[disc_argv.index("-p") + 1] - assert prompt.startswith("Solve this coding task.") and "demo-skill" not in prompt + assert prompt == task.instruction and "demo-skill" not in prompt # Same claude argv otherwise: plugin dir, tools, settings; only the prompt differs. def claude_argv(argv): - argv = argv[argv.index("/opt/dokimi-agent/claude"):] + argv = argv[argv.index("/opt/agent/claude"):] return argv[:argv.index("-p") + 1] + argv[argv.index("-p") + 2:] assert claude_argv(skill_argv) == claude_argv(disc_argv) - assert disc_argv[disc_argv.index("--plugin-dir") + 1] == "/opt/abskill" + assert disc_argv[disc_argv.index("--plugin-dir") + 1] == "/opt/claude-plugins" assert skill_m["skill_notice"] is True and disc_m["skill_notice"] is False assert disc_m["treatment"] == skill_m["treatment"] assert disc_m["n_skill_invocations"] == 1 and "solver_error" not in disc_m @@ -555,7 +555,7 @@ def test_a_bad_skill_fails_before_any_docker_step(tmp_path, cli, docker_calls, m def test_a_failed_plugin_copy_leaves_no_staged_plugin(tmp_path, cli, docker_calls, monkeypatch): skill = _demo_skill(tmp_path) cli.answer = lambda call: (reply(stderr="docker cp failed", returncode=1) - if call.argv[1] == "cp" and call.argv[-1].endswith(":/opt/abskill") else None) + if call.argv[1] == "cp" and call.argv[-1].endswith(":/opt/claude-plugins") else None) _fake_claude(cli, {}) out = tmp_path / "out" harness = create_harness(f"docker-claude-skill:{skill}", runner=cli) @@ -572,7 +572,7 @@ def test_container_baseline_gets_no_plugin_and_no_notice(tmp_path, cli, docker_c create_harness("docker-claude", runner=cli).run(task, model="haiku", output_dir=tmp_path / "out") argv = seen["argv"] assert "--plugin-dir" not in argv - assert argv[argv.index("-p") + 1].startswith("Solve this coding task.") + assert argv[argv.index("-p") + 1] == task.instruction assert "Skill" in argv[argv.index("--tools") + 1].split(",") @@ -612,7 +612,7 @@ def test_container_probe_runs_as_the_agent_with_container_paths_forbidden(tmp_pa assert seen["argv"][:4] == ["docker", "exec", "-u", AGENT_UID] assert "/solution" in seen["env"]["PROBE_TASK_FILES"].split() assert result.reward == 1.0 and result.passed is True - assert not any(c[0] == "cp" and c[-1].endswith(":/opt/dokimi-agent/claude") for c in docker_calls) + assert not any(c[0] == "cp" and c[-1].endswith(":/opt/agent/claude") for c in docker_calls) def test_a_hung_container_probe_is_untested_not_sealed(tmp_path, cli, docker_calls, monkeypatch): @@ -880,12 +880,12 @@ def test_ca_and_credential_files_are_copied_in_and_repointed(tmp_path, cli, dock _fake_claude(cli, seen) create_harness("docker-claude", runner=cli).run(load_task(_task(tmp_path)), model="haiku", output_dir=tmp_path / "out") - assert any(c[:2] == ["cp", str(ca)] and c[2].endswith(":/opt/dokimi-agent/files/NODE_EXTRA_CA_CERTS") + assert any(c[:2] == ["cp", str(ca)] and c[2].endswith(":/opt/agent/files/NODE_EXTRA_CA_CERTS") for c in docker_calls) def test_a_binary_that_cannot_run_in_the_image_fails_clearly(tmp_path, cli, docker_calls, monkeypatch): - cli.answer = lambda call: reply(stderr="exec /opt/dokimi-agent/claude: no such file", returncode=127) if ( + cli.answer = lambda call: reply(stderr="exec /opt/agent/claude: no such file", returncode=127) if ( call.argv[1] == "exec" and call.argv[-1] == "--version") else None # Errored, with the isolation and egress record kept rather than raised away. result = create_harness("docker-claude", runner=cli).run(load_task(_task(tmp_path)), model="haiku", @@ -1044,7 +1044,7 @@ def test_container_skill_arm_ships_the_skill_with_links_followed(tmp_path, cli, def record(call): args = call.argv[1:] - if args[0] == "cp" and args[-1].endswith(":/opt/abskill"): + if args[0] == "cp" and args[-1].endswith(":/opt/claude-plugins"): links = [p for p in Path(args[1].rstrip(".")).rglob("*") if p.is_symlink()] refs = list(Path(args[1].rstrip(".")).rglob("references/a.md")) staged.update(links=links, ref=refs[0].read_text(encoding="utf-8") if refs else None) @@ -1214,7 +1214,7 @@ def test_the_plugin_modes_are_reset_not_widened(tmp_path, monkeypatch): harness._install_plugin_in_container("c1", tmp_path / "out", io.StringIO(), {}) chmod = next(c for c in cli.args if "chmod" in c) # a+rX would keep a setuid or world-writable file from the source as it is. - assert chmod == ["exec", "-u", "root", "c1", "chmod", "-R", "u=rwX,go=rX", "/opt/abskill"] + assert chmod == ["exec", "-u", "root", "c1", "chmod", "-R", "u=rwX,go=rX", "/opt/claude-plugins"] def test_an_announced_arm_does_not_warn_and_the_notice_is_on_by_default( diff --git a/dokimi/tests/test_payload_audit.py b/dokimi/tests/test_payload_audit.py index 1d2d284b2..bcf4a5882 100644 --- a/dokimi/tests/test_payload_audit.py +++ b/dokimi/tests/test_payload_audit.py @@ -10,7 +10,9 @@ import dokimi.eval.agent_runner as ar_mod from dokimi.eval.agent_runner import run_agent_loop from dokimi.harnesses import load_task -from dokimi.harnesses.docker_solver import DockerSolverHarness +from dokimi.harnesses.agent_sandbox import AgentWorkspace +from dokimi.harnesses.container_agent import AGENT_BIN +from dokimi.harnesses.docker_solver import _CONTAINER_PLUGIN, DockerSolverHarness from dokimi.harnesses.payload_audit import added_framing, audit_payload, markers_for, new_leaks from dokimi.harnesses.skill_treatment import TREATMENT_NOTICE, Treatment from dokimi.skills import load_skill @@ -124,6 +126,33 @@ def test_docker_claude_discover_prompt_is_task_framing_only(tmp_path): prompt = argv[argv.index("-p") + 1] prompts[notice] = added_framing(prompt, task.instruction) assert audit_payload(prompt, markers_for(paths=[str(task.path)])) == [] - # Discover keeps dokimi's task framing but drops the notice that names the skill. - assert "sum-fast" not in prompts[False] and prompts[False].startswith("Solve this coding task.") + # Discover adds nothing to the task instruction; the skill arm adds only the notice. + assert prompts[False] == "" assert prompts[True].startswith(TREATMENT_NOTICE.format(skill_id=treatment.skill_id).strip()) + + +def test_docker_claude_arms_send_no_harness_artifacts(tmp_path): + # The whole argv (plugin dir, tools, prompt) and the agent's cwd, as the solver sees them. + task = load_task(_task(tmp_path)) + treatment = Treatment(name="sum-fast", plugin="sum-fast") + out = tmp_path / "out" + out.mkdir() + markers = markers_for(paths=[str(task.path), str(tmp_path)]) + allowed = [task.instruction, TREATMENT_NOTICE.format(skill_id=treatment.skill_id)] + for notice in (True, False): + h = DockerSolverHarness(harness_id="docker-claude", mode="claude", skill_notice=notice) + argv = h._claude_args(task, treatment, _CONTAINER_PLUGIN, "haiku", out) + assert _CONTAINER_PLUGIN in argv and "abskill" not in " ".join(argv) + # The command the agent runs: the in-container executable, then the arguments. + assert audit_payload([AGENT_BIN, *argv], markers, allowed=allowed) == [] + ws = AgentWorkspace.create() + try: + assert audit_payload({"cwd": str(ws.work), "root": ws.root.name}, markers) == [] + # The host arm runs the binary on the host, with its plugin dir under the workspace root. + host = DockerSolverHarness(harness_id="docker-claude-host", mode="claude", + agent_location="host", skill_notice=True) + plugin = str(ws.root / "plugin") + argv = host._claude_args(task, treatment, plugin, "haiku", out) + assert audit_payload(["claude", *argv], markers, allowed=allowed) == [] + finally: + ws.cleanup()