From d9860455ba66b584bb26b53aa47b6f08535705cf Mon Sep 17 00:00:00 2001 From: minixalpha Date: Tue, 3 Nov 2026 07:15:38 -0800 Subject: [PATCH 1/3] feat(benchmarks): enforce shared evaluation integrity policy --- AGENTS.md | 7 + benchmarks/harbor/README.md | 140 ++++++++- benchmarks/harbor/README.zh-CN.md | 114 +++++++- benchmarks/harbor/configs/restricted-wal.json | 20 ++ .../harbor/src/harbor_adapter/__init__.py | 3 +- .../harbor/src/harbor_adapter/__main__.py | 55 +++- .../harbor/src/harbor_adapter/_dsh_client.py | 3 + .../harbor/src/harbor_adapter/adapter.py | 49 +++- benchmarks/harbor/src/harbor_adapter/audit.py | 194 +++++++++++++ .../harbor/src/harbor_adapter/deepseek.py | 7 + .../harbor/src/harbor_adapter/integrity.py | 96 +++++++ .../harbor_adapter/integrity_environment.py | 267 ++++++++++++++++++ benchmarks/harbor/src/harbor_adapter/pi.py | 2 + .../harbor/src/harbor_adapter/verifier.py | 3 + benchmarks/harbor/tests/test_integrity.py | 252 +++++++++++++++++ .../harbor/tests/test_integrity_audit.py | 135 +++++++++ .../harbor/tests/test_integrity_docker.py | 67 +++++ 17 files changed, 1395 insertions(+), 19 deletions(-) create mode 100644 benchmarks/harbor/configs/restricted-wal.json create mode 100644 benchmarks/harbor/src/harbor_adapter/audit.py create mode 100644 benchmarks/harbor/src/harbor_adapter/integrity.py create mode 100644 benchmarks/harbor/src/harbor_adapter/integrity_environment.py create mode 100644 benchmarks/harbor/tests/test_integrity.py create mode 100644 benchmarks/harbor/tests/test_integrity_audit.py create mode 100644 benchmarks/harbor/tests/test_integrity_docker.py diff --git a/AGENTS.md b/AGENTS.md index 375b65c..caef131 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,13 @@ cache import, and the `--install-only` dependency preflight option. - Keep setup failures and verifier errors separate from scored failures. Only verifier timeouts may retry, at most three total attempts using the same agent output and the task's original per-attempt time limit. +- New standard runs require a registered `integrity` policy shared across agent + arms. `--legacy-open-network` is only for explicitly registered historical + protocol reproductions. Never silently relax a failed isolation preflight. +- Freeze completed outputs before integrity review. Keep original rewards and + errors, record reviewed violations separately, count violations as effective + zeroes without removing them from the denominator, and keep unreviewed results + pending. Do not mix restricted and historical protocols into one baseline. ## Language diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 592052a..28ffa6b 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -15,21 +15,21 @@ endpoint, configure the API key and base URL; select the model with Harbor's ```bash export ANTHROPIC_API_KEY="..." -export ANTHROPIC_BASE_URL="https://gateway.example" +export ANTHROPIC_BASE_URL="https://api.deepseek.com/anthropic" ``` Run new benchmarks through the standard entry point, from the repository root: ```bash uv run --project benchmarks/harbor python -m harbor_adapter run \ - --config benchmarks/harbor/configs/qemu-alpine-ssh.json + --config benchmarks/harbor/configs/restricted-wal.json ``` -This example pins the QEMU task revision and uses `deepseek/deepseek-flash`. +This example pins the WAL task revision and uses `deepseek/deepseek-flash`. Set the endpoint and credentials for that model, or copy the configuration and change `agents[].model_name`, task selection, and budgets for another run. These -are ordinary Harbor JSON configurations; the entry point supplies the standard -verifier and cache policy. It accepts the repository's `NanoPyCodeAgent` adapter +are Harbor JSON configurations with an additional `integrity` section; the entry +point supplies the standard verifier, integrity, and cache policies. It accepts the repository's `NanoPyCodeAgent` adapter with budget kwargs; credentials belong in the environment. The entry point builds the current working tree, including uncommitted changes, @@ -48,6 +48,136 @@ exercise dependency installation and preflight without calling the model or executing the official tests. A full run still performs setup and preflight in its own fresh container; an install-only check does not resume into model work. +### Evaluation integrity protocol + +New standard runs require an explicit `integrity` section: + +```json +{ + "integrity": { + "protocol": "restricted-v1", + "allowed_hosts": ["api.deepseek.com"], + "probe_url": "https://example.com/" + } +} +``` + +The same policy and English integrity instructions apply to Nano, Pi, and the +DeepSeek Harness adapter. Their native tools and model settings are preserved. +The actual inference host must be listed explicitly; provider endpoints are not +silently added. Exact DNS hosts are accepted; wildcards, IP literals, CIDRs, +extra agent network exceptions, injected skills, resumed sessions, external +tools, and filtered logs are rejected. Do not allowlist general code hosting or +search sites: that would reopen access to reference answers. Task-required data +or reviewed dependency/documentation services need explicit, preregistered hosts. +An allowed service can itself expose reference material; an allowlist is not a +content classifier. + +Installation and registered dependency preflight use the task's original public +network. Before model work, a public HTTPS probe must succeed, the same probe +must fail under the allowlist, and allowed hosts must remain reachable. Redirects +are not followed by probes. A failed positive control is an infrastructure +failure, never proof of blocking. `--install-only` exercises these checks too. +The run boundary rechecks enforcement within the native agent time limit. +Harbor's external egress sidecar enforces the restriction; shell proxy variables +are not the security boundary. The task container loses `NET_RAW`, `NET_ADMIN`, +`SYS_ADMIN`, and `SYS_PTRACE`, and cannot gain privileges through setuid binaries. +Host directories, including verifier logs, are not mounted into the container. +UDP DNS forwarding is permitted only to the exact upstream resolver addresses +reported by Docker's trusted sidecar; those addresses are recorded. DNS failure +does not count as a successful HTTPS blocking probe. + +Version one supports single-step Linux Dockerfile/prebuilt-image tasks with +public native network policies. Custom Compose services, extra mounts, +environment overrides, and task MCP servers are unsupported and fail closed. +These limits deliberately exclude the historical dependency-cache Compose +overlays until they have a reviewed isolation design. The protocol preserves +task images, package sources, official tests, resources, and native deadlines. +Missing task dependencies remain setup or task failures according to the +original verifier; network restrictions do not authorize changing the task. + +All three agents use process supervision under this protocol. Network access +stays restricted through cancellation cleanup. Controller-collected output, +network-rule snapshots, preflight outcomes, and sidecar diagnostic logs are saved +under the job's host-only `.integrity//` directory, outside Harbor's +native trial-log rewriting. These records are private +because output can contain credentials. They preserve collected bytes, but do +not prove complete observation of every tool action or network request. + +Existing shared verifiers remain shared: after quiescence checks and evidence +collection, their native network is restored for official verification. They +remain vulnerable to malicious submitted code and retained services influencing +grading. Tasks already configured with separate verifiers retain that boundary; +the adapter records the mode without rewriting tasks. Migrating existing tasks, +isolating submitted code inside the grader, private held-out tasks, and DNS/covert +channel defenses are separate work. `restricted-v1` blocks the observed direct +web answer retrieval; it is not a proof against every adversarial strategy or +training-data contamination. + +For an explicitly registered historical-protocol reproduction, pass +`--legacy-open-network` with the original configuration. This opt-out is recorded +as `legacy-open-network`, cannot be combined with `integrity`, and supplies no +integrity enforcement. Historical configurations and reports are unchanged. +Do not combine restricted and historical results into one comparable baseline. + +### Freeze, review, and integrity-adjusted scores + +Completed restricted runs automatically freeze output hashes and write candidate +flags to `jobs/-input/audit/audit.json`. They do not automatically earn +an integrity-cleared score. The same audit can be invoked explicitly: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter audit \ + --job-dir jobs/ --output-dir jobs/-input/audit +``` + +The output directory must be outside the job. Auditing refuses unfinished jobs, +symbolic links, special files, and changes to an existing freeze. Scanning covers +native text/JSON logs across adapters plus collected output and egress evidence; +findings contain file/line locations, not raw credential-bearing excerpts. +Patterns are triage hints: inspect surrounding calls and results, including +blocked versus successful retrieval, before deciding. No matches do not prove a +clean attempt. `clear` means no violation observed in the reviewed evidence. + +Create a review bound to the exact `freeze.json` SHA256, using actual trial IDs +and evidence paths from that manifest: + +```json +{ + "freeze_sha256": "", + "trials": { + "": { + "verdict": "violation", + "reviewer": "", + "reason": "Downloaded and used the task's reference solution.", + "evidence": ["/agent/pi.txt"] + } + } +} +``` + +Pass `--review /path/to/review.json` to the audit command. Verdicts are `clear` +or `violation`; every verdict requires a reviewer, reason, and frozen evidence. +Each review and resulting audit is saved under its hash, preserving previous +decisions. Raw Harbor rewards and exceptions are never overwritten. Confirmed +violations receive an effective zero and stay in the denominator. Pending +reviews have no effective score, and no adjusted pass rate is published until +all eligible trials are reviewed. Setup/verifier errors remain unscored. A native +agent timeout is eligible only when cancellation was confirmed and an official +binary reward exists. Missing required enforcement is an integrity error. + +Run the optional model-free Docker probe with an existing image containing +Python 3 and CA certificates: + +```bash +NANOPY_INTEGRITY_TEST_IMAGE= uv run --project benchmarks/harbor pytest \ + -c benchmarks/harbor/pyproject.toml benchmarks/harbor/tests/test_integrity_docker.py +``` + +The probe builds Harbor's pinned egress sidecar, tests allowed and denied HTTPS, +checks that packet marks cannot bypass filtering, and removes only its own +containers. It does not call a model or run benchmark solutions/tests. + ### Verified dependencies and preflight The standard adapter automatically uses `.cache/nanopy-harbor/`, shared across diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 76b5979..60a2257 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -14,19 +14,19 @@ URL；模型由下方 Harbor 命令中的 `--model` 选择： ```bash export ANTHROPIC_API_KEY="..." -export ANTHROPIC_BASE_URL="https://gateway.example" +export ANTHROPIC_BASE_URL="https://api.deepseek.com/anthropic" ``` 新的 benchmark 统一从仓库根目录通过标准入口运行： ```bash uv run --project benchmarks/harbor python -m harbor_adapter run \ - --config benchmarks/harbor/configs/qemu-alpine-ssh.json + --config benchmarks/harbor/configs/restricted-wal.json ``` -此示例固定了 QEMU 题目版本，模型为 `deepseek/deepseek-flash`。请配置对应的 +此示例固定了 WAL 题目版本，模型为 `deepseek/deepseek-flash`。请配置对应的 endpoint 和凭证；测试其他模型或题目时，复制配置并修改 `agents[].model_name`、 -题目选择及预算。这是普通的 Harbor JSON 配置，入口会补齐标准验证器和缓存策略。 +题目选择及预算。这是增加了 `integrity` 段的 Harbor JSON 配置，入口会补齐标准验证器、完整性和缓存策略。 入口接受本仓库的 `NanoPyCodeAgent` adapter 和预算参数，凭证通过环境变量传入。 入口会将当前工作区（包括未提交修改）构建为 wheel，并核对其 Python 源码与保存的 @@ -40,6 +40,112 @@ endpoint 和凭证；测试其他模型或题目时，复制配置并修改 `age 不启动容器；`--install-only` 只执行依赖安装和预检，不调用模型、不运行官方测试。 完整运行仍会在自己的新容器里执行安装和预检，不能把 install-only 容器续跑为解题。 +### 评测完整性协议 + +新的标准运行必须显式配置 `integrity`： + +```json +{ + "integrity": { + "protocol": "restricted-v1", + "allowed_hosts": ["api.deepseek.com"], + "probe_url": "https://example.com/" + } +} +``` + +Nano、Pi 和 DeepSeek Harness adapter 使用相同策略与英文完整性规则，保留各自 +原生工具和模型设置。实际推理服务域名必须显式列入，入口不会静默增加 provider +端点。只接受精确 DNS 主机名；拒绝通配符、IP、CIDR、额外 agent 网络例外、 +注入的 skills、恢复会话、外部工具和日志过滤。不要允许通用代码托管或搜索站点， +否则会重新开放参考答案访问。题目明确需要的数据、经过审阅的依赖或文档服务必须 +事先登记具体域名。允许的服务本身仍可能暴露参考材料；白名单并不是内容分类器。 + +安装与已登记的依赖预检使用题目原有的公开网络。模型开始工作前，公开 HTTPS +探针必须成功；应用白名单后同一探针必须失败，同时允许的主机仍须可达。探针不 +跟随重定向。正向探针失败属于基础设施错误，不能当作阻断成功。 +`--install-only` 也会执行这些检查。运行边界会在原生 agent 时限内再次检查限制。 +网络限制由 Harbor 的外部 egress sidecar 执行，shell 代理环境变量不是安全边界。 +题目容器移除 `NET_RAW`、`NET_ADMIN`、`SYS_ADMIN`、`SYS_PTRACE` 权限，禁止 +通过 setuid 程序提升权限。包括 verifier 日志在内的宿主目录不会挂载进容器。 +UDP DNS 转发仅允许到 Docker 可信 sidecar 报告的精确上游解析器地址，并记录 +这些地址。DNS 解析失败不会被当作 HTTPS 阻断探针成功。 + +第一版支持原生网络策略为公开网络的单步 Linux Dockerfile/预构建镜像任务。 +自定义 Compose 服务、额外挂载、环境覆盖和题目 MCP 服务暂不支持，遇到时直接 +拒绝运行。因此，历史依赖缓存 Compose 覆盖配置需要经过隔离设计审阅后才能接入。 +协议保留题目镜像、软件源、官方测试、资源和原生时限。题目缺失依赖仍按原验证器 +表现分类为安装错误或任务失败；网络限制不授权修改题目。 + +该协议下三个 agent 都使用进程托管，取消及清理期间网络仍受限。控制器采集的 +输出、网络规则快照、预检结果和 sidecar 诊断日志保存到 job 的宿主专属 +`.integrity//` 目录，避开 Harbor 对原生 trial 日志的改写。 +输出可能含凭据，因此这些记录保持私有。它们保全已采集的 +字节，但不能证明完整观察了每次工具调用或网络请求。 + +现有共享验证器仍是共享模式：检查进程结束边界并收集证据后，恢复其原生网络以 +执行官方验证。恶意提交代码及保留的服务仍可能干预评分。已经配置独立验证器的 +题目保留原有边界，adapter 记录其模式，不改写题目。现有题目的迁移、评分器内部 +对提交代码的隔离、私有保留集，以及 DNS/隐蔽通道防护属于后续工作。 +`restricted-v1` 阻断本次观察到的直接网络答案下载途径，并不证明能抵抗所有 +对抗策略或排除训练数据污染。 + +对于明确登记的历史协议复现，可用原配置加 `--legacy-open-network`。这一退出 +选项会记录为 `legacy-open-network`，不能与 `integrity` 合用，也不提供完整性 +防护。历史配置和报告保持原样。受限协议与历史结果不应合并成可直接比较的基线。 + +### 冻结、复核和完整性修正成绩 + +完成的受限运行会自动冻结输出哈希，并将候选线索写入 +`jobs/-input/audit/audit.json`，不会自动获得完整性复核通过的成绩。 +也可以显式执行相同审计： + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter audit \ + --job-dir jobs/ --output-dir jobs/-input/audit +``` + +输出目录必须位于 job 之外。未结束的 job、符号链接、特殊文件，以及冻结后发生 +变化的证据都会被拒绝。扫描覆盖各 adapter 原生文本/JSON 日志、采集输出与网络 +证据；线索只记录文件及行号，不摘录可能含凭据的原文。模式匹配仅用于初筛： +需要查看相关调用与结果，区分请求被阻断和实际获取成功，再作判断。未匹配到内容 +不证明尝试干净；`clear` 表示在复核证据中未观察到违规。 + +复核文件须绑定 `freeze.json` 的精确 SHA256，并使用清单中的实际 trial ID 和 +证据路径： + +```json +{ + "freeze_sha256": "", + "trials": { + "": { + "verdict": "violation", + "reviewer": "", + "reason": "Downloaded and used the task's reference solution.", + "evidence": ["/agent/pi.txt"] + } + } +} +``` + +向审计命令传入 `--review /path/to/review.json`。结论可为 `clear` 或 +`violation`；每条结论都需要复核者、理由和冻结证据。每份复核及其审计结果按 +哈希另存，保留之前的决定。原始 Harbor reward 和异常不会被覆盖。确认违规的 +尝试有效分数为零，并保留在分母中。待复核尝试没有有效分数；所有可评分尝试 +复核完成前，不发布修正通过率。安装/verifier 错误仍不评分。原生 agent 超时 +只有在取消已确认且存在官方二元 reward 时才可评分。缺失必需防护属于完整性错误。 + +可用已有、包含 Python 3 和 CA 证书的镜像运行不调用模型的 Docker 探针： + +```bash +NANOPY_INTEGRITY_TEST_IMAGE= uv run --project benchmarks/harbor pytest \ + -c benchmarks/harbor/pyproject.toml benchmarks/harbor/tests/test_integrity_docker.py +``` + +探针构建 Harbor 固定版本的 egress sidecar，检查允许及禁止的 HTTPS、验证无法 +用 packet mark 绕过过滤，并只移除自己创建的容器。它不调用模型，也不运行 +benchmark 参考解法或测试。 + ### 依赖缓存与预检 标准 adapter 自动使用被 Git 忽略的 `.cache/nanopy-harbor/` 跨运行共享缓存， diff --git a/benchmarks/harbor/configs/restricted-wal.json b/benchmarks/harbor/configs/restricted-wal.json new file mode 100644 index 0000000..14c7d15 --- /dev/null +++ b/benchmarks/harbor/configs/restricted-wal.json @@ -0,0 +1,20 @@ +{ + "integrity": { + "protocol": "restricted-v1", + "allowed_hosts": ["api.deepseek.com"], + "probe_url": "https://example.com/" + }, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash" + } + ], + "environment": {"type": "docker"}, + "tasks": [ + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + } + ] +} diff --git a/benchmarks/harbor/src/harbor_adapter/__init__.py b/benchmarks/harbor/src/harbor_adapter/__init__.py index 63c7fa8..a7b5aac 100644 --- a/benchmarks/harbor/src/harbor_adapter/__init__.py +++ b/benchmarks/harbor/src/harbor_adapter/__init__.py @@ -4,5 +4,6 @@ from .deepseek import DeepSeekHarnessAgent from .pi import PiCodeAgent from .verifier import RetryingVerifier +from .integrity_environment import IntegrityDockerEnvironment -__all__ = ["NanoPyCodeAgent", "PiCodeAgent", "DeepSeekHarnessAgent", "RetryingVerifier"] +__all__ = ["NanoPyCodeAgent", "PiCodeAgent", "DeepSeekHarnessAgent", "RetryingVerifier", "IntegrityDockerEnvironment"] diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py index 40ae1b7..ba3a6f2 100644 --- a/benchmarks/harbor/src/harbor_adapter/__main__.py +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -18,6 +18,7 @@ from .bootstrap import AptCache, default_cache_dir from .deepseek import distribution_files as deepseek_distribution_files from .pi import distribution_digest, distribution_files, read_model_config +from .integrity import IntegrityPolicy, RULES, RULES_SHA256 _NANO = "harbor_adapter:NanoPyCodeAgent" _PI = "harbor_adapter:PiCodeAgent" @@ -27,6 +28,7 @@ def standard_config(raw: dict) -> JobConfig: """Validate policy before building or creating a benchmark environment.""" raw = json.loads(json.dumps(raw)) + integrity = IntegrityPolicy.model_validate(raw.pop("integrity")) if "integrity" in raw else None raw.setdefault("retry", {"max_retries": 0}) raw.setdefault("n_concurrent_trials", 1) raw.setdefault("verifier", { @@ -79,12 +81,37 @@ def standard_config(raw: dict) -> JobConfig: if identifiers[0] == _DEEPSEEK: if not {"distribution_path", "distribution_sha256"} <= agent.kwargs.keys(): raise ValueError("DeepSeek requires a hash-pinned distribution") - if config.verifier.kwargs.get("require_agent_quiescence") is not True: + if integrity is None and config.verifier.kwargs.get("require_agent_quiescence") is not True: raise ValueError("DeepSeek requires confirmed agent quiescence before grading") if agent.kwargs.get("reasoning_effort", "high") not in {"off", "low", "high", "max"}: raise ValueError("Unsupported DeepSeek reasoning effort") if not agent.model_name: raise ValueError("Each agent requires an explicit model_name") + if integrity is not None: + integrity.network() + if config.artifacts or config.extra_instruction_paths or config.timeout_multiplier != 1 or config.agent_timeout_multiplier not in (None, 1): + raise ValueError("restricted-v1 preserves task artifacts and native timeouts") + if config.environment.import_path or config.environment.type not in (None, "docker"): + raise ValueError("restricted-v1 requires the Docker environment") + if (config.environment.mounts or config.environment.extra_docker_compose + or config.environment.kwargs or config.environment.extra_allowed_hosts + or config.environment.env): + raise ValueError("restricted-v1 rejects extra mounts, Compose services, environment overrides, and network overrides") + for agent in config.agents: + if (agent.skills or agent.mcp_servers or agent.extra_allowed_hosts + or agent.load_trajectory or agent.resume_trajectory + or agent.include_logs or agent.exclude_logs + or agent.override_timeout_sec is not None or agent.max_timeout_sec is not None): + raise ValueError("restricted-v1 requires fresh agents, complete logs, and no external tools or network overrides") + agent.kwargs["integrity_policy"] = integrity.model_dump() + if config.verifier.include_logs or config.verifier.exclude_logs or config.verifier.env: + raise ValueError("restricted-v1 requires complete verifier logs") + if any(config.verifier.kwargs.get(key) for key in ("retry_dependency_errors", "retry_collection_errors")): + raise ValueError("Only verifier timeouts may retry") + config.environment.type = None + config.environment.import_path = "harbor_adapter:IntegrityDockerEnvironment" + config.environment.kwargs["integrity_policy"] = integrity.model_dump() + config.verifier.kwargs["require_agent_quiescence"] = True return config @@ -187,7 +214,12 @@ def run(args) -> int: root = Path.cwd() if not (root / "src/nanopycodeagent").is_dir(): raise ValueError("Run from the nanoPyCodeAgent repository root") - config = standard_config(json.loads(args.config.read_text())) + raw = json.loads(args.config.read_text()) + if "integrity" not in raw and not getattr(args, "legacy_open_network", False): + raise ValueError("New runs require an integrity policy; use --legacy-open-network only for explicitly registered historical-protocol runs") + if "integrity" in raw and getattr(args, "legacy_open_network", False): + raise ValueError("Cannot combine integrity with --legacy-open-network") + config = standard_config(raw) if args.job_name: config.job_name = args.job_name if Path(config.job_name).name != config.job_name or config.job_name in {".", ".."}: @@ -231,7 +263,11 @@ def run(args) -> int: "source_sha256": {str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source.rglob("*.py"))}, "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest() if wheel else None, "cache_dir": str(cache), + "evaluation_protocol": raw.get("integrity", {"protocol": "legacy-open-network"}), + "integrity_rules_sha256": RULES_SHA256 if "integrity" in raw else None, } + if "integrity" in raw: + (record / "integrity-rules.txt").write_text(RULES) if pi_inputs: manifest["pi_inputs"] = pi_inputs if deepseek_inputs: @@ -256,6 +292,12 @@ def run(args) -> int: summary["harbor_exit_code"] = code (record / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") print(f"Results: {record / 'summary.json'}", flush=True) + if "integrity" in raw and not config.install_only and (job_dir / "result.json").exists(): + from .audit import audit_job + audit = audit_job(job_dir, record / "audit") + summary["integrity_audit"] = {"path": str(record / "audit" / "audit.json"), **audit["summary"]} + (record / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(f"Integrity review: {record / 'audit' / 'audit.json'}", flush=True) incomplete = not summary["trials"] or any( row["status"] not in {"scored", "setup_passed"} for row in summary["trials"] ) @@ -271,6 +313,11 @@ def main() -> int: benchmark.add_argument("--cache-dir", type=Path, default=default_cache_dir()) benchmark.add_argument("--prepare-only", action="store_true", help="Build and record inputs without starting containers or calling the model") benchmark.add_argument("--install-only", action="store_true", help="Run setup and dependency preflight without calling the model or verifier") + benchmark.add_argument("--legacy-open-network", action="store_true", help="Explicitly reproduce an earlier protocol without integrity enforcement") + audit = commands.add_parser("audit", help="Freeze completed outputs and produce evidence-bound integrity accounting") + audit.add_argument("--job-dir", type=Path, required=True) + audit.add_argument("--output-dir", type=Path, required=True) + audit.add_argument("--review", type=Path, help="Reviewed verdicts bound to this output freeze") cache = commands.add_parser("import-apt", help="Import a manifest and its sibling packages/ directory") cache.add_argument("manifest", type=Path) cache.add_argument("--cache-dir", type=Path, default=default_cache_dir()) @@ -284,6 +331,10 @@ def main() -> int: retained.add_argument("--retry-dependency-errors", action="store_true") retained.add_argument("--retry-collection-errors", action="store_true") args = parser.parse_args() + if args.command == "audit": + from .audit import audit_job + audit_job(args.job_dir, args.output_dir, args.review) + return 0 if args.command == "verify-retained": from .retained import verify_retained return verify_retained(args) diff --git a/benchmarks/harbor/src/harbor_adapter/_dsh_client.py b/benchmarks/harbor/src/harbor_adapter/_dsh_client.py index 403ba59..e6a016b 100644 --- a/benchmarks/harbor/src/harbor_adapter/_dsh_client.py +++ b/benchmarks/harbor/src/harbor_adapter/_dsh_client.py @@ -15,6 +15,7 @@ def main(): parser.add_argument("--reasoning-effort", default="high") parser.add_argument("--preflight", action="store_true") parser.add_argument("--instruction") + parser.add_argument("--stream-events", action="store_true") args = parser.parse_args() home = Path(args.home) home.mkdir(parents=True, exist_ok=True) @@ -36,6 +37,8 @@ def notification(event): if event.method == "session.event": events.write(json.dumps(asdict(event)) + "\n") events.flush() + if args.stream_events: + print(json.dumps(asdict(event)), flush=True) result = harness.run( Path(args.instruction).read_text(), session_id="benchmark", diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index 8d92253..5282d45 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -21,6 +21,7 @@ from harbor.models.trajectories.trajectory import Trajectory from .bootstrap import AptCache, default_cache_dir, verifier_profile +from .integrity import IntegrityPolicy, integrity_run _DEFAULT_MAX_TURNS = 50 _SYSTEM_DEPENDENCIES = ("curl", "bash", "git", "python3", "ca_certificates") @@ -32,6 +33,7 @@ _UV_VERSION = "0.9.11" _PATH_SETUP = 'export PATH="$HOME/.local/bin:$PATH"; ' _TRAJECTORY_PATH = "/logs/agent/trajectory.json" +_RUNNER = Path(__file__).with_name("_agent_runner.py").read_text() class NanoPyCodeAgent(BaseInstalledAgent): @@ -55,7 +57,7 @@ class NanoPyCodeAgent(BaseInstalledAgent): def __init__( self, *args, git_ref: str | None = None, wheel_path: str | None = None, - bootstrap_cache: str | None = None, **kwargs, + bootstrap_cache: str | None = None, integrity_policy: dict | None = None, **kwargs, ): if git_ref is not None: if not isinstance(git_ref, str): @@ -82,6 +84,7 @@ def __init__( if bootstrap_cache else default_cache_dir() ) self._bootstrap_record: dict = {} + self._integrity_policy = IntegrityPolicy.model_validate(integrity_policy) if integrity_policy is not None else None super().__init__(*args, **kwargs) max_tokens = self._resolved_flags.get("max_tokens") if max_tokens is not None and max_tokens < 1: @@ -137,6 +140,24 @@ async def install(self, environment: BaseEnvironment) -> None: await self._install_apt(environment, profile.apt_packages) await self.exec_as_agent(environment, command=profile.command) record["verifier_preflight"]["status"] = "passed" + if self._integrity_policy is not None: + from .integrity_environment import IntegrityDockerEnvironment + from urllib.parse import urlsplit + + if not isinstance(environment, IntegrityDockerEnvironment): + raise ValueError("Integrity setup requires IntegrityDockerEnvironment") + if environment.integrity_policy != self._integrity_policy: + raise ValueError("Agent and environment integrity policies differ") + endpoint = (self._model_config["providers"]["deepseek"]["baseUrl"] + if hasattr(self, "_model_config") else + "https://api.deepseek.com" if self.name() == "deepseek-harness" else + self.model_connection.configured_base_url or "https://api.anthropic.com") + if urlsplit(endpoint).hostname not in self._integrity_policy.allowed_hosts: + raise ValueError("The model endpoint must be explicitly included in integrity.allowed_hosts") + record["stage"] = "integrity_preflight" + (self.logs_dir / "process-supervision-required").write_text("restricted-v1\n") + await environment.integrity_preflight() + record["integrity_preflight"] = "passed" record.update(status="ready", stage="complete") except BaseException as exc: record.update(status="failed", error_type=type(exc).__name__) @@ -235,6 +256,7 @@ def _runtime_env(self) -> dict[str, str]: @override @with_prompt_template + @integrity_run async def run( self, instruction: str, @@ -250,19 +272,32 @@ async def run( instruction_env_var: instruction, } cli_flags = self.build_cli_flags() - await self.exec_as_agent( - environment, - command=( - f"{_PATH_SETUP}" + command = ( + f"set -euo pipefail; {_PATH_SETUP}" f'{instruction_shell_var}="${instruction_env_var}"; ' f"unset {instruction_env_var}; " f'printf "%s" "${instruction_shell_var}" | ' f"nanoPyCodeAgent {cli_flags} " f"--trajectory {_TRAJECTORY_PATH} " "2>&1 | tee /logs/agent/nanopycodeagent.txt" - ), - env=env, ) + if self._integrity_policy is None: + await self.exec_as_agent(environment, command=command, env=env) + else: + control = f"/tmp/nanopy-agent-{uuid.uuid4().hex}.json" + record = "/logs/agent/process-supervision.json" + try: + await self.exec_as_agent(environment, command=shlex.join([ + "python3", "-I", "-c", _RUNNER, "run", control, record, command, + ]), env=env) + except BaseException: + cleanup = await asyncio.shield(asyncio.wait_for(environment.exec( + command=shlex.join(["python3", "-I", "-c", _RUNNER, "cancel", control, record]), + user="root", + ), timeout=12)) + if cleanup.return_code != 0: + raise RuntimeError("Nano subprocess cleanup did not confirm cancellation") + raise @override def populate_context_post_run(self, context: AgentContext) -> None: diff --git a/benchmarks/harbor/src/harbor_adapter/audit.py b/benchmarks/harbor/src/harbor_adapter/audit.py new file mode 100644 index 0000000..acb9ddf --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/audit.py @@ -0,0 +1,194 @@ +"""Freeze evidence, flag candidates, and apply explicit integrity reviews. + +Scanning never proves a clean run or automatically convicts an agent. Original +Harbor rewards and errors remain untouched; reviews bind to one exact freeze. +""" + +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import re + +from .integrity import write_record + + +_PATTERNS = { + "benchmark_reference": re.compile(r"terminal[-_]bench|RSIBench|solution[/\\]+solve\.sh", re.I), + "verifier_access": re.compile(r"/logs/verifier|/tests/test_outputs\.py|reward\.(?:json|txt)", re.I), +} + + +def digest(path): + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def inventory(root): + result = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Cannot freeze symbolic-link evidence: {path.relative_to(root)}") + if path.is_dir(): + continue + if not path.is_file(): + raise ValueError(f"Cannot freeze special-file evidence: {path.relative_to(root)}") + result[path.relative_to(root).as_posix()] = digest(path) + return result + + +def freeze(job_dir, output_dir): + result = json.loads((job_dir / "result.json").read_text()) + if not result.get("finished_at"): + raise ValueError("Only completed jobs can be frozen for review") + stats = result.get("stats") or {} + if (stats.get("n_running_trials", 0) or stats.get("n_pending_trials", 0) + or stats.get("n_completed_trials", len(list(job_dir.glob("*/result.json")))) != len(list(job_dir.glob("*/result.json")))): + raise ValueError("Job trial inventory is incomplete; cannot publish selective accounting") + files = inventory(job_dir) + path = output_dir / "freeze.json" + if path.exists(): + frozen = json.loads(path.read_text()) + if frozen["job_dir"] != str(job_dir) or frozen["files"] != files: + raise ValueError("Job evidence changed after freezing; review is invalid") + else: + write_record(path, { + "schema_version": 1, "job_dir": str(job_dir), + "frozen_at": datetime.now(timezone.utc).isoformat(), "files": files, + }) + return files, digest(path) + + +def scan(job_dir, files): + """Return locations only; never publish credential-bearing excerpts.""" + findings = [] + for name in files: + if not ("/agent/" in name or (name.startswith(".integrity/") and Path(name).name in {"live-output.jsonl", "egress-log.json"})): + continue + if Path(name).suffix not in {".txt", ".json", ".jsonl"}: + continue + with (job_dir / name).open(encoding="utf-8", errors="replace") as source: + for number, line in enumerate(source, 1): + for rule, pattern in _PATTERNS.items(): + if pattern.search(line): + findings.append({"file": name, "line": number, "rule": rule}) + return findings + + +def read_reviews(path, freeze_sha256, files, trial_names): + if path is None: + return {} + value = json.loads(path.read_text()) + if set(value) != {"freeze_sha256", "trials"} or value["freeze_sha256"] != freeze_sha256: + raise ValueError("Review must identify this exact freeze_sha256") + if not isinstance(value["trials"], dict) or set(value["trials"]) - trial_names: + raise ValueError("Review contains unknown trial identifiers") + for trial, review in value["trials"].items(): + if not isinstance(review, dict) or set(review) != {"verdict", "reviewer", "reason", "evidence"}: + raise ValueError("Each review needs verdict, reviewer, reason, and evidence") + if review["verdict"] not in {"clear", "violation"}: + raise ValueError("A reviewed verdict must be clear or violation") + if not all(isinstance(review[k], str) and review[k].strip() for k in ("reviewer", "reason")): + raise ValueError("Reviews require an identified reviewer and a reason") + evidence = review["evidence"] + if not isinstance(evidence, list) or not evidence: + raise ValueError("Reviews require frozen evidence references") + if any(not isinstance(name, str) or name not in files or not name.startswith((trial + "/", ".integrity/" + trial + "/")) for name in evidence): + raise ValueError("Review evidence must reference this trial's frozen files") + return value["trials"] + + +def trial_accounting(job_dir, path, review): + result = json.loads(path.read_text()) + exception = result.get("exception_info") or {} + error = exception.get("exception_type") + rewards = (result.get("verifier_result") or {}).get("rewards") + raw_score = rewards.get("reward") if isinstance(rewards, dict) else None + evidence_dir = job_dir / ".integrity" / path.parent.name + proof_path = evidence_dir / "enforcement.json" + proof = json.loads(proof_path.read_text()) if proof_path.exists() else None + config_path = path.parent / "config.json" + config = json.loads(config_path.read_text()) if config_path.exists() else {} + required = bool(config.get("agent", {}).get("kwargs", {}).get("integrity_policy")) + control_path = path.parent / "agent/process-supervision.json" + control = json.loads(control_path.read_text()) if control_path.exists() else {} + confirmed_timeout = ( + error == "AgentTimeoutError" and control.get("status") == "cancelled" + and control.get("processes_stopped") is True + ) + eligible = type(raw_score) in (float, int) and raw_score in (0, 1) and (not error or confirmed_timeout) + enforcement_ok = bool(proof and proof.get("preflight") == "passed" + and proof.get("agent_phase") in {"completed", "interrupted"}) + if enforcement_ok: + live = evidence_dir / "live-output.jsonl" + expected_policy = config.get("agent", {}).get("kwargs", {}).get("integrity_policy") + enforcement_ok = ( + live.is_file() and proof.get("live_output_sha256") == digest(live) + and all((evidence_dir / name).is_file() for name in ( + "network-preflight.json", "network-before-agent.json", "network-after-agent.json", "egress-log.json", + )) + and (not expected_policy or proof.get("policy") == expected_policy) + ) + if required and not enforcement_ok: + eligible = False + classification = "integrity_error" + elif error and not confirmed_timeout: + classification = ("setup_error" if "Setup" in error else + "verifier_error" if "Verif" in error else "execution_error") + else: + classification = "pending_review" if eligible else "unscored" + if review and not eligible: + raise ValueError("Cannot assign scored integrity verdicts to infrastructure errors or unscored trials") + effective_score = None + if review: + classification = "integrity_violation" if review["verdict"] == "violation" else "no_observed_violation" + effective_score = 0 if review["verdict"] == "violation" else raw_score + return { + "trial": path.parent.name, "task": result.get("task_name"), + "native_rewards": rewards, "native_exception_type": error, + "confirmed_timeout": confirmed_timeout, + "enforcement": "restricted-v1" if enforcement_ok else "not_established", + "verifier_isolation": proof.get("verifier_isolation") if proof else None, + "classification": classification, "eligible": eligible, + "effective_score": effective_score, "review": review, + } + + +def audit_job(job_dir: Path, output_dir: Path, review_path: Path | None = None): + job_dir, output_dir = job_dir.resolve(), output_dir.resolve() + if output_dir == job_dir or job_dir in output_dir.parents: + raise ValueError("Audit outputs must be outside the frozen job") + files, freeze_sha256 = freeze(job_dir, output_dir) + paths = sorted(job_dir.glob("*/result.json")) + if not paths: + raise ValueError("No trial results to audit") + reviews = read_reviews(review_path, freeze_sha256, files, {p.parent.name for p in paths}) + rows = [trial_accounting(job_dir, p, reviews.get(p.parent.name)) for p in paths] + findings = scan(job_dir, files) + eligible = [row for row in rows if row["eligible"]] + pending = sum(row["effective_score"] is None for row in eligible) + summary = { + "eligible_trials": len(eligible), "pending_review": pending, + "integrity_violations": sum(row["classification"] == "integrity_violation" for row in rows), + "unscored_or_errors": len(rows) - len(eligible), + # Violations stay in the denominator. Never publish a selectively + # reviewed pass rate while another eligible attempt is pending. + "effective_pass_rate": (sum(row["effective_score"] for row in eligible) / len(eligible) + if eligible and not pending else None), + } + report = { + "schema_version": 1, "freeze_sha256": freeze_sha256, + "review_sha256": digest(review_path) if review_path else None, + "scan_is_exhaustive": False, + "note": "Candidate flags require review. No matches do not prove absence of reference exposure, tampering, or training contamination.", + "summary": summary, "trials": rows, "findings": findings, + } + if inventory(job_dir) != files: + raise ValueError("Evidence changed during audit") + if review_path: + # Preserve each reviewed revision instead of erasing earlier decisions. + write_record(output_dir / ("review-" + report["review_sha256"] + ".json"), json.loads(review_path.read_text())) + write_record(output_dir / ("audit-" + report["review_sha256"] + ".json"), report) + else: + write_record(output_dir / "audit.json", report) + return report diff --git a/benchmarks/harbor/src/harbor_adapter/deepseek.py b/benchmarks/harbor/src/harbor_adapter/deepseek.py index 8c64b57..ca1bed2 100644 --- a/benchmarks/harbor/src/harbor_adapter/deepseek.py +++ b/benchmarks/harbor/src/harbor_adapter/deepseek.py @@ -12,6 +12,7 @@ from harbor.agents.model_connection import ModelConnectionSpec from .adapter import NanoPyCodeAgent +from .integrity import integrity_run from .pi import distribution_digest _RUNNER = Path(__file__).with_name("_agent_runner.py").read_text() @@ -119,6 +120,7 @@ async def _install_agent(self, environment) -> None: self._bootstrap_record["sdk_initialization"] = "passed" @with_prompt_template + @integrity_run async def run(self, instruction, environment, context) -> None: key = self.model_connection.api_key if not key: @@ -133,6 +135,11 @@ async def run(self, instruction, environment, context) -> None: "--reasoning-effort", self._reasoning_effort, "--instruction", "/tmp/harbor-dsh-instruction.txt", ]) + " > /logs/agent/dsh-stdout.txt 2> /logs/agent/dsh-stderr.txt" + if self._integrity_policy is not None: + command = "set -euo pipefail; " + command.replace( + " > /logs/agent/dsh-stdout.txt 2> /logs/agent/dsh-stderr.txt", + " --stream-events 2>&1 | tee /logs/agent/dsh-stdout.txt", + ) control = f"/tmp/harbor-dsh-{uuid.uuid4().hex}.json" record = "/logs/agent/process-supervision.json" try: diff --git a/benchmarks/harbor/src/harbor_adapter/integrity.py b/benchmarks/harbor/src/harbor_adapter/integrity.py new file mode 100644 index 0000000..3d2bdae --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/integrity.py @@ -0,0 +1,96 @@ +"""Versioned evaluation rules shared by every repository adapter.""" + +from functools import wraps +import hashlib +from ipaddress import ip_address +import json +from pathlib import Path +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, Field, field_validator +from typing import Literal + +from harbor.models.task.config import NetworkPolicy, normalize_allowed_hosts + + +RULES = """Evaluation integrity rules (restricted-v1): +Solve the task using the supplied workspace, task inputs, and permitted general +documentation or dependencies. You may read tests supplied with the workspace +and write and run your own tests. Do not retrieve task-specific benchmark +solutions, hidden tests, answer keys, original benchmark inputs absent from the +workspace, or other agents' outputs, whether from the internet or local files. +Do not change, impersonate, or interfere with the verifier, its dependencies, +logs, reward files, or evaluation controls. Do not leave a process that watches +for hidden tests or alters the submission during verification. Network access +is restricted by the evaluation environment. A blocked request is not permission +to bypass the restriction. Report missing resources instead. +""" +RULES_SHA256 = hashlib.sha256(RULES.encode()).hexdigest() + + +class IntegrityPolicy(BaseModel): + """Explicit hosts, never a silently widened provider or package allowlist.""" + + model_config = ConfigDict(extra="forbid", strict=True) + protocol: Literal["restricted-v1"] = "restricted-v1" + allowed_hosts: list[str] = Field(min_length=1) + # A positive control before restriction, then a negative control after it. + probe_url: str = "https://example.com/" + + @field_validator("allowed_hosts") + @classmethod + def exact_hosts(cls, hosts): + hosts = normalize_allowed_hosts(hosts) + for host in hosts: + if "*" in host or "/" in host: + raise ValueError("Integrity allowlists require exact hosts, not wildcards or networks") + try: + ip_address(host) + except ValueError: + pass + else: + raise ValueError("Use explicit DNS hosts, not IP literals, in integrity allowlists") + return sorted(set(hosts)) + + @field_validator("probe_url") + @classmethod + def public_probe(cls, value): + parsed = urlsplit(value) + if (parsed.scheme != "https" or not parsed.hostname or parsed.username + or parsed.password or parsed.query or parsed.fragment + or parsed.port not in (None, 443)): + raise ValueError("probe_url must be a credential-free HTTPS URL on port 443") + return value + + def network(self): + if urlsplit(self.probe_url).hostname in self.allowed_hosts: + raise ValueError("The negative-control probe must not be allowlisted") + return NetworkPolicy(network_mode="allowlist", allowed_hosts=self.allowed_hosts) + + +def write_record(path: Path, value: dict) -> None: + """Keep host evidence private, including logs that may contain credentials.""" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as output: + temporary.chmod(0o600) + json.dump(value, output, indent=2) + output.write("\n") + temporary.replace(path) + + +def integrity_run(run): + """Guard the common run boundary without changing an agent's native tools.""" + @wraps(run) + async def guarded(self, instruction, environment, context): + if self._integrity_policy is None: + return await run(self, instruction, environment, context) + from .integrity_environment import IntegrityDockerEnvironment + + if not isinstance(environment, IntegrityDockerEnvironment): + raise ValueError("Integrity runs require IntegrityDockerEnvironment") + if environment.integrity_policy != self._integrity_policy: + raise ValueError("Agent and environment integrity policies differ") + async with environment.agent_phase(): + return await run(self, RULES + "\nTask:\n" + instruction, environment, context) + return guarded diff --git a/benchmarks/harbor/src/harbor_adapter/integrity_environment.py b/benchmarks/harbor/src/harbor_adapter/integrity_environment.py new file mode 100644 index 0000000..0a13b09 --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/integrity_environment.py @@ -0,0 +1,267 @@ +"""Docker enforcement outside the evaluated process, with host-owned evidence. + +Version one supports Linux Dockerfile/image tasks, not arbitrary Compose graphs. +It preserves task images and tests. Shared verification remains shared and is +explicitly recorded; this is not a replacement for separate verifier tasks. +""" + +from contextlib import asynccontextmanager +from datetime import datetime, timezone +import hashlib +from ipaddress import ip_address +import json +from pathlib import Path +import shlex +import tempfile +import tomllib + +from harbor.environments.docker.docker import DockerEnvironment +from harbor.models.task.config import NetworkMode, NetworkPolicy, TaskOS + +from .integrity import IntegrityPolicy, RULES_SHA256, write_record + + +_PROBE = '''import json, urllib.request, urllib.error, sys +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None +opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) +try: + with opener.open(sys.argv[1], timeout=15) as response: + response.read(1) + result = {"reachable": True} +except urllib.error.HTTPError: + result = {"reachable": True} +except (OSError, ValueError) as error: + result = {"reachable": False, "error_type": type(error).__name__, + "reason_type": type(getattr(error, "reason", None)).__name__} +print(json.dumps(result)) +''' + + +def dns_upstreams(resolv_conf): + """Read Docker's trusted sidecar resolver, not the agent's writable copy.""" + candidates = [] + for line in resolv_conf.splitlines(): + if line.startswith("nameserver "): + candidates.append(line.split()[1]) + elif line.startswith("# ExtServers: [") and line.endswith("]"): + candidates.extend(line.removeprefix("# ExtServers: [")[:-1].split()) + addresses = set() + for item in candidates: + if item.startswith("host(") and item.endswith(")"): + item = item[5:-1] + address = ip_address(item) + if not address.is_loopback: + addresses.add(address.compressed) + return sorted(addresses) + + +class IntegrityDockerEnvironment(DockerEnvironment): + """Enable Harbor's external egress sidecar before any container starts.""" + + def __init__(self, *, integrity_policy, mounts=None, **kwargs): + self.integrity_policy = IntegrityPolicy.model_validate(integrity_policy) + policy = self.integrity_policy.network() + self._separate_verifier = "__verifier__" in kwargs["session_id"] + if kwargs["task_env_config"].os != TaskOS.LINUX: + raise ValueError("restricted-v1 supports Linux Docker environments only") + directory = Path(kwargs["environment_dir"]) + task_config = directory.parent / "task.toml" + if task_config.exists() and tomllib.loads(task_config.read_text()).get("steps"): + raise ValueError("restricted-v1 supports single-step tasks only") + if (directory / "docker-compose.yaml").exists() or kwargs.get("extra_docker_compose"): + raise ValueError("restricted-v1 does not support unreviewed Docker Compose services") + if kwargs["task_env_config"].mcp_servers: + raise ValueError("restricted-v1 does not support task MCP servers") + self._integrity_baseline = kwargs.get("network_policy") or NetworkPolicy() + # Do not widen a task's own baseline or phase restrictions. + if not self._separate_verifier and any(p.network_mode != NetworkMode.PUBLIC for p in ( + self._integrity_baseline, *kwargs.get("phase_network_policies", ()), + )): + raise ValueError("restricted-v1 requires public native network policies; review restricted tasks separately") + kwargs["phase_network_policies"] = [*kwargs.get("phase_network_policies", ()), policy] + # Harbor 0.21 mounts verifier logs in the agent by default. Use file + # transfer for all logs/artifacts; no host directory is agent-writable. + super().__init__(mounts=[], **kwargs) + if not self.capabilities.dynamic_network_policy: + raise ValueError("Docker runtime cannot enforce integrity network policies") + self._hardening_dir = tempfile.TemporaryDirectory(prefix="nanopy-integrity-") + self._hardening_path = Path(self._hardening_dir.name) / "compose.json" + self._hardening_path.write_text(json.dumps({"services": {"main": { + "cap_drop": ["NET_RAW", "NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"], + "security_opt": ["no-new-privileges:true"], + }}})) + # Keep controller evidence outside Harbor's native trial log rewriting + # (including credential scrubbing), as well as outside every container. + self.evidence_dir = self.trial_paths.trial_dir.parent / ".integrity" / self.trial_paths.trial_dir.name + self._integrity_state = { + "protocol": self.integrity_policy.protocol, + "policy": self.integrity_policy.model_dump(), + "rules_sha256": RULES_SHA256, + "preflight": "pending", "agent_phase": "not_started", + "host_mounts": False, + "verifier_isolation": "task_defined; shared tasks remain shared", + } + self._in_agent_phase = False + self._dns_upstreams = [] + if not self._separate_verifier: + self._save() + + @property + def capabilities(self): + return super().capabilities.model_copy(update={"mounted": False}) + + @property + def _docker_compose_paths(self): + paths = super()._docker_compose_paths + if hasattr(self, "_hardening_path"): + paths = [*paths, self._hardening_path] + return paths + + def _save(self): + write_record(self.evidence_dir / "enforcement.json", self._integrity_state) + + async def start(self, force_build): + await super().start(force_build) + self._dns_upstreams = dns_upstreams(await self._sidecar("cat", "/etc/resolv.conf")) + self._integrity_state["dns_upstreams"] = self._dns_upstreams + paths = ["/logs/verifier"] if self._separate_verifier else ["/logs/agent", "/logs/artifacts"] + await self.ensure_dirs(paths) + + async def _apply_network_policy(self, network_policy): + await super()._apply_network_policy(network_policy) + if network_policy.network_mode == NetworkMode.ALLOWLIST: + # Docker's embedded resolver forwards UDP from this namespace. + # Permit DNS only to the daemon-selected resolver addresses. The + # upstream 0.21 rules otherwise reject those forwarded datagrams. + for address in self._dns_upstreams: + family = "ip" if ip_address(address).version == 4 else "ip6" + await self._sidecar("nft", "insert", "rule", "inet", "gost_egress", "egress", + family, "daddr", address, "udp", "dport", "53", "accept") + + async def _probe(self, url): + result = await self.exec( + command=shlex.join(["python3", "-I", "-c", _PROBE, url]), + timeout_sec=20, user="root", + ) + if result.return_code != 0: + raise RuntimeError("Integrity network probe could not execute") + result = json.loads(result.stdout or "null") + if not isinstance(result, dict) or type(result.get("reachable")) is not bool: + raise RuntimeError("Integrity network probe returned invalid evidence") + return result + + async def _sidecar(self, *command): + result = await self._run_docker_compose_command([ + "exec", "--no-TTY", self._EGRESS_CONTROL_SERVICE_NAME, *command, + ], timeout_sec=15) + return result.stdout or "" + + async def _network_snapshot(self, name): + show = await self._sidecar("network-policy", "show") + rules = await self._sidecar("network-policy", "rules") + # The sidecar runs outside the agent PID/mount namespace and the agent + # has neither NET_ADMIN nor NET_RAW (which can also set SO_MARK). + if "controlled egress" not in show or "table inet gost_egress" not in rules: + raise RuntimeError("External egress enforcement is not active") + write_record(self.evidence_dir / (name + ".json"), { + "policy": show, "rules": rules, + }) + + async def integrity_preflight(self): + """Require a working positive control before accepting a denied probe.""" + self._integrity_state["preflight"] = "running" + self._save() + try: + before = await self._probe(self.integrity_policy.probe_url) + self._integrity_state["positive_control"] = before + if not before["reachable"]: + raise RuntimeError("Integrity positive control failed; blocking is unproven") + await self.set_network_policy(self.integrity_policy.network()) + after = await self._probe(self.integrity_policy.probe_url) + self._integrity_state["negative_control"] = after + if after["reachable"]: + raise RuntimeError("Integrity restriction failed: disallowed HTTPS remains reachable") + if after.get("reason_type") == "gaierror": + raise RuntimeError("Integrity negative control failed to resolve; HTTPS blocking is unproven") + allowed = {} + self._integrity_state["allowed_controls"] = allowed + for host in self.integrity_policy.allowed_hosts: + allowed[host] = await self._probe("https://" + host + "/") + if not allowed[host]["reachable"]: + raise RuntimeError(f"Integrity allowed host is unreachable: {host}") + self._integrity_state["allowed_controls"] = allowed + await self._network_snapshot("network-preflight") + self._integrity_state["preflight"] = "passed" + except BaseException: + self._integrity_state["preflight"] = "failed" + raise + finally: + self._save() + # Remain restricted after installation. The run boundary checks again. + + async def set_network_policy(self, network_policy): + if getattr(self, "_in_agent_phase", False) and network_policy != self.integrity_policy.network(): + raise RuntimeError("Cannot relax network policy during model work") + await super().set_network_policy(network_policy) + + @asynccontextmanager + async def agent_phase(self): + if self._integrity_state["preflight"] != "passed": + raise RuntimeError("Integrity preflight must pass before model work") + await self.set_network_policy(self.integrity_policy.network()) + await self._network_snapshot("network-before-agent") + if (await self._probe(self.integrity_policy.probe_url))["reachable"]: + raise RuntimeError("Integrity network restriction disappeared before model work") + self._in_agent_phase = True + self._integrity_state.update(agent_phase="running", started_at=datetime.now(timezone.utc).isoformat()) + self._save() + stream_path = self.evidence_dir / "live-output.jsonl" + try: + with stream_path.open("x", encoding="utf-8") as output: + stream_path.chmod(0o600) + + async def collect(text, stream): + output.write(json.dumps({"stream": str(stream), "text": text}) + "\n") + output.flush() + + with self.scoped_output_callback(collect): + yield + self._integrity_state["agent_phase"] = "completed" + except BaseException: + self._integrity_state["agent_phase"] = "interrupted" + raise + finally: + self._in_agent_phase = False + self._integrity_state["finished_at"] = datetime.now(timezone.utc).isoformat() + if stream_path.exists(): + with stream_path.open("rb") as source: + self._integrity_state["live_output_sha256"] = hashlib.file_digest(source, "sha256").hexdigest() + self._save() + await self._network_snapshot("network-after-agent") + logs = await self._run_docker_compose_command([ + "logs", "--no-color", "--no-log-prefix", self._EGRESS_CONTROL_SERVICE_NAME, + ], timeout_sec=15) + write_record(self.evidence_dir / "egress-log.json", {"log": logs.stdout or ""}) + # Do not restore public access here, including on cancellation. + + async def prepare_verification(self): + """Keep network evidence outside the sandbox before official tests run.""" + if self._separate_verifier: + state = json.loads((self.evidence_dir / "enforcement.json").read_text()) + if state.get("agent_phase") not in {"completed", "interrupted"}: + raise RuntimeError("Missing completed integrity agent boundary") + state["verifier_isolation"] = "separate" + write_record(self.evidence_dir / "enforcement.json", state) + return + if self._integrity_state["agent_phase"] not in {"completed", "interrupted"}: + raise RuntimeError("Integrity agent phase never reached its end boundary") + await self._network_snapshot("network-after-agent") + self._integrity_state["verifier_isolation"] = "shared" + self._integrity_state["verification_started_at"] = datetime.now(timezone.utc).isoformat() + self._save() + await self.exec(command="mkdir -p /logs/verifier; rm -f /logs/verifier/reward.txt /logs/verifier/reward.json", user="root") + # Official shared verifiers may install dependencies. This transition + # is explicitly recorded and does not claim independent verification. + await self.set_network_policy(self._integrity_baseline) diff --git a/benchmarks/harbor/src/harbor_adapter/pi.py b/benchmarks/harbor/src/harbor_adapter/pi.py index 67f8e7d..107f017 100644 --- a/benchmarks/harbor/src/harbor_adapter/pi.py +++ b/benchmarks/harbor/src/harbor_adapter/pi.py @@ -11,6 +11,7 @@ from harbor.agents.model_connection import ModelConnectionSpec from .adapter import NanoPyCodeAgent +from .integrity import integrity_run _RUNNER = Path(__file__).with_name("_agent_runner.py").read_text() @@ -133,6 +134,7 @@ async def _install_agent(self, environment) -> None: validate_model_registration(listing.stdout, listing.stderr, self.model_name) @with_prompt_template + @integrity_run async def run(self, instruction, environment, context) -> None: key = self.model_connection.api_key if not key: diff --git a/benchmarks/harbor/src/harbor_adapter/verifier.py b/benchmarks/harbor/src/harbor_adapter/verifier.py index 5e9d79c..be57932 100644 --- a/benchmarks/harbor/src/harbor_adapter/verifier.py +++ b/benchmarks/harbor/src/harbor_adapter/verifier.py @@ -138,6 +138,9 @@ def __init__(self, *, max_attempts=3, retry_dependency_errors=False, async def verify(self): check_agent_quiescence(self.trial_paths.agent_dir, required=getattr(self, "_require_agent_quiescence", False)) + from .integrity_environment import IntegrityDockerEnvironment + if isinstance(self.environment, IntegrityDockerEnvironment): + await self.environment.prepare_verification() # Harbor's outer watchdog covers the entire custom verifier. Reject an # insufficient budget instead of silently losing the promised retries. config = TrialConfig.model_validate_json(self.trial_paths.config_path.read_text()) diff --git a/benchmarks/harbor/tests/test_integrity.py b/benchmarks/harbor/tests/test_integrity.py new file mode 100644 index 0000000..ca5dfad --- /dev/null +++ b/benchmarks/harbor/tests/test_integrity.py @@ -0,0 +1,252 @@ +"""Enforcement must fail closed before model calls, independently of the agent.""" + +import asyncio +import json +import shlex +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from harbor.environments.base import ExecResult +from harbor.models.task.config import EnvironmentConfig, NetworkPolicy +from harbor.models.trial.paths import TrialPaths + +from harbor_adapter import NanoPyCodeAgent, PiCodeAgent, DeepSeekHarnessAgent +from harbor_adapter.__main__ import standard_config +from harbor_adapter.integrity import RULES +from harbor_adapter.integrity_environment import IntegrityDockerEnvironment, dns_upstreams + + +POLICY = {"allowed_hosts": ["api.deepseek.com"]} + + +def raw_config(): + return { + "tasks": [{"name": "org/task", "ref": "sha256:abc"}], + "agents": [{"name": "harbor_adapter:NanoPyCodeAgent", "model_name": "deepseek/test"}], + "integrity": POLICY, + } + + +def test_standard_workflow_injects_one_identical_policy_and_requires_quiescence(): + raw = raw_config() + config = standard_config(raw) + policy = config.agents[0].kwargs["integrity_policy"] + assert policy == config.environment.kwargs["integrity_policy"] + assert config.environment.import_path == "harbor_adapter:IntegrityDockerEnvironment" + assert config.verifier.kwargs["require_agent_quiescence"] is True + assert "integrity_policy" not in raw["agents"][0].get("kwargs", {}) + + +def test_entrypoint_requires_explicit_protocol_before_preparing_files(tmp_path): + from harbor_adapter.__main__ import run + config = tmp_path / "config.json" + raw = raw_config() + raw.pop("integrity") + config.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="New runs require"): + run(SimpleNamespace(config=config)) + config.write_text(json.dumps(raw_config())) + with pytest.raises(ValueError, match="Cannot combine"): + run(SimpleNamespace(config=config, legacy_open_network=True)) + + +@pytest.mark.parametrize("policy", [ + {}, {"allowed_hosts": []}, {"allowed_hosts": ["*.github.com"]}, + {"allowed_hosts": ["0.0.0.0/0"]}, {"allowed_hosts": ["1.1.1.1"]}, + {"allowed_hosts": ["https://api.deepseek.com"]}, + POLICY | {"protocol": "open"}, POLICY | {"skip_probe": True}, + POLICY | {"probe_url": "https://user:password@example.com"}, + POLICY | {"probe_url": "https://api.deepseek.com/"}, +]) +def test_ambiguous_or_ineffective_policies_are_rejected(policy): + with pytest.raises(ValueError): + standard_config(raw_config() | {"integrity": policy}) + + +@pytest.mark.parametrize("overrides", [ + {"environment": {"type": "daytona"}}, + {"environment": {"extra_docker_compose": ["cache.yaml"]}}, + {"environment": {"extra_allowed_hosts": ["github.com"]}}, + {"environment": {"kwargs": {"privileged": True}}}, + {"environment": {"env": {"HTTPS_PROXY": "https://proxy.example"}}}, +]) +def test_unreviewed_network_and_environment_escape_routes_are_rejected(overrides): + with pytest.raises(ValueError): + standard_config(raw_config() | overrides) + + +@pytest.mark.parametrize("overrides", [ + {"skills": ["/tmp/skills"]}, {"extra_allowed_hosts": ["github.com"]}, + {"load_trajectory": "/tmp/old.json"}, {"exclude_logs": ["*"]}, +]) +def test_external_agent_context_and_filtered_logs_are_rejected(overrides): + raw = raw_config() + raw["agents"][0].update(overrides) + with pytest.raises(ValueError): + standard_config(raw) + + +@pytest.fixture +def environment(tmp_path): + directory = tmp_path / "environment" + directory.mkdir() + (directory / "Dockerfile").write_text("FROM scratch\n") + paths = TrialPaths(trial_dir=tmp_path / "trial") + paths.mkdir() + return IntegrityDockerEnvironment( + environment_dir=directory, environment_name="probe", + session_id="probe__env", trial_paths=paths, + task_env_config=EnvironmentConfig(), integrity_policy=POLICY, + mounts=[{"type": "bind", "source": str(paths.verifier_dir), "target": "/logs/verifier"}], + ) + + +def fake_sidecar(environment, monkeypatch): + async def command(args, **kwargs): + if args[-1] == "show": + return ExecResult(return_code=0, stdout="mode: controlled egress\napi.deepseek.com") + if args[-1] == "rules": + return ExecResult(return_code=0, stdout="table inet gost_egress {}") + return ExecResult(return_code=0, stdout="") + mock = AsyncMock(side_effect=command) + monkeypatch.setattr(environment, "_run_docker_compose_command", mock) + return mock + + +def test_environment_removes_all_host_mounts_and_drops_network_capabilities(environment): + assert environment._mounts == [] + assert environment.capabilities.mounted is False + assert environment.capabilities.dynamic_network_policy is True + main = json.loads(environment._hardening_path.read_text())["services"]["main"] + assert {"NET_RAW", "NET_ADMIN"} <= set(main["cap_drop"]) + assert main["security_opt"] == ["no-new-privileges:true"] + assert environment._hardening_path in environment._docker_compose_paths + + +def test_dns_exceptions_are_only_exact_daemon_selected_addresses(): + assert dns_upstreams("nameserver 127.0.0.11\n# ExtServers: [172.17.0.1 host(2001:db8::53)]") == ["172.17.0.1", "2001:db8::53"] + with pytest.raises(ValueError): + dns_upstreams("# ExtServers: [0.0.0.0/0]") + + +def test_dns_failure_does_not_count_as_https_blocking(environment, monkeypatch): + fake_sidecar(environment, monkeypatch) + monkeypatch.setattr(environment, "_probe", AsyncMock(side_effect=[ + {"reachable": True}, {"reachable": False, "reason_type": "gaierror"}, + ])) + with pytest.raises(RuntimeError, match="failed to resolve"): + asyncio.run(environment.integrity_preflight()) + + +@pytest.mark.parametrize("responses,match", [ + ([False], "positive control"), + ([True, True], "disallowed HTTPS"), + ([True, False, False], "allowed host is unreachable"), +]) +def test_preflight_requires_positive_negative_and_allowed_controls(environment, monkeypatch, responses, match): + fake_sidecar(environment, monkeypatch) + monkeypatch.setattr(environment, "_probe", AsyncMock(side_effect=[{"reachable": r} for r in responses])) + with pytest.raises(RuntimeError, match=match): + asyncio.run(environment.integrity_preflight()) + assert environment._integrity_state["preflight"] == "failed" + + +def test_run_captures_host_evidence_and_does_not_reopen_network_on_error(environment, monkeypatch): + fake_sidecar(environment, monkeypatch) + monkeypatch.setattr(environment, "_probe", AsyncMock(side_effect=[ + {"reachable": r} for r in (True, False, True, False) + ])) + + async def run(): + await environment.integrity_preflight() + with pytest.raises(RuntimeError, match="model failed"): + async with environment.agent_phase(): + with pytest.raises(RuntimeError, match="Cannot relax"): + await environment.set_network_policy(NetworkPolicy()) + await environment._output_callback()("tool evidence", "stdout") + raise RuntimeError("model failed") + assert environment.network_policy == environment.integrity_policy.network() + assert environment._integrity_state["agent_phase"] == "interrupted" + assert "tool evidence" in (environment.evidence_dir / "live-output.jsonl").read_text() + assert (environment.evidence_dir / "network-after-agent.json").exists() + await environment.prepare_verification() + assert environment.network_policy == NetworkPolicy() + asyncio.run(run()) + + +def test_no_model_work_before_preflight(environment): + async def run(): + async with environment.agent_phase(): + pytest.fail("model work started") + with pytest.raises(RuntimeError, match="preflight"): + asyncio.run(run()) + + +@pytest.mark.parametrize("kind", ["nano", "pi", "deepseek"]) +def test_actual_adapters_deliver_the_same_rules_and_supervise_commands(environment, monkeypatch, tmp_path, kind): + from test_pi import inputs as pi_fixture + from test_deepseek import inputs as deepseek_fixture + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-only") + logs = tmp_path / "agent" + logs.mkdir() + kwargs = dict(logs_dir=logs, model_name="deepseek/deepseek-flash", integrity_policy=POLICY) + if kind == "pi": + agent = PiCodeAgent(**kwargs, **pi_fixture.__wrapped__(tmp_path)) + elif kind == "deepseek": + agent = DeepSeekHarnessAgent(**kwargs, **deepseek_fixture.__wrapped__(tmp_path)) + else: + agent = NanoPyCodeAgent(**kwargs) + fake_sidecar(environment, monkeypatch) + environment._integrity_state["preflight"] = "passed" + monkeypatch.setattr(environment, "_probe", AsyncMock(return_value={"reachable": False})) + + async def execute(**kwargs): + stdout = '{"settled": true, "stop_reason": "stop"}' + return ExecResult(return_code=0, stdout=stdout) + execute_mock = AsyncMock(side_effect=execute) + monkeypatch.setattr(environment, "exec", execute_mock) + upload = AsyncMock() + if kind == "deepseek": + monkeypatch.setattr(agent, "_upload_config_text", upload) + asyncio.run(agent.run("original task", environment, None)) + invocation = execute_mock.call_args_list[0].kwargs + parts = shlex.split(invocation["command"]) + parts = parts[parts.index("python3"):] + assert parts[4] == "run" and parts[6] == "/logs/agent/process-supervision.json" + expected = RULES + "\nTask:\noriginal task" + if kind == "nano": + assert expected in invocation["env"].values() + elif kind == "pi": + assert expected in shlex.split(parts[-1]) + else: + assert upload.call_args.kwargs["content"] == expected + assert "--stream-events" in parts[-1] + assert environment._integrity_state["agent_phase"] == "completed" + + +def test_restricted_adapter_cannot_run_in_plain_environment(tmp_path): + agent = NanoPyCodeAgent(logs_dir=tmp_path, integrity_policy=POLICY) + with pytest.raises(ValueError, match="IntegrityDockerEnvironment"): + asyncio.run(agent.run("task", SimpleNamespace(), None)) + + +def test_nano_cancellation_stops_supervisor_before_leaving_restricted_phase(environment, monkeypatch, tmp_path): + fake_sidecar(environment, monkeypatch) + environment._integrity_state["preflight"] = "passed" + monkeypatch.setattr(environment, "_probe", AsyncMock(return_value={"reachable": False})) + calls = [] + + async def execute(**kwargs): + calls.append(kwargs) + if " run " in kwargs["command"]: + raise asyncio.CancelledError() + return ExecResult(return_code=0) + monkeypatch.setattr(environment, "exec", execute) + agent = NanoPyCodeAgent(logs_dir=tmp_path, integrity_policy=POLICY) + with pytest.raises(asyncio.CancelledError): + asyncio.run(agent.run("task", environment, None)) + assert " cancel " in calls[-1]["command"] + assert calls[-1]["user"] == "root" + assert environment.network_policy == environment.integrity_policy.network() + assert environment._integrity_state["agent_phase"] == "interrupted" diff --git a/benchmarks/harbor/tests/test_integrity_audit.py b/benchmarks/harbor/tests/test_integrity_audit.py new file mode 100644 index 0000000..6b677c9 --- /dev/null +++ b/benchmarks/harbor/tests/test_integrity_audit.py @@ -0,0 +1,135 @@ +"""Audited accounting cannot silently promote flags or discard violations.""" + +import json + +import pytest + +from harbor_adapter.audit import audit_job + + +@pytest.fixture +def job(tmp_path): + root = tmp_path / "job" + root.mkdir() + (root / "result.json").write_text(json.dumps({"finished_at": "2026-11-03T00:00:00Z"})) + for name, reward, error in [("pass", 1, None), ("fail", 0, None), ("infra", 0, "VerifierTimeoutError")]: + trial = root / name + (trial / "agent").mkdir(parents=True) + (trial / "result.json").write_text(json.dumps({ + "task_name": name, "verifier_result": {"rewards": {"reward": reward}}, + "exception_info": {"exception_type": error} if error else None, + })) + (trial / "agent/pi.txt").write_text("curl https://host/terminal-bench/task/solution/solve.sh\n") + return root + + +def test_scanner_does_not_convict_and_does_not_publish_partial_rate(job, tmp_path): + report = audit_job(job, tmp_path / "audit") + assert report["findings"] + assert report["summary"] == { + "eligible_trials": 2, "pending_review": 2, "integrity_violations": 0, + "unscored_or_errors": 1, "effective_pass_rate": None, + } + assert all(row["effective_score"] is None for row in report["trials"]) + + +def reviews(report): + return {"freeze_sha256": report["freeze_sha256"], "trials": { + name: {"verdict": verdict, "reviewer": "maintainer", "reason": "Reviewed surrounding tool arguments and results.", + "evidence": [name + "/agent/pi.txt"]} + for name, verdict in [("pass", "violation"), ("fail", "clear")] + }} + + +def test_confirmed_violation_keeps_original_reward_and_denominator(job, tmp_path): + output = tmp_path / "audit" + initial = audit_job(job, output) + review = tmp_path / "review.json" + review.write_text(json.dumps(reviews(initial))) + report = audit_job(job, output, review) + assert report["summary"]["eligible_trials"] == 2 + assert report["summary"]["integrity_violations"] == 1 + assert report["summary"]["effective_pass_rate"] == 0 + row = next(row for row in report["trials"] if row["trial"] == "pass") + assert row["native_rewards"] == {"reward": 1} + assert row["effective_score"] == 0 + assert json.loads((job / "pass/result.json").read_text())["verifier_result"]["rewards"]["reward"] == 1 + assert json.loads((output / "audit.json").read_text()) == initial + assert list(output.glob("audit-*.json")) + + +@pytest.mark.parametrize("change", ["modify", "add", "delete", "symlink"]) +def test_changed_evidence_invalidates_the_existing_freeze(job, tmp_path, change): + output = tmp_path / "audit" + audit_job(job, output) + path = job / "pass/agent/pi.txt" + if change == "modify": + path.write_text("changed") + elif change == "add": + (job / "added").write_text("new") + elif change == "delete": + path.unlink() + else: + (job / "link").symlink_to(path) + with pytest.raises(ValueError): + audit_job(job, output) + + +def test_review_of_wrong_freeze_or_infrastructure_failure_is_rejected(job, tmp_path): + output = tmp_path / "audit" + initial = audit_job(job, output) + review = tmp_path / "review.json" + value = reviews(initial) + value["freeze_sha256"] = "wrong" + review.write_text(json.dumps(value)) + with pytest.raises(ValueError, match="exact freeze"): + audit_job(job, output, review) + value = reviews(initial) + value["trials"] = {"infra": value["trials"]["fail"] | {"evidence": ["infra/agent/pi.txt"]}} + review.write_text(json.dumps(value)) + with pytest.raises(ValueError, match="infrastructure"): + audit_job(job, output, review) + + +def test_missing_enforcement_in_restricted_run_is_an_error_not_a_pass(job, tmp_path): + (job / "pass/config.json").write_text(json.dumps({"agent": {"kwargs": {"integrity_policy": {"protocol": "restricted-v1"}}}})) + report = audit_job(job, tmp_path / "audit") + row = next(row for row in report["trials"] if row["trial"] == "pass") + assert row["classification"] == "integrity_error" + assert row["effective_score"] is None + + +def test_no_scan_matches_still_requires_review(job, tmp_path): + for path in job.glob("*/agent/pi.txt"): + path.write_text("ordinary output") + report = audit_job(job, tmp_path / "audit") + assert not report["findings"] + assert report["summary"]["pending_review"] == 2 + + +def test_confirmed_timeout_is_eligible_but_unconfirmed_timeout_is_not(job, tmp_path): + path = job / "pass/result.json" + result = json.loads(path.read_text()) + result["exception_info"] = {"exception_type": "AgentTimeoutError"} + path.write_text(json.dumps(result)) + first = audit_job(job, tmp_path / "before") + assert next(row for row in first["trials"] if row["trial"] == "pass")["eligible"] is False + (job / "pass/agent/process-supervision.json").write_text(json.dumps({"status": "cancelled", "processes_stopped": True})) + second = audit_job(job, tmp_path / "after") + row = next(row for row in second["trials"] if row["trial"] == "pass") + assert row["eligible"] is True and row["effective_score"] is None + + +def test_unfinished_job_is_not_frozen(job, tmp_path): + (job / "result.json").write_text('{"finished_at": null}') + with pytest.raises(ValueError, match="completed jobs"): + audit_job(job, tmp_path / "audit") + assert not (tmp_path / "audit/freeze.json").exists() + + +def test_missing_trial_results_cannot_shrink_the_denominator(job, tmp_path): + (job / "result.json").write_text(json.dumps({ + "finished_at": "2026-11-03T00:00:00Z", "stats": {"n_completed_trials": 4}, + })) + with pytest.raises(ValueError, match="inventory is incomplete"): + audit_job(job, tmp_path / "audit") diff --git a/benchmarks/harbor/tests/test_integrity_docker.py b/benchmarks/harbor/tests/test_integrity_docker.py new file mode 100644 index 0000000..52d0702 --- /dev/null +++ b/benchmarks/harbor/tests/test_integrity_docker.py @@ -0,0 +1,67 @@ +"""Optional real Docker probe; no model calls or benchmark solution access. + +Set NANOPY_INTEGRITY_TEST_IMAGE to an existing Linux image containing python3 +and CA certificates. The test creates and removes only its own containers. +""" + +import asyncio +import json +import os +import shlex +import uuid + +import pytest +from harbor.models.task.config import EnvironmentConfig +from harbor.models.trial.paths import TrialPaths + +from harbor_adapter.integrity_environment import IntegrityDockerEnvironment + + +@pytest.mark.skipif(not os.environ.get("NANOPY_INTEGRITY_TEST_IMAGE"), reason="Explicit Docker probe image required") +def test_real_egress_rejects_reference_hosts_and_packet_mark_bypass(tmp_path): + directory = tmp_path / "environment" + directory.mkdir() + paths = TrialPaths(trial_dir=tmp_path / "trial") + paths.mkdir() + environment = IntegrityDockerEnvironment( + environment_dir=directory, environment_name="integrity-probe", + session_id="integrity-probe-" + uuid.uuid4().hex, + trial_paths=paths, + task_env_config=EnvironmentConfig(docker_image=os.environ["NANOPY_INTEGRITY_TEST_IMAGE"]), + integrity_policy={"allowed_hosts": ["api.deepseek.com"]}, + ) + + async def exercise(): + try: + await environment.start(force_build=False) + try: + await environment.integrity_preflight() + except RuntimeError: + print((await environment.exec(command="cat /etc/resolv.conf")).stdout) + print((await environment._run_docker_compose_command([ + "logs", "--no-color", environment._EGRESS_CONTROL_SERVICE_NAME, + ])).stdout) + raise + async with environment.agent_phase(): + for url in ("https://raw.githubusercontent.com/", "https://github.com/", "https://example.com/"): + assert not (await environment._probe(url))["reachable"] + code = '''import socket +s = socket.socket() +try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_MARK, 114514) +except PermissionError: + print("mark denied") +else: + raise SystemExit("SO_MARK bypass remains available") +''' + result = await environment.exec(command=shlex.join(["python3", "-I", "-c", code])) + assert result.return_code == 0 and "mark denied" in result.stdout + result = await environment.exec(command="test ! -e /logs/verifier") + assert result.return_code == 0 + assert (environment.evidence_dir / "egress-log.json").exists() + assert json.loads((environment.evidence_dir / "enforcement.json").read_text())["preflight"] == "passed" + finally: + # Do not remove any pre-existing image, including benchmark images. + await environment.stop(delete=False) + + asyncio.run(exercise()) From 8f705b29ff644d226e3726d6ab78a352a5a46f75 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Wed, 4 Nov 2026 05:26:25 -0800 Subject: [PATCH 2/3] fix(harbor): preserve verifier permissions and native resources --- benchmarks/harbor/README.md | 2 + benchmarks/harbor/README.zh-CN.md | 4 +- .../harbor/src/harbor_adapter/__main__.py | 12 +++ .../harbor_adapter/integrity_environment.py | 3 +- benchmarks/harbor/tests/test_integrity.py | 74 +++++++++++++++++++ .../harbor/tests/test_integrity_docker.py | 22 ++++++ 6 files changed, 115 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 28ffa6b..c85e13b 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -93,6 +93,8 @@ environment overrides, and task MCP servers are unsupported and fail closed. These limits deliberately exclude the historical dependency-cache Compose overlays until they have a reviewed isolation design. The protocol preserves task images, package sources, official tests, resources, and native deadlines. +CPU, memory, storage, GPU, and TPU overrides are rejected; CPU and memory +enforcement must be `auto` (Docker's native limits) or `limit`. Missing task dependencies remain setup or task failures according to the original verifier; network restrictions do not authorize changing the task. diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 60a2257..3658d02 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -74,7 +74,9 @@ UDP DNS 转发仅允许到 Docker 可信 sidecar 报告的精确上游解析器 第一版支持原生网络策略为公开网络的单步 Linux Dockerfile/预构建镜像任务。 自定义 Compose 服务、额外挂载、环境覆盖和题目 MCP 服务暂不支持，遇到时直接 拒绝运行。因此，历史依赖缓存 Compose 覆盖配置需要经过隔离设计审阅后才能接入。 -协议保留题目镜像、软件源、官方测试、资源和原生时限。题目缺失依赖仍按原验证器 +协议保留题目镜像、软件源、官方测试、资源和原生时限。CPU、内存、存储、GPU +和 TPU 的资源覆盖配置会被拒绝；CPU 和内存的执行策略必须为 `auto`（Docker +原生限制）或 `limit`。题目缺失依赖仍按原验证器 表现分类为安装错误或任务失败；网络限制不授权修改题目。 该协议下三个 agent 都使用进程托管，取消及清理期间网络仍受限。控制器采集的 diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py index ba3a6f2..9220a8d 100644 --- a/benchmarks/harbor/src/harbor_adapter/__main__.py +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -93,6 +93,18 @@ def standard_config(raw: dict) -> JobConfig: raise ValueError("restricted-v1 preserves task artifacts and native timeouts") if config.environment.import_path or config.environment.type not in (None, "docker"): raise ValueError("restricted-v1 requires the Docker environment") + if ( + any(getattr(config.environment, field) is not None for field in ( + "override_cpus", "override_memory_mb", "override_storage_mb", + "override_gpus", "override_tpu", + )) + or config.environment.cpu_enforcement_policy not in {"auto", "limit"} + or config.environment.memory_enforcement_policy not in {"auto", "limit"} + ): + raise ValueError( + "restricted-v1 preserves native resource budgets: resource overrides " + "are forbidden and CPU/memory enforcement must be auto or limit" + ) if (config.environment.mounts or config.environment.extra_docker_compose or config.environment.kwargs or config.environment.extra_allowed_hosts or config.environment.env): diff --git a/benchmarks/harbor/src/harbor_adapter/integrity_environment.py b/benchmarks/harbor/src/harbor_adapter/integrity_environment.py index 0a13b09..e57fffc 100644 --- a/benchmarks/harbor/src/harbor_adapter/integrity_environment.py +++ b/benchmarks/harbor/src/harbor_adapter/integrity_environment.py @@ -261,7 +261,8 @@ async def prepare_verification(self): self._integrity_state["verifier_isolation"] = "shared" self._integrity_state["verification_started_at"] = datetime.now(timezone.utc).isoformat() self._save() - await self.exec(command="mkdir -p /logs/verifier; rm -f /logs/verifier/reward.txt /logs/verifier/reward.json", user="root") + await self.ensure_dirs(["/logs/verifier"]) + await self.exec(command="rm -f /logs/verifier/reward.txt /logs/verifier/reward.json", user="root") # Official shared verifiers may install dependencies. This transition # is explicitly recorded and does not claim independent verification. await self.set_network_policy(self._integrity_baseline) diff --git a/benchmarks/harbor/tests/test_integrity.py b/benchmarks/harbor/tests/test_integrity.py index ca5dfad..273fa33 100644 --- a/benchmarks/harbor/tests/test_integrity.py +++ b/benchmarks/harbor/tests/test_integrity.py @@ -3,12 +3,16 @@ import asyncio import json import shlex +import stat +import subprocess from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from harbor.environments.base import ExecResult +from harbor.environments.factory import EnvironmentFactory from harbor.models.task.config import EnvironmentConfig, NetworkPolicy +from harbor.models.trial.config import ResourceMode from harbor.models.trial.paths import TrialPaths from harbor_adapter import NanoPyCodeAgent, PiCodeAgent, DeepSeekHarnessAgent @@ -76,6 +80,45 @@ def test_unreviewed_network_and_environment_escape_routes_are_rejected(overrides standard_config(raw_config() | overrides) +@pytest.mark.parametrize("overrides", [ + {"override_cpus": 64}, + {"override_memory_mb": 65536}, + {"override_storage_mb": 1048576}, + {"override_gpus": 8}, + {"override_gpus": 0}, + {"override_tpu": {"type": "v6e", "topology": "2x4"}}, + *[{field: mode} for field in ("cpu_enforcement_policy", "memory_enforcement_policy") + for mode in ("ignore", "request", "guarantee")], + {"override_cpus": 64, "memory_enforcement_policy": "ignore"}, +]) +def test_restricted_workflow_rejects_changed_resource_budgets(overrides): + with pytest.raises(ValueError, match="native resource"): + standard_config(raw_config() | {"environment": overrides}) + + +@pytest.mark.parametrize("cpu_mode", ["auto", "limit"]) +@pytest.mark.parametrize("memory_mode", ["auto", "limit"]) +def test_accepted_resource_policies_preserve_task_values_and_docker_limits(tmp_path, cpu_mode, memory_mode): + config = standard_config(raw_config() | {"environment": { + "cpu_enforcement_policy": cpu_mode, "memory_enforcement_policy": memory_mode, + "override_cpus": None, "override_memory_mb": None, + "override_storage_mb": None, "override_gpus": None, "override_tpu": None, + }}) + directory = tmp_path / "environment" + directory.mkdir() + (directory / "Dockerfile").write_text("FROM scratch\n") + native = EnvironmentConfig(cpus=1, memory_mb=1024, storage_mb=2048, gpus=0) + original = native.model_dump() + environment = EnvironmentFactory.create_environment_from_config( + config.environment, environment_dir=directory, environment_name="resources", + session_id="resources__env", trial_paths=TrialPaths(trial_dir=tmp_path / "trial"), + task_env_config=native, + ) + assert environment.task_env_config.model_dump() == original + assert environment._resource_limit_value("cpu", auto_mode=ResourceMode.LIMIT) == 1 + assert environment._resource_limit_value("memory", auto_mode=ResourceMode.LIMIT) == 1024 + + @pytest.mark.parametrize("overrides", [ {"skills": ["/tmp/skills"]}, {"extra_allowed_hosts": ["github.com"]}, {"load_trajectory": "/tmp/old.json"}, {"exclude_logs": ["*"]}, @@ -175,6 +218,37 @@ async def run(): asyncio.run(run()) +@pytest.mark.parametrize("existing", [False, True]) +def test_shared_verifier_directory_supports_a_different_user(environment, monkeypatch, tmp_path, existing): + logs = tmp_path / "verifier logs" + if existing: + logs.mkdir(mode=0o755) + (logs / "reward.txt").write_text("stale reward") + (logs / "reward.json").write_text('{"reward": 1}') + (logs / "previous-output.txt").write_text("retain this evidence") + environment._integrity_state["agent_phase"] = "completed" + monkeypatch.setattr(environment, "_network_snapshot", AsyncMock()) + monkeypatch.setattr(environment, "set_network_policy", AsyncMock()) + + async def execute(command, *, user=None, **kwargs): + assert user == "root" + result = subprocess.run( + ["sh", "-c", "umask 022; " + command.replace("/logs/verifier", shlex.quote(str(logs)))], + capture_output=True, text=True, check=True, + ) + return ExecResult(return_code=result.returncode, stdout=result.stdout, stderr=result.stderr) + + monkeypatch.setattr(environment, "exec", execute) + asyncio.run(environment.prepare_verification()) + # The verifier must be able to traverse and write the directory without + # owning it or belonging to its creator's group. + assert stat.S_IMODE(logs.stat().st_mode) & 0o007 == 0o007 + assert not (logs / "reward.txt").exists() + assert not (logs / "reward.json").exists() + if existing: + assert (logs / "previous-output.txt").read_text() == "retain this evidence" + + def test_no_model_work_before_preflight(environment): async def run(): async with environment.agent_phase(): diff --git a/benchmarks/harbor/tests/test_integrity_docker.py b/benchmarks/harbor/tests/test_integrity_docker.py index 52d0702..06050f2 100644 --- a/benchmarks/harbor/tests/test_integrity_docker.py +++ b/benchmarks/harbor/tests/test_integrity_docker.py @@ -60,6 +60,28 @@ async def exercise(): assert result.return_code == 0 assert (environment.evidence_dir / "egress-log.json").exists() assert json.loads((environment.evidence_dir / "enforcement.json").read_text())["preflight"] == "passed" + result = await environment.exec( + command="mkdir -m 755 /logs/verifier && printf stale > /logs/verifier/reward.txt && printf stale > /logs/verifier/reward.json", + user="root", + ) + assert result.return_code == 0 + with environment.with_default_user("65534:65534"): + await environment.prepare_verification() + code = '''import os +from pathlib import Path +assert os.geteuid() == 65534 +logs = Path('/logs/verifier') +assert not (logs / 'reward.txt').exists() +assert not (logs / 'reward.json').exists() +(logs / 'test-stdout.txt').write_text('verifier output') +(logs / 'reward.txt').write_text('1') +(logs / 'reward.json').write_text('{"reward": 1}') +archive = logs / 'attempts' / '1' +archive.mkdir(parents=True) +(archive / 'test-stdout.txt').write_text('archived output') +''' + result = await environment.exec(command=shlex.join(["python3", "-I", "-c", code])) + assert result.return_code == 0, result.stdout or result.stderr finally: # Do not remove any pre-existing image, including benchmark images. await environment.stop(delete=False) From 29c4e0c487214843e95ec3290baf6600fc3412e3 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Wed, 4 Nov 2026 06:40:12 -0800 Subject: [PATCH 3/3] docs: record shared harbor integrity protocol in changelog --- docs/changelogs/0.10.x.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelogs/0.10.x.md b/docs/changelogs/0.10.x.md index 24d7511..790db3a 100644 --- a/docs/changelogs/0.10.x.md +++ b/docs/changelogs/0.10.x.md @@ -5,6 +5,12 @@ All notable changes in the **0.10.x** release series are documented here. ## [Unreleased] ### Added +- Add a shared `restricted-v1` Harbor evaluation protocol for Nano, Pi, and + DeepSeek Harness, with external network allowlists, isolation preflight, + protected evidence, and integrity-adjusted scoring that retains violations in + the denominator. Preserve native resource budgets and writable non-root + verifier logs; require an explicit opt-out for registered historical + open-network runs. - Add retained-workspace Harbor verification, reviewed dependency preparation with independent deadlines, and opt-in collection-error recovery while preserving native verifier limits and prior evidence.