From d6ed70a5d6a1c5b0f9b6c9d20b4a1c2b49153f67 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sat, 24 Oct 2026 07:04:28 -0700 Subject: [PATCH 01/13] feat(benchmarks): support pinned pi agent baselines --- benchmarks/harbor/README.md | 28 +++ benchmarks/harbor/README.zh-CN.md | 21 ++ benchmarks/harbor/scripts/run_comparison.py | 54 +++-- .../harbor/src/harbor_adapter/__init__.py | 3 +- .../harbor/src/harbor_adapter/__main__.py | 79 +++++-- benchmarks/harbor/src/harbor_adapter/pi.py | 193 ++++++++++++++++++ benchmarks/harbor/tests/test_comparison.py | 7 + benchmarks/harbor/tests/test_pi.py | 174 ++++++++++++++++ 8 files changed, 527 insertions(+), 32 deletions(-) create mode 100644 benchmarks/harbor/src/harbor_adapter/pi.py create mode 100644 benchmarks/harbor/tests/test_pi.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 82412b7..c387aab 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -89,6 +89,34 @@ when it runs; a successful preflight is not a guarantee against later outages. Full standard runs enable the timeout retry policy below by default, with zero whole-trial retries. CI checks these contracts without model calls or Docker. +### Pi agent baselines + +The same entry point also supports `harbor_adapter:PiCodeAgent`. Register a local +unmodified Pi binary distribution with `distribution_path` and +`distribution_sha256`, a public `model_config_path`, and optionally `thinking` +in the agent kwargs. The digest covers every distribution file and its mode. +The workflow saves one content-addressed distribution snapshot in the shared +cache, verifies it again before installation, and records its file manifest and +public model configuration with every job. Pi runs its native prompt, tools, +compaction, and retry behavior; no nanoPyCodeAgent wheel is installed or built. + +Currently this adapter supports the official DeepSeek OpenAI-compatible API. +The public models file must reference `${DEEPSEEK_API_KEY}`; supply the actual +key through the process environment. Personal Pi configuration, extensions, +skills, and session history are not copied into task environments. Catalog +refresh is disabled so the recorded model configuration remains reproducible. +Installation-only checks validate the actual binary and provider registration +without a model call and use the same reviewed verifier dependency profiles. +Native sessions and completed JSON events remain in `agent/pi/` and `agent/pi.txt`. +Usage includes compaction when Pi supplies it, and reported costs are estimates. +Missing usage stays diagnosed. Pi does not currently generate an ATIF trajectory. + +The comparison runner supports Pi arms and `--phase all` for installation checks +followed by the model queue. A registered `minimum_free_disk_bytes` stops new +jobs before they start when disk space is insufficient. Existing jobs and +failed attempts are preserved. Agent-specific budget differences must be +declared in the experiment instead of assuming native agent defaults match. + ### Cache original task images If Docker image pulls repeatedly stall, the optional cache utility downloads the diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 10823d0..6ffb09f 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -71,6 +71,27 @@ uv 0.9.5、Python 3.13、pytest 8.4.1 和 pytest-json-ctrf 0.3.5，不执行题 通过不能保证后续没有网络故障。完整标准运行默认启用下述验证超时重试，整题重试 次数为零。CI 会验证这些约定，不调用模型、不启动 Docker。 +### Pi agent baseline + +同一个标准入口也支持 `harbor_adapter:PiCodeAgent`。在 agent kwargs 中用 +`distribution_path` 和 `distribution_sha256` 注册本地未修改的 Pi 二进制发行包， +用 `model_config_path` 指定公开模型配置，并可选设置 `thinking`。 +摘要覆盖发行包内每个文件及其权限。工作流会在共享缓存中保存一份按内容寻址的 +发行包快照，安装前再次校验，并为每个任务记录文件清单与公开模型配置。 +Pi 使用原生提示词、工具、上下文压缩和重试行为；不会构建或安装 nanoPyCodeAgent wheel。 + +目前该 adapter 支持 DeepSeek 官方 OpenAI 兼容 API。公开模型文件必须引用 +`${DEEPSEEK_API_KEY}`，真实密钥通过进程环境提供。个人 Pi 配置、扩展、skills +和历史会话不会复制到任务环境。禁用模型目录自动更新，以保证已记录的配置可复现。 +仅安装检查会验证实际二进制和模型注册，不调用模型，并沿用相同的已审阅 verifier +依赖 profile。原生会话及完整事件保存在 `agent/pi/` 和 `agent/pi.txt`。 +用量统计在 Pi 提供数据时包含上下文压缩；费用为估算值，缺失用量会单独标注。 +Pi 目前不生成 ATIF trajectory。 + +比较运行器支持 Pi arm，以及先安装检查、再执行模型队列的 `--phase all`。 +注册的 `minimum_free_disk_bytes` 会在空间不足时阻止启动新任务。 +已有任务和失败尝试均保留。各 agent 的预算差异必须在实验中说明，不能假设原生默认值相同。 + ### 缓存原始任务镜像 如果 Docker 拉取镜像反复停滞，可用缓存工具从同一 Docker Hub 公共仓库分段并行下载： diff --git a/benchmarks/harbor/scripts/run_comparison.py b/benchmarks/harbor/scripts/run_comparison.py index 87ff60e..b4e1506 100644 --- a/benchmarks/harbor/scripts/run_comparison.py +++ b/benchmarks/harbor/scripts/run_comparison.py @@ -11,6 +11,7 @@ import hashlib import json from pathlib import Path +import shutil import subprocess @@ -57,11 +58,16 @@ def validate_plan(root, plan_path, plan): ["git", "diff", "--exit-code", "HEAD", "--", "src", "benchmarks/harbor/src"], cwd=source, check=True, stdout=subprocess.DEVNULL, ) - expected = plan_path.parent / arm["prompt_file"] - if digest(expected) != arm["prompt_sha256"]: - raise ValueError(f"Prompt file changed: {expected}") - if prompt_from_source(source / "src/nanopycodeagent/agent.py") != expected.read_text(): - raise ValueError(f"Source prompt does not match the plan: {source}") + if arm.get("agent") == "pi": + from harbor_adapter.pi import distribution_digest, distribution_files + if distribution_digest(distribution_files(Path(arm["distribution_path"]))) != arm["distribution_sha256"]: + raise ValueError("Registered Pi distribution changed") + else: + expected = plan_path.parent / arm["prompt_file"] + if digest(expected) != arm["prompt_sha256"]: + raise ValueError(f"Prompt file changed: {expected}") + if prompt_from_source(source / "src/nanopycodeagent/agent.py") != expected.read_text(): + raise ValueError(f"Source prompt does not match the plan: {source}") for task in plan["tasks"]: cache = Path.home() / ".cache/harbor/tasks/packages" / task["name"] / task["ref"].split(":")[1] for name, expected in task["task_file_sha256"].items(): @@ -84,6 +90,9 @@ def run_job(root, plan, job, record, install_only): if previous["status"] == "finished": return previous raise RuntimeError(f"Previously started job requires inspection; refusing to retry: {status_path}") + minimum = plan.get("minimum_free_disk_bytes", 0) + if shutil.disk_usage(root).free < minimum: + raise RuntimeError(f"Insufficient free disk space: this plan requires {minimum} bytes") source = root / plan["arms"][job["arm"]]["source_root"] command = [ "uv", "run", "--project", "benchmarks/harbor", "python", "-m", "harbor_adapter", @@ -166,23 +175,37 @@ def checked(path, expected_hash): if config.get("install_only") is not True or manifest["commit"] != arm["source_ref"]: raise ValueError("Accepted installation used a different mode or source revision") expected_sources = {} - for prefix, package in (("src", "nanopycodeagent"), ("benchmarks/harbor/src", "harbor_adapter")): + packages = [("benchmarks/harbor/src", "harbor_adapter")] + if arm.get("agent") != "pi": + packages.append(("src", "nanopycodeagent")) + for prefix, package in packages: for path in (source / prefix / package).rglob("*.py"): expected_sources[str(path.relative_to(source / prefix))] = digest(path) if not expected_sources or manifest["source_sha256"] != expected_sources: raise ValueError("Accepted installation source snapshot differs from the frozen source") - wheel = Path(result["config"]["agent"]["kwargs"]["wheel_path"]).resolve() - if not wheel.is_relative_to((inputs / "dist").resolve()): - raise ValueError("Accepted installation wheel must belong to its recorded inputs") - if not (digest(wheel) == manifest["wheel_sha256"] == bootstrap["wheel_sha256"]): - raise ValueError("Accepted installation wheel digest differs from its provenance") + if arm.get("agent") == "pi": + pin = arm["distribution_sha256"] + pi_input, = manifest["pi_inputs"] + if not (pi_input["distribution_sha256"] == bootstrap["distribution_sha256"] == pin): + raise ValueError("Accepted Pi distribution differs from its provenance") + from harbor_adapter.pi import distribution_digest + if distribution_digest(pi_input["files"]) != pin: + raise ValueError("Accepted Pi file manifest differs from its provenance") + if digest(inputs / "pi-models.json") != pi_input["model_config_sha256"]: + raise ValueError("Accepted Pi model config differs from its provenance") + else: + wheel = Path(result["config"]["agent"]["kwargs"]["wheel_path"]).resolve() + if not wheel.is_relative_to((inputs / "dist").resolve()): + raise ValueError("Accepted installation wheel must belong to its recorded inputs") + if not (digest(wheel) == manifest["wheel_sha256"] == bootstrap["wheel_sha256"]): + raise ValueError("Accepted installation wheel digest differs from its provenance") return True def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--plan", type=Path, required=True) - parser.add_argument("--phase", choices=["preflight", "model"], required=True) + parser.add_argument("--phase", choices=["preflight", "model", "all"], required=True) args = parser.parse_args() root, plan_path = Path.cwd(), args.plan.resolve() plan = json.loads(plan_path.read_text()) @@ -205,9 +228,12 @@ def main(): len(json.loads((root / job["config"]).read_text())["tasks"]) for job in plan["preflight"] ] - if args.phase == "preflight": + if args.phase in {"preflight", "all"}: states = [run_job(root, plan, j, record, True) for j in plan["preflight"]] - return 0 if accepted or preflight_passed(states, expected_counts) else 1 + if not accepted and not preflight_passed(states, expected_counts): + return 1 + if args.phase == "preflight": + return 0 if not accepted: preflight = [json.loads((record / (j["job_name"] + ".json")).read_text()) for j in plan["preflight"]] if not preflight_passed(preflight, expected_counts): diff --git a/benchmarks/harbor/src/harbor_adapter/__init__.py b/benchmarks/harbor/src/harbor_adapter/__init__.py index 5e8c83c..b27118f 100644 --- a/benchmarks/harbor/src/harbor_adapter/__init__.py +++ b/benchmarks/harbor/src/harbor_adapter/__init__.py @@ -1,6 +1,7 @@ """Public import for the repository-local nanoPyCodeAgent Harbor adapter.""" from .adapter import NanoPyCodeAgent +from .pi import PiCodeAgent from .verifier import RetryingVerifier -__all__ = ["NanoPyCodeAgent", "RetryingVerifier"] +__all__ = ["NanoPyCodeAgent", "PiCodeAgent", "RetryingVerifier"] diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py index d41cf39..8f048e3 100644 --- a/benchmarks/harbor/src/harbor_adapter/__main__.py +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -14,6 +14,10 @@ from harbor.models.job.config import JobConfig from .bootstrap import AptCache, default_cache_dir +from .pi import distribution_digest, distribution_files, read_model_config + +_NANO = "harbor_adapter:NanoPyCodeAgent" +_PI = "harbor_adapter:PiCodeAgent" def standard_config(raw: dict) -> JobConfig: @@ -50,22 +54,54 @@ def standard_config(raw: dict) -> JobConfig: raise ValueError("Select at least one agent and one trial attempt") for agent in config.agents: identifiers = [value for value in (agent.import_path, agent.name) if value] - if not identifiers or any( - value != "harbor_adapter:NanoPyCodeAgent" for value in identifiers - ): - raise ValueError("The standard workflow requires harbor_adapter:NanoPyCodeAgent") - if agent.env or set(agent.kwargs) - { - "max_turns", "max_tokens", "time_budget_seconds", - }: + if not identifiers or len(set(identifiers)) != 1 or identifiers[0] not in {_NANO, _PI}: + raise ValueError("The standard workflow requires a repository benchmark adapter") + allowed = ({"max_turns", "max_tokens", "time_budget_seconds"} + if identifiers[0] == _NANO else { + "distribution_path", "distribution_sha256", "model_config_path", "thinking", + }) + if agent.env or set(agent.kwargs) - allowed: raise ValueError( "Pass model credentials through the environment; " - "agent kwargs may only set budgets" + "agent kwargs must use the adapter's supported public settings" ) + if identifiers[0] == _PI: + required = {"distribution_path", "distribution_sha256", "model_config_path"} + if not required <= agent.kwargs.keys(): + raise ValueError("Pi requires a hash-pinned distribution and public model config") + read_model_config(Path(agent.kwargs["model_config_path"])) if not agent.model_name: raise ValueError("Each agent requires an explicit model_name") return config +def snapshot_pi(agent, cache: Path, record: Path) -> dict: + """Share one content-addressed binary snapshot across the full task pool.""" + original = Path(agent.kwargs["distribution_path"]).expanduser().resolve() + files = distribution_files(original) + digest = distribution_digest(files) + if digest != agent.kwargs["distribution_sha256"]: + raise ValueError("Pi distribution differs from its registered SHA256") + destination = cache / "pi-distributions" / digest + if not destination.exists(): + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(digest + ".tmp-" + record.name) + shutil.copytree(original, temporary) + if distribution_files(temporary) != files: + raise ValueError("Pi distribution changed while snapshotting") + temporary.rename(destination) + elif distribution_files(destination) != files: + raise ValueError("Cached Pi distribution is corrupt") + public_models = read_model_config(Path(agent.kwargs["model_config_path"])) + models = record / "pi-models.json" + models.write_text(json.dumps(public_models, indent=2) + "\n") + agent.kwargs.update(distribution_path=str(destination), model_config_path=str(models)) + return { + "distribution_sha256": digest, "files": files, + "model_config_sha256": hashlib.sha256(models.read_bytes()).hexdigest(), + } + + def summarize(job_dir: Path, *, install_only: bool = False) -> dict: trials = [] for path in sorted(job_dir.glob("*/result.json")): @@ -111,16 +147,23 @@ def run(args) -> int: record.chmod(0o700) source = record / "source" shutil.copytree(root / "benchmarks/harbor/src/harbor_adapter", source / "harbor_adapter", ignore=shutil.ignore_patterns("__pycache__")) - shutil.copytree(root / "src/nanopycodeagent", source / "nanopycodeagent", ignore=shutil.ignore_patterns("__pycache__")) - subprocess.run(["uv", "build", "--wheel", "--out-dir", str(record / "dist")], check=True) - wheel, = (record / "dist").glob("*.whl") - with ZipFile(wheel) as archive: - for path in (source / "nanopycodeagent").rglob("*.py"): - if archive.read(path.relative_to(source).as_posix()) != path.read_bytes(): - raise ValueError("Source changed during preparation; prepare a new job") cache = args.cache_dir.expanduser().resolve() + wheel = None + if any((agent.import_path or agent.name) == _NANO for agent in config.agents): + shutil.copytree(root / "src/nanopycodeagent", source / "nanopycodeagent", ignore=shutil.ignore_patterns("__pycache__")) + subprocess.run(["uv", "build", "--wheel", "--out-dir", str(record / "dist")], check=True) + wheel, = (record / "dist").glob("*.whl") + with ZipFile(wheel) as archive: + for path in (source / "nanopycodeagent").rglob("*.py"): + if archive.read(path.relative_to(source).as_posix()) != path.read_bytes(): + raise ValueError("Source changed during preparation; prepare a new job") + pi_inputs = [] for agent in config.agents: - agent.kwargs.update(wheel_path=str(wheel), bootstrap_cache=str(cache)) + if (agent.import_path or agent.name) == _PI: + pi_inputs.append(snapshot_pi(agent, cache, record)) + else: + agent.kwargs["wheel_path"] = str(wheel) + agent.kwargs["bootstrap_cache"] = str(cache) config_path = record / "config.json" config_path.write_text(config.model_dump_json(indent=2) + "\n") manifest = { @@ -129,9 +172,11 @@ def run(args) -> int: "branch": subprocess.check_output(["git", "branch", "--show-current"], text=True).strip(), "source_kind": "working tree including uncommitted files", "source_sha256": {str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source.rglob("*.py"))}, - "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest() if wheel else None, "cache_dir": str(cache), } + if pi_inputs: + manifest["pi_inputs"] = pi_inputs (record / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") (record / "working-tree.patch").write_bytes(subprocess.check_output(["git", "diff", "HEAD"])) print(f"Prepared {record}", flush=True) diff --git a/benchmarks/harbor/src/harbor_adapter/pi.py b/benchmarks/harbor/src/harbor_adapter/pi.py new file mode 100644 index 0000000..0621203 --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/pi.py @@ -0,0 +1,193 @@ +"""Run an unmodified, hash-pinned Pi distribution with shared Harbor bootstrap.""" + +import hashlib +import json +from pathlib import Path +import shlex + +from harbor.agents.installed.base import CliFlag, with_prompt_template +from harbor.agents.model_connection import ModelConnectionSpec + +from .adapter import NanoPyCodeAgent + + +def distribution_files(root: Path) -> dict: + """Include executable permissions and reject links outside the snapshot.""" + files = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("Pi distributions must not contain symbolic links") + if path.is_file(): + with path.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + files[path.relative_to(root).as_posix()] = { + "sha256": digest, "mode": path.stat().st_mode & 0o777, + } + if "pi" not in files or "package.json" not in files: + raise ValueError("Pi distribution requires pi and package.json") + return files + + +def distribution_digest(files: dict) -> str: + return hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() + + +def read_model_config(path: Path) -> dict: + """Accept a public DeepSeek connection; credentials stay in the environment.""" + config = json.loads(path.read_text()) + if set(config) != {"providers"} or set(config["providers"]) != {"deepseek"}: + raise ValueError("Pi benchmark models must configure only DeepSeek") + provider = config["providers"]["deepseek"] + if (set(provider) - {"api", "apiKey", "baseUrl", "models"} + or provider.get("apiKey") != "${DEEPSEEK_API_KEY}" + or provider.get("api") != "openai-completions" + or provider.get("baseUrl") != "https://api.deepseek.com"): + raise ValueError("Pi models must use the official API and an environment key reference") + if not provider.get("models"): + raise ValueError("Pi models must explicitly register at least one model") + return config + + +class PiCodeAgent(NanoPyCodeAgent): + """Share verified dependencies/preflight, replacing Nano's install and run.""" + + SUPPORTS_ATIF = False + MODEL_CONNECTION = ModelConnectionSpec(api_key_envs=("DEEPSEEK_API_KEY",)) + CLI_FLAGS = [CliFlag("thinking", cli="--thinking", type="enum", choices=[ + "off", "minimal", "low", "medium", "high", "xhigh", "max", + ])] + + def __init__(self, *args, distribution_path: str, distribution_sha256: str, + model_config_path: str, **kwargs): + self._distribution = Path(distribution_path).resolve() + self._distribution_sha256 = distribution_sha256 + self._model_config = read_model_config(Path(model_config_path)) + package = json.loads((self._distribution / "package.json").read_text()) + self._pi_version = package["version"] + super().__init__(*args, version=self._pi_version, **kwargs) + provider, separator, model = (self.model_name or "").partition("/") + if provider != "deepseek" or not separator or model not in { + item["id"] for item in self._model_config["providers"][provider]["models"] + }: + raise ValueError("Pi model_name must identify a registered deepseek/model") + + @staticmethod + def name() -> str: + return "pi" + + def get_version_command(self) -> str: + return "/opt/harbor-pi/pi --version" + + def parse_version(self, stdout: str) -> str: + return stdout.strip().splitlines()[-1] + + async def _install_agent(self, environment) -> None: + actual = distribution_digest(distribution_files(self._distribution)) + if actual != self._distribution_sha256: + raise ValueError("Pi distribution changed after registration") + await environment.upload_dir(self._distribution, "/opt/harbor-pi") + await self.exec_as_root(environment, command="chmod 755 /opt/harbor-pi/pi") + result = await self.exec_as_agent(environment, command=self.get_version_command()) + if self.parse_version(result.stdout) != self._pi_version: + raise ValueError("Installed Pi version differs from its distribution") + self._bootstrap_record.update( + pi_version=self._pi_version, distribution_sha256=actual, + ) + await self.exec_as_agent(environment, command="mkdir -p /tmp/harbor-pi-agent") + await self._upload_config_text( + environment, content=json.dumps(self._model_config), + remote_path="/tmp/harbor-pi-agent/models.json", filename="models.json", + ) + # Validate the real binary and provider configuration without a model call. + await self.exec_as_agent( + environment, + command="/opt/harbor-pi/pi --offline --list-models deepseek-flash", + env={"PI_CODING_AGENT_DIR": "/tmp/harbor-pi-agent", "DEEPSEEK_API_KEY": "preflight-only"}, + ) + + @with_prompt_template + async def run(self, instruction, environment, context) -> None: + key = self.model_connection.api_key + if not key: + raise ValueError("DEEPSEEK_API_KEY is required for Pi model work") + _, model = self.model_name.split("/", 1) + # Keep completed events and native sessions; streaming deltas duplicate + # their contents and can consume substantial disk space. + event_filter = ( + "import json,sys\n" + "for line in sys.stdin:\n" + " try: skip=json.loads(line).get('type') == 'message_update'\n" + " except (ValueError,AttributeError): skip=False\n" + " if not skip: print(line,end='',flush=True)\n" + ) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; /opt/harbor-pi/pi --print --mode json --offline " + "--session-dir /logs/agent/pi/sessions " + f"--provider deepseek --model {shlex.quote(model)} " + f"{self.build_cli_flags()} -- {shlex.quote(instruction)} " + "/logs/agent/pi-stderr.txt | " + f"python3 -u -c {shlex.quote(event_filter)} | tee /logs/agent/pi.txt" + ), + env={"PI_CODING_AGENT_DIR": "/tmp/harbor-pi-agent", "DEEPSEEK_API_KEY": key}, + ) + # Pi may finish its event stream after an API error with exit code zero. + # Do not misclassify that as an ordinary scored task failure. + check = ( + "import json\n" + "last={}; assistant={}\n" + "for line in open('/logs/agent/pi.txt'):\n" + " try: event=json.loads(line)\n" + " except ValueError: continue\n" + " last=event\n" + " if event.get('type')=='message_end' and event.get('message',{}).get('role')=='assistant': assistant=event['message']\n" + "print(json.dumps({'settled': last.get('type')=='agent_settled' and not last.get('aborted'), 'stop_reason': assistant.get('stopReason')}))\n" + ) + output = await self.exec_as_agent(environment, command=f"python3 -c {shlex.quote(check)}") + status = json.loads(output.stdout) + if not status["settled"] or status["stop_reason"] != "stop": + raise RuntimeError(f"Pi did not finish successfully: {status}") + + def populate_context_post_run(self, context) -> None: + context.metadata = context.metadata or {} + context.metadata["bootstrap"] = self._bootstrap_record + output = self.logs_dir / "pi.txt" + if not output.exists(): + return + usages, missing = [], 0 + errors = [] + for line in output.read_text().split("\n"): + try: + event = json.loads(line) + except ValueError: + continue + if not isinstance(event, dict): + continue + message = event.get("message", {}) + if event.get("type") == "message_end" and message.get("role") == "assistant": + if message.get("stopReason") in {"error", "aborted"}: + errors.append(message.get("stopReason")) + if "usage" in message: + usages.append(message["usage"]) + else: + missing += 1 + elif event.get("type") == "compaction_end" and event.get("result"): + usage = event["result"].get("usage") + if usage: + usages.append(usage) + else: + missing += 1 + context.metadata["pi"] = { + "version": self._pi_version, "usage_records": len(usages), + "missing_usage_records": missing, "assistant_errors": errors, + "cost_is_estimated": True, + "usage_is_partial": missing > 0, + } + if usages: + context.n_input_tokens = sum(u.get("input", 0) + u.get("cacheRead", 0) + u.get("cacheWrite", 0) for u in usages) + context.n_output_tokens = sum(u.get("output", 0) for u in usages) + context.n_cache_tokens = sum(u.get("cacheRead", 0) for u in usages) + costs = [u["cost"]["total"] for u in usages if "total" in u.get("cost", {})] + context.cost_usd = sum(costs) if len(costs) == len(usages) else None + context.metadata["pi"]["cost_is_partial"] = missing > 0 or len(costs) != len(usages) diff --git a/benchmarks/harbor/tests/test_comparison.py b/benchmarks/harbor/tests/test_comparison.py index fc2f5d6..64aa099 100644 --- a/benchmarks/harbor/tests/test_comparison.py +++ b/benchmarks/harbor/tests/test_comparison.py @@ -57,6 +57,13 @@ def test_interrupted_job_cannot_silently_restart(tmp_path): comparison.run_job(tmp_path, {}, job, tmp_path, False) +def test_low_disk_space_stops_before_recording_or_starting_a_trial(tmp_path, monkeypatch): + monkeypatch.setattr(comparison.shutil, "disk_usage", lambda path: SimpleNamespace(free=1)) + with pytest.raises(RuntimeError, match="Insufficient free disk"): + comparison.run_job(tmp_path, {"minimum_free_disk_bytes": 2}, {"job_name": "new"}, tmp_path, False) + assert not (tmp_path / "new.json").exists() + + def test_registered_transport_files_cannot_change(tmp_path): path = tmp_path / "apt-cache.conf" path.write_text("original configuration") diff --git a/benchmarks/harbor/tests/test_pi.py b/benchmarks/harbor/tests/test_pi.py new file mode 100644 index 0000000..342e32e --- /dev/null +++ b/benchmarks/harbor/tests/test_pi.py @@ -0,0 +1,174 @@ +"""Pi provenance, native execution, and usage accounting contracts.""" + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest +from harbor.models.agent.context import AgentContext + +from harbor_adapter import PiCodeAgent +from harbor_adapter.__main__ import snapshot_pi, standard_config +from harbor_adapter.pi import distribution_digest, distribution_files, read_model_config + + +@pytest.fixture +def inputs(tmp_path): + distribution = tmp_path / "distribution" + distribution.mkdir() + (distribution / "pi").write_text("binary") + (distribution / "pi").chmod(0o755) + (distribution / "package.json").write_text(json.dumps({"version": "1.4.0"})) + models = tmp_path / "models.json" + models.write_text(json.dumps({"providers": {"deepseek": { + "api": "openai-completions", "baseUrl": "https://api.deepseek.com", + "apiKey": "${DEEPSEEK_API_KEY}", "models": [{"id": "deepseek-flash"}], + }}})) + return { + "distribution_path": str(distribution), + "distribution_sha256": distribution_digest(distribution_files(distribution)), + "model_config_path": str(models), "thinking": "high", + } + + +def adapter(tmp_path, inputs): + return PiCodeAgent(logs_dir=tmp_path, model_name="deepseek/deepseek-flash", **inputs) + + +class Environment: + default_user = None + + def __init__(self): + self.calls = [] + self.uploads = [] + self.completion = {"settled": True, "stop_reason": "stop"} + + async def exec(self, **kwargs): + self.calls.append(kwargs) + command = kwargs["command"] + stdout = "" + if command.endswith("--version"): + stdout = "1.4.0\n" + elif "; python3 -c " in command: + stdout = json.dumps(self.completion) + return SimpleNamespace(return_code=0, stdout=stdout, stderr="") + + async def upload_dir(self, source, target): + self.uploads.append((source, target)) + + async def upload_file(self, source, target): + self.uploads.append((Path(source).read_text(), target)) + + +def test_public_pi_config_is_accepted_without_allowing_credentials(tmp_path, inputs): + raw = {"tasks": [{"name": "org/task"}], "agents": [{ + "name": "harbor_adapter:PiCodeAgent", "model_name": "deepseek/deepseek-flash", + "kwargs": inputs, + }]} + assert standard_config(raw).agents[0].kwargs["thinking"] == "high" + raw["agents"][0]["kwargs"] = inputs | {"api_key": "private"} + with pytest.raises(ValueError, match="credentials"): + standard_config(raw) + models = Path(inputs["model_config_path"]) + models.write_text(models.read_text().replace("${DEEPSEEK_API_KEY}", "private")) + with pytest.raises(ValueError, match="environment key"): + read_model_config(models) + + +def test_snapshot_reuses_identical_binary_but_detects_corruption(tmp_path, inputs): + cache = tmp_path / "cache" + agent = SimpleNamespace(kwargs=inputs.copy()) + first, second = tmp_path / "first", tmp_path / "second" + first.mkdir() + second.mkdir() + manifest = snapshot_pi(agent, cache, first) + assert snapshot_pi(SimpleNamespace(kwargs=inputs.copy()), cache, second) == manifest + destination = Path(agent.kwargs["distribution_path"]) + assert destination != Path(inputs["distribution_path"]) + (destination / "pi").write_text("corrupted") + with pytest.raises(ValueError, match="corrupt"): + snapshot_pi(SimpleNamespace(kwargs=inputs.copy()), cache, second) + + +def test_changed_executable_permissions_and_symlinks_are_rejected(tmp_path, inputs): + root = Path(inputs["distribution_path"]) + (root / "pi").chmod(0o644) + assert distribution_digest(distribution_files(root)) != inputs["distribution_sha256"] + (root / "link").symlink_to(tmp_path / "outside") + with pytest.raises(ValueError, match="symbolic links"): + distribution_files(root) + + +def test_install_verifies_binary_and_provider_without_calling_model(tmp_path, inputs): + agent = adapter(tmp_path, inputs) + env = Environment() + asyncio.run(agent.install(env)) + record = json.loads((tmp_path / "bootstrap.json").read_text()) + assert record["status"] == "ready" + assert record["distribution_sha256"] == inputs["distribution_sha256"] + assert record["verifier_preflight"]["status"] == "not_configured" + assert env.calls[-1]["command"].endswith("--list-models deepseek-flash") + assert env.calls[-1]["env"]["DEEPSEEK_API_KEY"] == "preflight-only" + assert not any("--print" in call["command"] for call in env.calls) + + +def test_install_refuses_modified_binary(tmp_path, inputs): + agent = adapter(tmp_path, inputs) + Path(inputs["distribution_path"], "pi").write_text("changed") + with pytest.raises(ValueError, match="changed after registration"): + asyncio.run(agent.install(Environment())) + assert json.loads((tmp_path / "bootstrap.json").read_text())["status"] == "failed" + + +def test_native_run_preserves_failure_status_and_uses_only_environment_key(tmp_path, inputs, monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-secret") + agent = adapter(tmp_path, inputs) + env = Environment() + asyncio.run(agent.run("Handle '$HOME' and `literal`", env, AgentContext())) + call = env.calls[0] + assert "set -euo pipefail" in call["command"] + assert "--thinking high" in call["command"] + assert "--session-dir /logs/agent/pi/sessions" in call["command"] + assert "test-secret" not in call["command"] + assert call["env"]["DEEPSEEK_API_KEY"] == "test-secret" + assert "--system-prompt" not in call["command"] + + +def test_usage_includes_compaction_and_cache_without_pricing_missing_calls_as_zero(tmp_path, inputs): + agent = adapter(tmp_path, inputs) + usage = {"input": 10, "output": 5, "cacheRead": 20, "cacheWrite": 3, "cost": {"total": 0.2}} + (tmp_path / "pi.txt").write_text("\n".join(json.dumps(event) for event in [ + {"type": "message_end", "message": {"role": "assistant", "usage": usage}}, + {"type": "compaction_end", "result": {"usage": usage}}, + {"type": "message_end", "message": {"role": "assistant"}}, + ])) + context = AgentContext() + agent.populate_context_post_run(context) + assert context.n_input_tokens == 66 + assert context.n_output_tokens == 10 + assert context.n_cache_tokens == 40 + assert context.cost_usd == 0.4 + assert context.metadata["pi"]["missing_usage_records"] == 1 + assert context.metadata["pi"]["cost_is_estimated"] is True + assert context.metadata["pi"]["cost_is_partial"] is True + + +@pytest.mark.parametrize("completion", [ + {"settled": True, "stop_reason": "error"}, + {"settled": False, "stop_reason": "stop"}, + {"settled": True, "stop_reason": None}, +]) +def test_zero_exit_with_failed_or_missing_response_is_an_execution_error(tmp_path, inputs, monkeypatch, completion): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-secret") + environment = Environment() + environment.completion = completion + with pytest.raises(RuntimeError, match="did not finish successfully"): + asyncio.run(adapter(tmp_path, inputs).run("test", environment, AgentContext())) + + +def test_missing_output_keeps_usage_unknown(tmp_path, inputs): + context = AgentContext() + adapter(tmp_path, inputs).populate_context_post_run(context) + assert context.n_input_tokens is None + assert context.cost_usd is None From b90c03a8123c6c276592a31b712083766dddc1fc Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sat, 24 Oct 2026 07:10:25 -0700 Subject: [PATCH 02/13] chore(benchmarks): register pi deepseek flash dev34 baseline --- .../README.md | 121 ++ .../adaptive-rejection-sampler.json | 32 + .../apt-cache.conf | 8 + .../build-cython-ext.json | 32 + .../build-pov-ray.json | 32 + .../caffe-cifar-10.json | 32 + .../cancel-async-tasks.json | 32 + .../circuit-fibsqrt.json | 32 + .../cobol-modernization.json | 32 + .../constraints-scheduling.json | 32 + .../custom-memory-heap-crash.json | 32 + .../db-wal-recovery.json | 32 + .../dependency-cache.compose.json | 100 ++ .../dependency-release-manifest.json | 58 + .../dna-assembly.json | 32 + .../experiment.json | 1409 +++++++++++++++++ .../fix-code-vulnerability.json | 32 + .../git-multibranch.json | 32 + .../image-digests.json | 312 ++++ .../install-only.json | 165 ++ .../kv-store-grpc.json | 32 + .../llm-inference-batching-scheduler.json | 32 + .../log-summary-date-ranges.json | 32 + .../merge-diff-arc-agi-task.json | 32 + .../model-extraction-relu-logits.json | 32 + .../mteb-leaderboard.json | 32 + .../multi-source-data-merger.json | 32 + .../nginx-request-logging.json | 32 + .../openssl-selfsigned-cert.json | 32 + .../path-tracing.json | 32 + .../pi-distribution.json | 852 ++++++++++ .../pi-models.json | 42 + .../prior-image-digests.json | 311 ++++ .../pypi-server.json | 32 + .../pytorch-model-cli.json | 32 + .../pytorch-model-recovery.json | 32 + .../qemu-alpine-ssh.json | 32 + .../query-optimize.json | 32 + .../regex-chess.json | 32 + .../regex-log.json | 32 + .../schemelike-metacircular-eval.json | 32 + .../selection.json | 150 ++ .../torch-pipeline-parallelism.json | 32 + .../torch-tensor-parallelism.json | 32 + .../write-compressor.json | 32 + 45 files changed, 4616 insertions(+) create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/adaptive-rejection-sampler.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/apt-cache.conf create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-cython-ext.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-pov-ray.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/caffe-cifar-10.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cancel-async-tasks.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/circuit-fibsqrt.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cobol-modernization.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/constraints-scheduling.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/custom-memory-heap-crash.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/db-wal-recovery.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-release-manifest.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dna-assembly.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/fix-code-vulnerability.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/git-multibranch.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/image-digests.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/install-only.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/kv-store-grpc.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/llm-inference-batching-scheduler.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/log-summary-date-ranges.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/merge-diff-arc-agi-task.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/model-extraction-relu-logits.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/mteb-leaderboard.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/multi-source-data-merger.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/nginx-request-logging.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/openssl-selfsigned-cert.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/path-tracing.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-distribution.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/prior-image-digests.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pypi-server.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-cli.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-recovery.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/qemu-alpine-ssh.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/query-optimize.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-chess.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-log.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/schemelike-metacircular-eval.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/selection.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-pipeline-parallelism.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-tensor-parallelism.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/write-compressor.json diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md new file mode 100644 index 0000000..7944263 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md @@ -0,0 +1,121 @@ +# Pi + DeepSeek Flash dev34 agent baseline + +**Status: prepared, not started.** The system disk had approximately 824 MiB +free at the last preparation check. No container installation preflight, model +trial, or official verifier has run for this experiment. The runner requires +at least 15 GiB free before starting new jobs. Existing recovery containers are +preserved pending a storage decision. + +This experiment establishes the first external-agent baseline on the existing +34-task development pool. The [selection](selection.json) is byte-identical to +the October 12 selection. All task revisions and registered task-file hashes +match, and all 34 local Docker image identities and repository digests were +checked against the October 19 registration. The other 55 tasks remain unopened. + +## Frozen agent and model + +- Agent: upstream Pi **1.4.0**, using the existing local binary distribution. +- Runtime adapter checkout: `jobs/runtime-source`, detached at + `d6ed70a5d6a1c5b0f9b6c9d20b4a1c2b49153f67`. +- [Distribution manifest](pi-distribution.json): all files and permissions, + pinned by `60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9`. + This pins the installed local bytes; an upstream release archive checksum has + not been independently checked. No Pi source or binary is modified. +- Model: `deepseek/deepseek-flash`, through the official OpenAI-compatible API at + `https://api.deepseek.com`. +- Reasoning: **high**, provisionally selected pending maintainer preference, + before any model work. Temperature is unset. +- [Public model configuration](pi-models.json): the local DeepSeek provider's + capabilities and compatibility settings, with credentials replaced by the + `${DEEPSEEK_API_KEY}` environment reference. No other provider is copied. + +[DeepSeek's official documentation](https://api-docs.deepseek.com/quick_start/agent_integrations/) +was checked on October 24 and maps `deepseek-flash` to +DeepSeek-V4.1-Flash-1017. This alias is mutable. The registered model metadata +declares a 1,000,000-token context and 384,000-token output ceiling. Actual +generation behavior remains Pi's native behavior. The configuration retains +peak-rate token estimates; [official pricing](https://api-docs.deepseek.com/quick_start/pricing/) +also has off-peak discounts, so these estimates are not reconciled bills. + +Pi retains its own system prompt, default tools, context compaction, and internal +request retry behavior. Personal extensions, skills, session history, settings, +and other providers are not copied. Model catalog refresh is disabled with +`--offline`; model requests still reach the official API. The adapter keeps +native sessions and completed JSON events rather than generating an ATIF trace. + +## Task, budget, and infrastructure policy + +The [experiment plan](experiment.json) registers all 34 fresh model trials, +their fixed order, configurations, source revision, distribution, infrastructure +hashes, and installation checks. Maximum concurrency is two, with consecutive +pair barriers and no whole-trial retries. Preserve every failed setup or launch +attempt; do not replace an existing model output after seeing its outcome. + +Each task keeps its original image, resource limits, native agent timeout, test +scripts, package sources, and native verifier time limit. Setup separately +allows 1,800 seconds. Only verifier timeouts may retry, at most three total +attempts against the same output. Dependency-error recovery is disabled. + +This run uses stock Pi loop and generation behavior. It does not inject Nano's +100-reply cap or its native-timeout-minus-180-seconds work budget and finalization +reminders. The historical Nano configuration requested 65,536 output tokens; +the Pi model metadata retains the local 384,000-token ceiling. Report these +differences when comparing scores: this is an agent baseline, not an experiment +that isolates harness behavior under identical internal budgets. + +Fresh installation-only checks must finish for all 34 tasks before any model +trial. QEMU has the reviewed `qemu-alpine-ssh-v1` verifier profile. The other +33 tasks must explicitly report `not_configured`; successful Pi installation +does not certify their verifier dependencies. At preparation time **all 34 +native installation checks are pending**, including QEMU. + +The [dependency transport](dependency-cache.compose.json) reuses the reviewed +loopback APT-index and original uv/Python release-cache sidecars. The +[release manifest](dependency-release-manifest.json) is unchanged. Cache use +preserves upstream filenames, sizes, SHA256 digests, and native package checks. +The worktree's ignored `.cache` link points to the repository's shared cache. +No task images, official tests, or package origins are substituted. + +## Run after resolving disk space + +From this experiment worktree's root, provide `DEEPSEEK_API_KEY` in the process +environment and use an isolated virtual environment. The command below first +runs all installation checks; models start only if those checks succeed. + +```bash +export UV_PROJECT_ENVIRONMENT=/tmp/nanopy-pi-dev34-harbor-venv +uv run --frozen --project jobs/runtime-source/benchmarks/harbor \ + python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment.json \ + --phase all +``` + +The runner executes each job through the standard repository entry point: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run --config +``` + +Each invocation runs from the frozen runtime checkout. Native jobs and input +manifests are stored under `jobs/runtime-source/jobs/`; orchestration state is +stored under `jobs/pi-deepseek-flash-dev34-20261024-orchestration/`. Existing +records cannot be overwritten. A previous interrupted job requires inspection. +If preflight fails, preserve it and register any corrective installation check +separately before authorizing model work from the resulting evidence. + +During execution, inspect operational stages only. Freeze all 34 outputs before +reviewing new task trajectories or hidden-test content. Keep setup and verifier +errors separate from scored task failures. Once the run finishes, produce both +complete English and Simplified Chinese reports together, plus audited results; +no result report exists while the experiment remains unstarted. + +## Preparation evidence + +- The Harbor workflow suite passed **187 tests** before the final usage/error + accounting changes; all **51 relevant tests** passed after those changes. +- All **35** Harbor configurations and the frozen experiment plan validated. +- The actual Pi 1.4.0 binary accepted the registered provider via + `--offline --list-models deepseek-flash`, using a dummy key and no model call. +- All **34** registered task-file sets and Docker image identities matched the + previous run. See [image evidence](image-digests.json). +- Container dependency preflight: **pending**. Model trials: **0/34 started**. diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/adaptive-rejection-sampler.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/adaptive-rejection-sampler.json new file mode 100644 index 0000000..e06791a --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/adaptive-rejection-sampler.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/apt-cache.conf b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/apt-cache.conf new file mode 100644 index 0000000..e191ddb --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/apt-cache.conf @@ -0,0 +1,8 @@ +// Route only public APT HTTP origins through the verified index cache. +Acquire::http::Proxy::deb.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::security.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::archive.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::archive.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::security.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::ports.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::old-releases.ubuntu.com "http://127.0.0.1:18742/"; diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-cython-ext.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-cython-ext.json new file mode 100644 index 0000000..4fa595d --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-cython-ext.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-pov-ray.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-pov-ray.json new file mode 100644 index 0000000..ba6f8e5 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-pov-ray.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/caffe-cifar-10.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/caffe-cifar-10.json new file mode 100644 index 0000000..c8820d9 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/caffe-cifar-10.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cancel-async-tasks.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cancel-async-tasks.json new file mode 100644 index 0000000..71d40e3 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cancel-async-tasks.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/circuit-fibsqrt.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/circuit-fibsqrt.json new file mode 100644 index 0000000..f200396 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/circuit-fibsqrt.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cobol-modernization.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cobol-modernization.json new file mode 100644 index 0000000..f452d50 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cobol-modernization.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/constraints-scheduling.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/constraints-scheduling.json new file mode 100644 index 0000000..706d120 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/constraints-scheduling.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/custom-memory-heap-crash.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/custom-memory-heap-crash.json new file mode 100644 index 0000000..1ceaa2e --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/custom-memory-heap-crash.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/db-wal-recovery.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/db-wal-recovery.json new file mode 100644 index 0000000..9dab4a8 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/db-wal-recovery.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json new file mode 100644 index 0000000..b6c62ff --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json @@ -0,0 +1,100 @@ +{ + "services": { + "main": { + "volumes": [ + { + "type": "bind", + "source": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/apt-cache.conf", + "target": "/etc/apt/apt.conf.d/99nanopy-index-cache", + "read_only": true + } + ], + "environment": { + "UV_INSTALLER_GITHUB_BASE_URL": "http://127.0.0.1:18743", + "UV_PYTHON_INSTALL_MIRROR": "http://127.0.0.1:18743/astral-sh/python-build-standalone/releases/download" + } + }, + "apt-index-cache": { + "image": "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "network_mode": "service:main", + "entrypoint": [ + "python3", + "/proxy/apt_index_proxy.py", + "--bind", + "127.0.0.1", + "--port", + "18742", + "--cache", + "/cache", + "--workers", + "16" + ], + "user": "1000:1000", + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "mem_limit": "256m", + "cpus": 0.5, + "volumes": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/scripts/apt_index_proxy.py:/proxy/apt_index_proxy.py:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/scripts/cache_image.py:/proxy/cache_image.py:ro", + "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/apt-indexes:/cache" + ], + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + "import socket; socket.create_connection((\"127.0.0.1\",18742), timeout=2).close()" + ], + "interval": "1s", + "timeout": "3s", + "retries": 30 + } + }, + "uv-release-cache": { + "image": "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "network_mode": "service:main", + "entrypoint": [ + "python3", + "/proxy/uv_release_cache.py", + "serve", + "--cache", + "/cache", + "--manifest", + "/manifest.json" + ], + "user": "1000:1000", + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cpus": 0.5, + "mem_limit": "256m", + "volumes": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/scripts/uv_release_cache.py:/proxy/uv_release_cache.py:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/scripts/cache_image.py:/proxy/cache_image.py:ro", + "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/uv-releases:/cache:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-release-manifest.json:/manifest.json:ro" + ], + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + "import socket; socket.create_connection((\"127.0.0.1\",18743), timeout=2).close()" + ], + "interval": "1s", + "timeout": "3s", + "retries": 30 + } + } + } +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-release-manifest.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-release-manifest.json new file mode 100644 index 0000000..e762efb --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-release-manifest.json @@ -0,0 +1,58 @@ +{ + "registered_at": "2026-10-18T13:25:43.131474+00:00", + "metadata_source": "Original release metadata preserved by the source manifests; no versions or payloads substituted.", + "source_manifests": { + "benchmarks/harbor/configs/dev34-main-baseline-20261012/dependency-release-manifest-v5.json": "bdf3d3ef8b7743700c5294c4a511c686a3928c178a77fd4abcba90ee4557ffcc", + "python-agent-release-manifest.json": "78aecd7c397c469196f9d246a160bf1fc99ad2d51cb71320b7d26ebf839647ce" + }, + "assets": [ + { + "version": "0.9.11", + "filename": "uv-x86_64-unknown-linux-gnu.tar.gz", + "url": "https://github.com/astral-sh/uv/releases/download/0.9.11/uv-x86_64-unknown-linux-gnu.tar.gz", + "digest": "sha256:817c0722b437b4b45b9a7e0231616a09db76bab1b8d178ba7a9680c690db19f0", + "size": 21699161, + "asset_id": 318993196, + "release_metadata_sha256": "c1a1378ece65ce1aa8343e57bdbd305289a8fc6d3d9180e8bb6b0f69acd91413" + }, + { + "version": "0.9.5", + "filename": "uv-x86_64-unknown-linux-gnu.tar.gz", + "url": "https://github.com/astral-sh/uv/releases/download/0.9.5/uv-x86_64-unknown-linux-gnu.tar.gz", + "digest": "sha256:2cf10babba653310606f8b49876cfb679928669e7ddaa1fb41fb00ce73e64f66", + "size": 21370871, + "asset_id": 306970260, + "release_metadata_sha256": "1921b17d3370aafb690b29d1d121ee8719ef3d3dbc0c7cf3b948c3059a927a5d" + }, + { + "version": "20251014", + "filename": "cpython-3.13.9+20251014-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251014/cpython-3.13.9%2B20251014-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:c0ccda275948c79996e46993c2c5476ff5cca606dee530f1dea712179131b348", + "size": 33517986, + "asset_id": 304305340, + "release_metadata_sha256": "8e5db21cfcc2bcafcb2127f269f00cbfdd1f74842a685acd2a14e3c1ad6f8394", + "repository": "astral-sh/python-build-standalone" + }, + { + "version": "20251120", + "filename": "cpython-3.14.0+20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251120/cpython-3.14.0%2B20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:66f385305ae0eefd6b65e2cf942bc91d943a61e26fb7581e029f760a2b04f393", + "size": 35376131, + "asset_id": 318881850, + "release_metadata_sha256": "e3a737862d2dddf11c73bfef6c2f2579e9d4ce5bd271cca53790c998590dee58", + "repository": "astral-sh/python-build-standalone" + }, + { + "version": "20251120", + "filename": "cpython-3.13.9+20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251120/cpython-3.13.9%2B20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:1bc9a94ab2425fb6ae58e45f69cab61647759b57895649701390822d976fa923", + "size": 34257108, + "asset_id": 318881608, + "release_metadata_sha256": "705690ab2fd466e90b4c868635c529f3c7fd0cb597d399e350f399e3f3e35e6c", + "repository": "astral-sh/python-build-standalone" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dna-assembly.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dna-assembly.json new file mode 100644 index 0000000..914ef6c --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dna-assembly.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment.json new file mode 100644 index 0000000..7b11cd4 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment.json @@ -0,0 +1,1409 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-24T14:06:54.025652+00:00", + "experiment": "pi-deepseek-flash-dev34-20261024", + "total_model_runs": 34, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "minimum_free_disk_bytes": 16106127360, + "log_dir": "jobs/pi-deepseek-flash-dev34-20261024-orchestration", + "arms": { + "pi": { + "agent": "pi", + "source_root": "jobs/runtime-source", + "source_ref": "d6ed70a5d6a1c5b0f9b6c9d20b4a1c2b49153f67", + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9" + } + }, + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/adaptive-rejection-sampler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "scientific-computing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "95408217b9c484ffc18a02a91b7706323b6902919c72b24866a32e27a23543e2", + "environment/Dockerfile": "e644a3343e339303230f1441b7383f076d21c69a5a7aa6430b4f8a9d27f15c8b", + "environment/protected.tar.gz.enc": "7fc9bf1347bb97da8dede04b2c58638f225070913e222d76627068aebd1b1df2", + "instruction.md": "e6d28c28acd76bef61817427c74c2a29e3e8d75f535cbd708afcbfd02c54013f", + "solution/solve.sh": "fe443a8082b5183ba1623311ba0ee958888aefa633eff93766b9ae14e9bdd84d", + "task.toml": "6ed7b37f74beb46e9d47b0f5fad53025667c9548baff4e96d467b56d429fab1a", + "tests/test.sh": "38b43560d173cc2b952c3a3e17b8a480216d84e33450515e047bcb0d806b1e0a", + "tests/test_outputs.py": "547dc6e107f034f41703722aeceb6d0236e3fb69116fc3f2fbaba11884de352f" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/build-cython-ext:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d5f5113941c3963292bae396baa9369571026f46e59437f98ddcc93475dd7729", + "environment/Dockerfile": "cb6cf421cb9b69d43bd25cd491e49bda982fbb865d47515178d017eda727194e", + "instruction.md": "7e52874e45f88505bd21540229719de6639a401477557d814489981d472f0e34", + "solution/solve.sh": "41bbdde88617070366a70e518eedca466a4ea1acd7e96899b2fb7da988e40abc", + "task.toml": "c472f0e7522c128fb7f03e00b50c8cdf2e5e6474daa1e094e7441f4b54e770b2", + "tests/test.sh": "0156c9c4085258b696ce424cf1ce96cba44d1263c5d73bdf54083fd80237f915", + "tests/test_outputs.py": "6549f1a8e04b6163ec08877e6f6b9868bf94563f3a3c72ac1fba9c512c31689e" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "native_agent_timeout_seconds": 12000, + "verifier_timeout_seconds": 12000.0, + "docker_image": "alexgshaw/build-pov-ray:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "38380f64bc631916e264f80ce4d37feedbc783310001475b367d03989d114fdf", + "environment/Dockerfile": "44f2b5fc7311d8f336daaaca99dd4216b49355e67f50b494ebacf1b0f7c14e92", + "environment/deps/illum1.pov": "5a1f7f53ee0730c4923fcd21cd4a9c34719024d8557920441551ba7d96e75dc4", + "instruction.md": "56c8ecdd0084a07f8c97986c43af5c37ed03dd5aa994da6735cf59806162d1b2", + "solution/solve.sh": "3da9dbf589656512cd7a7004dc3752d3552a6af939cfe59eff79f91695fa42c7", + "task.toml": "b90a47ec26c1191d3c523cc094ea96b170e42fbfe176ef1f814c57d05390c097", + "tests/reference_illum1.png": "bb084f430bbc9d5b623a656920670dc06cee6d9132a9d550be4a1b99e41ab9e0", + "tests/test.sh": "928d1497ad21aef0c456294e3784f1fddf8493410f23584da89f9e02ec6bd134", + "tests/test_outputs.py": "64d6ce107503714ada346e5606832ee886be9950971cf81523e73d8ba28e8d37" + }, + "prior_nanopy_work_budget_seconds": 11820, + "pi_agent_timeout_seconds": 12000 + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf", + "native_agent_timeout_seconds": 3600, + "verifier_timeout_seconds": 1200.0, + "docker_image": "alexgshaw/caffe-cifar-10:20260403", + "resources": { + "cpus": 4, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "machine-learning", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e918e6852c5c034422b81a6be22d7596fddcee53abe088f56727b2482da32728", + "environment/Dockerfile": "b3b8adc915a1afa035b604273af3bc1aa8b90999d2dfd72763218b34cb87abcd", + "instruction.md": "46907b0ee048f74269c29a15e0e086546b2b1220149597017da8b5ccc05a0cea", + "solution/solve.sh": "771887de1ed557269c7f3160c777519aa9538e1e9b59771b2f6ef38c8952490c", + "task.toml": "2831165ad9d9e8bbc646e6d85c0a2290640dcc4f13749547a2937c7675012a8b", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "e99b9f6284e923388fbc21488b1af99fdde46d4a809d3557b0788dbf1cb6b281" + }, + "prior_nanopy_work_budget_seconds": 3420, + "pi_agent_timeout_seconds": 3600 + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/cancel-async-tasks:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9adf86b1279cc44f83fe16e6d340595aa992240f1d87243d204cae32e453693b", + "environment/Dockerfile": "1ee7bdb4a7536f5f114c11a3ac6ff42790abfea901fcba6b2a8ea8685f19140e", + "instruction.md": "23d22f7039b50682941652c716f96dac314b8585c32c133614cac77dc067f47e", + "solution/solve.sh": "4938ca4ad5b7dcd36b414b42a1abb10c058ecefce7462b678f9bd05c9fae1844", + "task.toml": "14db0675e04408f2133b0067e8dbddf66a1d6e7baf6b8d38bef8ecccfc4e58a8", + "tests/test.py": "f533c0a64db068f582b4733c5b81f2e66b8fa52fcd3efa65752e6bd6d24780aa", + "tests/test.sh": "79600a6b863def31b4daa008297423010cd2d6243e689d67503e118e4dc53397", + "tests/test_outputs.py": "d6f510b51fafc81b4e9b5cc95535aa13491c8753e0e4171db42c45f4de460ee7" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3", + "native_agent_timeout_seconds": 3600, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/circuit-fibsqrt:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "201f7c889dd6f73dd83ca078e347272b33720fa622f8c757ea6d5770453f4cbd", + "environment/Dockerfile": "780c23986778f3e5f842567868af30b1135ed095ca9449e1c90b7f0e0483d3d4", + "environment/gates.txt": "032786f94f16605767c62bb61ce4dbba7ab7d3f1046712c975f01c9a04b6ad29", + "environment/tests/sim.c": "700645ac6156cdc88ec1e210f7c7d8d9969bb42aec969c410597e8efca796173", + "environment/tests/test_outputs.py": "6117b8edb7d567b30e32b16a9244bec4c131ec01e9065d52f6f53f87e36f0438", + "instruction.md": "a7f500266a878d04ac9de39fba76220cfad216d8f6b8063bba93bebb3d4ab9ea", + "solution/solve.sh": "a4ffab4183815a0eb7646abc1de1fc67695014c060d7ff16e8bca9ec94385d0b", + "task.toml": "9f5aa1f7ebbbcec90fd109121e9bd509af3cfff6b4cec701bd91947988f40972", + "tests/sim.c": "700645ac6156cdc88ec1e210f7c7d8d9969bb42aec969c410597e8efca796173", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "6117b8edb7d567b30e32b16a9244bec4c131ec01e9065d52f6f53f87e36f0438" + }, + "prior_nanopy_work_budget_seconds": 3420, + "pi_agent_timeout_seconds": 3600 + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/cobol-modernization:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "c89318cf59efcc7040e3d946022e2616161b6733931a080cdeea7b1918e7ecc4", + "environment/Dockerfile": "8b175c83fc03ec7d16d6101516bc5100230b592f480e435279f4fe051c9280b2", + "environment/data/ACCOUNTS.DAT": "46f63eb29442ebc6716bcd8a0c36f4367910f2783f6b84a307ed9a61ccd1046e", + "environment/data/BOOKS.DAT": "db04da646207cd87f8e9c5b6f0b57d95953899f3eef22de1a81423f06ef4a9fd", + "environment/data/TRANSACTIONS.DAT": "1fa4cc5420ded94d585c563e8196b85166ebbae2bb8924b20014717fb2fb9ee3", + "environment/src/INPUT.DAT": "52fc984f0cf14af1495285d7efe02b9bae13159eea59255560ee938c4f78afee", + "environment/src/program.cbl": "27cb0e56f65f52fa75f346720b37a180b8e034ebe7d7622b979f195087bd6ebb", + "instruction.md": "a87074f1141b7345394cd88053d0debb0dc8f7de95308882c6538ee9a3b359da", + "solution/solve.sh": "7d4a0504e1686c1790bfbc074d4e598f4c631f54c2bc5f3d5fd786574b4810b9", + "task.toml": "4b620211dc3d6b1b989f0210c5ecd2f606974581c8bce27bd78659d48cf0cfbd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "009180a7092583fc8143bb3ffc3957923aa0d61d0aac3a471f76eeb4aae54a88" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "native_agent_timeout_seconds": 1200, + "verifier_timeout_seconds": 1200.0, + "docker_image": "alexgshaw/constraints-scheduling:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "personal-assistant", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d54a690970149e1216bd4fc066b45069c7948c13e3b1ef2a213a74342805f209", + "environment/Dockerfile": "7d9aba072a4a907229728b9739c8cf821fc2c98450babd64452682caf3759c84", + "environment/inputs/alice_calendar.ics": "5a5d45804bffd339c2479d49e76fe817b38244d14d389afe1de583443808ecbc", + "environment/inputs/bob_calendar.ics": "c97e85dbcdd756758f8a80f4db5fd5c1dcacee79298e4d2262f46de5ff680b50", + "environment/inputs/carol_calendar.ics": "32a0f8687637df0a12b8e5c56d49c4192bc3e790856943632d70f237c7c1363b", + "instruction.md": "342def78177f9bc41bea4d11af679bc385a9dda2ac8a1e425be384a5b7f937b6", + "solution/solve.sh": "802d81ec064fad3050bbade95a2f847ac24e0b7f80fa69b0bcee0347dc92fe77", + "task.toml": "7b9842de16d74e8faf8c7c0aa03b0b81145941cb12dd322d81ae23fc48607250", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "1422b345ba3aacdfc4184e962811b6379988435896c50d8cf62903e70990c5d1" + }, + "prior_nanopy_work_budget_seconds": 1020, + "pi_agent_timeout_seconds": 1200 + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "native_agent_timeout_seconds": 1800, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/custom-memory-heap-crash:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a06ed23a2c467aa36d0d4a88e5c020b1076cc25b8d483517239658e9c596dde9", + "environment/Dockerfile": "6ee81e5a4fa2540443d7f27d577ba840728fc6aab9f301ab7919103673c78080", + "environment/patches/locale_classes.tcc.patch": "16951f4a4df6bf253886ac75f5a39fdbc328e415194493f63cb7589a364aa817", + "environment/patches/locale_init.cc.patch": "e97f133ceebd86f8881f972e0e45b0ec0a13a9bfa428d8032008707683d54292", + "environment/program/main.cpp": "b94538c76887cc2b2ab739b5ac4731cddd0faa8f4a9892a6265c062bbbeede14", + "environment/program/user.cpp": "92b203786fed1bf0b3aa1675c9cd2d82b2651170675064da8ffee73b98ef14f9", + "environment/program/user.h": "f43d155995a56c22b9cd1417daef7c3b5a976cdc662a98eef0feb438ae1eee77", + "instruction.md": "53c68ae14a3c3af9a97c40bfd9bdc30a5d98b37eaa2f69f763475745f70f108d", + "solution/solve.sh": "b0b1d10bea1cd5be14a82b35d522df616df1e67028f90d2a6bef90aaa98511a1", + "task.toml": "ed474be1983e212913aa8866ebb66216acbe36aef2339ac375d232aa46617481", + "tests/test.sh": "70a2b55f0c5757f6b123a216dfd202be4a646680bced28dddcd31fd8959a058f", + "tests/test_outputs.py": "4e1cb33f777fa7fb5e1536f9bea9eace6deccd515c67d4f87570adbe692914ba" + }, + "prior_nanopy_work_budget_seconds": 1620, + "pi_agent_timeout_seconds": 1800 + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/db-wal-recovery:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "file-operations", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "600762031a1a02ecc3b46d70f7683f3ff930b47c8a3d0e27cc31b146158c3126", + "environment/Dockerfile": "63a260531c62f4ae72060db17f295c0aa03cfc6321f8162c324b7aa6b3f81302", + "environment/main.db": "68cb288eb29e76d814b4e7ee6887b37c415675e644f88173bc191348e290d119", + "environment/main.db-wal.encrypted": "68753bdce26196a53194260d001544cc6bb8f9f38e2f2f6826d09798e31e6e26", + "instruction.md": "5a20375162921b8911c67035d1905df7fd335d1bed03044271e608b2c3c93b26", + "solution/solve.sh": "ed8530c94103bf19783a1d41bc63fe4db478cdd9629f22fee25c45a45699f5e8", + "task.toml": "1cdc2c615e259bc10265efbe71b518332070c1c002e226566399e9788f18d0aa", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "88dc5c2a3abba75bca0e0f9136223ae5bb585d5984b8d9eba51a7829c1d2ba49" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "native_agent_timeout_seconds": 1800, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/dna-assembly:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "scientific-computing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e7855c23d9822f688eec4dd89ca0f1caf91c6a69e1c8f3c9eb544b4ab2a94821", + "environment/Dockerfile": "26e9e43a98f0b1d132bb638f44b7dc882b0d0ca32d56fa33048adeee00ab600d", + "environment/sequences.fasta": "ebdc361b5fba28eb0a2c7a8a8a444ccb12cc842c8cbc6df5a2c24254f130736f", + "instruction.md": "88998c98beabdd966ab46c04ede822bbc74796696aa1bf87c567a9131210533c", + "solution/solve.sh": "c976cb118c8ef1befd846ba05fd5cadb1b1f529bc76b9f2ea9cb3049e6e11a47", + "task.toml": "7b571544b64d9be1c91526837e5d805c2181afe0e6fd57a7f0676bba36c868d7", + "tests/test.sh": "271c8317c98539e3b49233d017fe51fb0e5fba8114e76830fe698565499b2b1d", + "tests/test_outputs.py": "5c2d0fd2e5f6303263e9750f67acda7db491ef04841832b4fc62bcfb61b08924" + }, + "prior_nanopy_work_budget_seconds": 1620, + "pi_agent_timeout_seconds": 1800 + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/fix-code-vulnerability:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "security", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "87f28dbad183c3a634887696d328ca68a8a0858f132bc48cb7341c22267d38f7", + "environment/Dockerfile": "0917470986224df4daf4854a71302ae81c73ba5dd6e25be1232d29b758a5b3ab", + "instruction.md": "89b1abdf0af19399f720233b5cdf6b2e8e0ff7b6e3fe3e2a3af0d4d298a8641a", + "solution/solve.sh": "7a0131b369d83ae8546ecc3f9b4e23d38240f1bd1f0941e4430fa85b5182ee03", + "task.toml": "9aa2bd14ce9408289d199d0608bc3fa65b8b5e23374c9641020de62197dbf1e2", + "tests/test.sh": "f1611da693cb4cfc5e8482698c6a1858a5c67220ae9db89b50aed66584e9b17f", + "tests/test_outputs.py": "c006a5de95b69bd78055aa26cc055734dbd50718ede25d331f767bfee4ef7f26" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/kv-store-grpc:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "51fdc2ad06f0adb1270164f247ea41799b2ee2ad15490e48ddbf5e79d3767aaa", + "environment/Dockerfile": "47c9982789092437f950736110b2bd18899c39b8fd20f3af3c95b848e564a197", + "instruction.md": "f4b3facc199d5b1c78db6e603ffa29ec1b93a77ff7d291e1a92b42e6bf582bed", + "solution/solve.sh": "6bee2938ac9731d1156b0ca02b28c3d5793bf2cbe5f995c863d397093af01cb7", + "task.toml": "6ff0f30a526bfff86a67b0b53439b9965229c0f368167dc1a5d05f34d82299a3", + "tests/test.sh": "c348467a076bf614d0a850e25b2263f85bbf7b4c7a8d344fbd5454a0611a0df8", + "tests/test_outputs.py": "115de2e964bbe689d07930527113422ef8a9e4be4eb958ad84594e3fcd2b1b0e" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67", + "native_agent_timeout_seconds": 1800, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/llm-inference-batching-scheduler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "machine-learning", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "ecc9dd635357c184742cf72fc1e452a8946e207b183fc9a7eebbcf07133b911f", + "environment/Dockerfile": "10a6193904ee0022ac098504182989893b196dfc74e626a3ca48534a8eacde48", + "environment/task_file/input_data/requests_bucket_1.jsonl": "8b3cf1e2b06074162038064dd2c114abe308acd32cec7d365ee63ee81978f7de", + "environment/task_file/input_data/requests_bucket_2.jsonl": "e15e2d4c4d55f875c42a6042610d6282f6c0657c39d3e0d0f087268252525cb5", + "environment/task_file/scripts/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "environment/task_file/scripts/baseline_packer.py": "547d230d5e6d93197803480cb81cab0ec6ac63fcfa0a24b246f549523f216c4e", + "environment/task_file/scripts/cost_model.py": "0cf417fd5ff0112defcd46970d37a290c0763c5a9092e91cb3ebb86c3f20e6ce", + "instruction.md": "0817374767533fbf51bb93345ee1bba266263c216c6cc55c096e71b4a1d37dfc", + "solution/solve.sh": "cb9efb586d76a893c3027e62029e286eb6c7583dc92dbf1840a3adc236a926d7", + "task.toml": "52aaac942f917e6a91ff658e0272195bd1cd842d41d71e302414c52603cc588b", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/cost_model_for_tests.py": "d3e810a99d40327fb8317f6512d879c52c6aeebdf5dcdc0c5e117c7df61c9ab9", + "tests/test.sh": "005f980332355e2fe143d4ff4aac11828adf070594bbe7055399c238f06953ca", + "tests/test_outputs.py": "80b06b7c4ba0db28832b767d48be74bb462d7f5df5798d9cb58c4f1160d96ffa" + }, + "prior_nanopy_work_budget_seconds": 1620, + "pi_agent_timeout_seconds": 1800 + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/log-summary-date-ranges:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "026000415cb577f3d3d5132902670262f7d5d8f69841903ec06a882292f756b4", + "environment/Dockerfile": "b479f6e987b63a6ec118fa1691b8bb85c437b2a6a0fe63feab5ed7154957fdfc", + "environment/log_generator_deterministic.py": "beb6eff59979e9aa0be63ce8a0f1b0df1178f92871f04ba9f76fa70cf8beb81c", + "instruction.md": "c1d4516ad4ec2238209740eaf01291df9ab48b0028a3fc5df383c6f831453484", + "solution/solve.sh": "9f6a8ee8947bdf1c0f31b530b6194ae998d66841a3b785fa2e246ee840350ab6", + "task.toml": "9a9593673d6805ef3a301ea734e9c4dbaf80da1b37c0fe9d1f2ceea2a8c57e18", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2fc06a6ce1944ad82f84fb437395fa75ed8c0eadd385b92e375c27e5bc9cb2b4" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/merge-diff-arc-agi-task:20251031", + "resources": { + "cpus": 1, + "memory_mb": 4096, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d6b48ed0890b12eb68f72c12cd02689bd6bfae2191dd2688949cde6fe77f6a94", + "environment/Dockerfile": "4a03ae6f0c60b9c2f45c33f0413b5aa75c9811870254e3ff1968fd0119173fc6", + "environment/task-deps/bundle1.bundle": "a759f9b6ffc1a2eee50778d72a1899ad7d724becab84fa30e7aac71ee9e443a5", + "environment/task-deps/bundle2.bundle": "75e3e3916a01c980a03715de67b16c729e03661d94a694b5f4223d229bd6593a", + "environment/task-deps/examples.json": "5426569e4ea078bbb74039ec712a2d4d35f1f9fa3a5f26bf2a0de4a2c8ef5db8", + "instruction.md": "2d826479873e5ac1c94c1f802763bf85c22bb04624384685ebdf6b153fe0278e", + "solution/solve.sh": "48d4becc41fda5b6063fe96947474ffe778a29d6924d7b2bb235eb61e5ce555d", + "task.toml": "6ced40e06b43b169857509765eb7f07791db558f9437cc7367d2536771c2ca5b", + "tests/test.json": "20f116a6b718bbf4874bbe203d15a5c0f85a1d88cb622e2d7529c7e4fb2d4c28", + "tests/test.sh": "c9052954cd6ce25c4a375a2fb48e765b1efa5b39f7991b93993cc793af8ab9d2", + "tests/test_outputs.py": "2754ba6c5c5f75c9db5ed2c5729d9f5d7e21cb7fdb18aa0c65abdc3e7e85c149" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/model-extraction-relu-logits:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "mathematics", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9465accba6c57e5de9a196eed776a16eeb0852a8afc18fc3f8979e2de71b0c7b", + "environment/Dockerfile": "598b1f80d3b7194e97cbf02c8d245de30e303909e0cbe6465125d1d3179fddb5", + "environment/forward.py": "af1a23efd8e4390d16a29777c28116b2f8623ae3f7b628fe14db9f94dac26f5b", + "instruction.md": "c9e179dd099065a34c362a865b0eb9fab5f7d96be582cf117e7519580c036ebe", + "solution/solve.sh": "4cea5094f074ff21453d4423aa1b415a104ed90caa55a4acbb4f09941a3994ea", + "task.toml": "eb64dfaf8ae88868911ae28855cdc4f767de464001e2704919015b595be6dc29", + "tests/test.sh": "3b2d8f5e0353c2bab3515d6387468a863a4f9f241ee131c73f545f3fdf9e2d52", + "tests/test_outputs.py": "e5189c7eb9942ede55e1b48a80388590b80a486b75497613f257b3373ad9b2c9" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c", + "native_agent_timeout_seconds": 3600, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/mteb-leaderboard:20260430", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-science", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "ed147a5bc31e9865b16848b2f01d2f9b1e677af57a7c632d7ea2311c0553ca76", + "environment/Dockerfile": "271fdb5b34520c0bf2cb6462365363ec52058f127cd9eea3f5da2589df5c1044", + "instruction.md": "06ea44106eca0405c25bab00f5b3cfeb611c872093f6c0f82ed057839d1d13de", + "solution/solve.sh": "4ed68831d4acf5d80d2d73eec1f21efd0896a6f51f23020f572cfc0af6da0410", + "task.toml": "86574bd1c6c4fefd2e64eee9c2caa53d87c4bcf2e51547912440a0ddbd0badb8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "b28f8fc8b7614801305813c402be52f4be1e75de6658ecdfdf9383aa793892cf" + }, + "prior_nanopy_work_budget_seconds": 3420, + "pi_agent_timeout_seconds": 3600 + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/nginx-request-logging:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d79997586d1c14ed8dbfea050e3580b25500e04c8345acfd3f294152433c795a", + "environment/Dockerfile": "43003466d89ab83484a3fa618e35b58cdb74a002b1e8716ef30f12b3ac8302d8", + "instruction.md": "26a7ac98aced6107f147206790ada77bbf8ed3c25fb360cf323d5fc6889edf99", + "solution/solve.sh": "2454bbbf7c201a9a778165f286373ba2cb991a0eb1d026a8005d57ae71250dd9", + "task.toml": "b7a78d00be13656f75069197100b7b8c4fd268decf6f78929c853b3222a2729d", + "tests/test.sh": "27761195f53cbfe5941fe39b573631f6c926869e64b7c43220377e1e116528e7", + "tests/test_outputs.py": "045cc716c14efde3b0dcff5fc7c85ec5d18bfc6ce66f8b40a418fa2a3a4acda0" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/openssl-selfsigned-cert:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "security", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "5dfa88651890cda06fa2a25babd3b813818767635e88bbabecc8305d5bb3942f", + "environment/Dockerfile": "f590b523038be43afcaf5291d7b9e8ed1333c48d9994e77b8dee2f2c352f0f40", + "instruction.md": "38054a3371f7c2c853478a2f9af0bff858437d5b28a9a07483238f06e566921f", + "solution/solve.sh": "58cf3c844e907745cd81fb4bd75efb3d080e2f0e82b589c56c954ae6e1756510", + "task.toml": "75c8913f0fa15890786c11999cde920b8bfc10d4ce8a14edf1e76985a2ac5800", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "90970bb3c372e8e7fb6bb9ac5f649ba73eb38e1a5ea2e922a9057a72b6b4192d" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c", + "native_agent_timeout_seconds": 1800, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/path-tracing:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3e4fc952b3f9a860c7ee7837b3f3c686f1283ee7821b014265dba7c94d7fc849", + "environment/Dockerfile": "7aca02e4e20b04ed2269d627a29dd4e7337a02764a9652b641452bf09836a3d5", + "environment/orig.c": "665c1bde0907740890ac114d6b36af1f3fd00449fa287583817f778bb5da200c", + "instruction.md": "9f36e02d636f5dc0946619bc518f2e51c24fa75251da9d1f0ca9eca612cb9572", + "solution/solve.sh": "d107abde101eecee6838da80cc52a3d480c93956fff3a9343a299ea01b3fe9e6", + "task.toml": "cf47013f69bf1b43242da984f231d4daf364fdf44d3845f2eea2ca2cb9308c46", + "tests/test.sh": "a719dbb1ba7ae76df095e9d815941dfa4c55edaf11f173b9c3c3097afa838548", + "tests/test_outputs.py": "572e68b2843191d29592faf42a405861b7958d8e894868b74c5a8886b53eb879" + }, + "prior_nanopy_work_budget_seconds": 1620, + "pi_agent_timeout_seconds": 1800 + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pypi-server:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "906a096a2cc7b9afc738ff5ad5fbd605be65c103f5069c6673915223b9b187e3", + "environment/Dockerfile": "17a4e67580e4f2c6a151d04b9d74eb020666e7eec8e5259c4a5411f9e93858d0", + "instruction.md": "1c537e4a311a8e71155e121f57bfc03f742fbfb99d05314cd03e433996a28a31", + "solution/solve.sh": "1143c0eade4932ec3aeaa5013fb5e23bbcccca15dead294aa03b6af88750775a", + "task.toml": "07c39d7a733c89adadf74bb63a3e906c3694feb6c490d5fc85ac187c003fd447", + "tests/test.sh": "d2b1220e42bcfdcce87907ac5bbd69ac210a3c94926545ca2439636786e7784e", + "tests/test_outputs.py": "82c6351722e9aa984ff1d701bf57df3150ce51466df2b70afa1aa62d2650be4c" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-recovery:20260430", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e080308a05703b40edd16f53aea41a0e6b9874750ecee35bb432029d0d44bf53", + "environment/Dockerfile": "8a6b7edbb9e7c241e447f15a71e13e5d043b25169be2fd9802b84f5be3d56eb6", + "environment/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "environment/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "instruction.md": "2bef07dc28be3ab2c35ae389584034c2e4dde92c2c31b3eb6bd829628c8c3f4a", + "solution/solve.sh": "175a7802777b43e0fa4f629d71f7c68fb40706813c342aaddc72108d89bd5b2c", + "task.toml": "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd", + "tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/qemu-alpine-ssh:20251031", + "resources": { + "cpus": 1, + "memory_mb": 4096, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "qemu-alpine-ssh-v1", + "task_file_sha256": { + "README.md": "72f1c83fa1e3e6381ef85dd82fd109ee18cdeac090311cf3212a0a8cff0a32a5", + "environment/Dockerfile": "015ce0601ff54758f66b33f4952e0d3a577d84c09aed8f3ec5945a570e8d168a", + "instruction.md": "8348c60490cb90bc690cba259dfe98996da4c3294c6d42bba1119b00c64de4de", + "solution/solve.sh": "1cd31186c427b91bb00499d22aa4f0214fc1ca93b40b6e4cfa969611b848db2a", + "task.toml": "10a43ec6ee732eeba852c377f12e23178f7fa5a4868143f3699a15565ffb64c7", + "tests/test.sh": "3b244479776c4ee3639631112799f2cde22cbb80ac10511715e3a8f32909c289", + "tests/test_outputs.py": "83ccb4bdcd259b342212bebefe395f289858b4dc55b2ee7ae0dab238ad2e9718" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/query-optimize:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-science", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a82c53e333a0ac0b0b410b5e26cb8e7305dece31db84d340248ffb13d5a45160", + "environment/Dockerfile": "064818771f200e0517d22068f87229a7a95d0a82b0778c84ca7203bf1926a0b8", + "environment/task-deps/my-sql-query.sql": "43382792f0f438625d643e34abe144febd5669518f4ab1ab550a15c6d9314a21", + "instruction.md": "473e12d320d2732f679fb631c2a4dab4a01deea6775686718ae0c417b8a39447", + "solution/solve.sh": "0d95f3cedb287ff9d47c796cbaf812c300d1222c10fa80c49e8acb0e81eb6978", + "task.toml": "6e88f2a86df699594bcb1bf81df4ed694505f79c9420f671335b02524bbe3868", + "tests/golden.sql": "4da1885d56b3ffa7b7b22b422996d018795611f6963499d551582881b74b84fd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2068ae80c57af8d52a7ae510f0666732c057806e556b480970f36f14bae683f6" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78", + "native_agent_timeout_seconds": 3600, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/regex-chess:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9296cbee592fc996c58d228b51fee303647e5ecfae82d5666d42a40cb30ec355", + "environment/Dockerfile": "2616cd775252303839dfda0ee2f047eb862952b053b0103bd6b2460ef90c11b7", + "environment/check.py": "f9c416ae9ea01dcdd5f6a43170c8434b8c0a0b389a4a30876a2048e277d191c7", + "instruction.md": "a6ed3fcc54fc92d71a84af921942422d299ca89b410dcd92cd05a53d0d06436c", + "solution/solve.sh": "b728bf0ea392a3d244afbd0818d36933c8d3e190e47743c1382cd518d518a2e9", + "task.toml": "07fe52636fcc4bc9fff25d5630af145b97a5331219d719cc8decb4ca67ea21d4", + "tests/test.sh": "8f7f00a1c4dde7fcca6c918c768f2aba432b8c4007c15989716c68091950da87", + "tests/test_outputs.py": "4983c44d21c56dd8191381eff0304afc8b87bb714527ea9036b3319c015a30b4" + }, + "prior_nanopy_work_budget_seconds": 3420, + "pi_agent_timeout_seconds": 3600 + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/regex-log:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "29ffd98c55dbb6f4844f1b9d8c1769b2155c117ee475e0df6bbc5ef330d239b3", + "environment/Dockerfile": "5f264ff0bf62a49aee3c6398447de5108df141e7a2ad35c6de8ef39fa62049c6", + "instruction.md": "4f7ac05e70cf9220ea0f1e5a052c5f908cd0fa884e847d80b0bd51bae2e96f9c", + "solution/solve.sh": "7e670d4f2b4bccb1e4db38f2a173e085ceda028c38167912b466b0a84fcc0999", + "task.toml": "6bc98525852755caf1fe34c174c2f10f4328adb4ce6c1968dcafe1c877e4e160", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "345c3bd09ab6f6fe8c8361a58c0a47bf0a13b3fcb38a5ac7824e44ff855e8f72" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "native_agent_timeout_seconds": 2400, + "verifier_timeout_seconds": 2400.0, + "docker_image": "alexgshaw/schemelike-metacircular-eval:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "095d3ec6dcc0fe0c7a7e92355456dcea6992f91d1f8d397669b8cfe18475f506", + "environment/Dockerfile": "9b3df76222e16aad360fbd34cb0f71a0be8986052c065b10b92c0cdf417b25c7", + "environment/tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "environment/tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "environment/tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "environment/tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "environment/tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "environment/tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "environment/tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "environment/tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "environment/tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "environment/tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "environment/tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "environment/tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "environment/tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "environment/tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "environment/tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "environment/tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "environment/tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "environment/tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "environment/tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "environment/tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "environment/tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "environment/tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "environment/tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "environment/tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "environment/tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "environment/tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "environment/tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "environment/tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "environment/tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "environment/tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "environment/tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "environment/tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "environment/tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "environment/tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "environment/tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "environment/tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "environment/tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "environment/tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "environment/tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "environment/tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "environment/tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "environment/tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "environment/tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "environment/tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "environment/tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "environment/tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "environment/tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "environment/tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "environment/tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "environment/tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "environment/tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "environment/tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "environment/tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "environment/tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "environment/tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "environment/tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "environment/tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "environment/tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "environment/tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "environment/tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "environment/tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "environment/tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "environment/tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "environment/tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "environment/tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "environment/tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed", + "instruction.md": "3af5f9567b62f93683db7518ec6fcb609f83796258aa02166b5a7091c5b69031", + "solution/solve.sh": "e2c5b0ac2bef4801a1891e13c108813381877426c1a7f01bab86b4fc1f712510", + "task.toml": "8dae8eb226183b0ca92331c1106152a7abc93b03b4e5a95291b86a4ddc9b44e0", + "tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed" + }, + "prior_nanopy_work_budget_seconds": 2220, + "pi_agent_timeout_seconds": 2400 + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/torch-pipeline-parallelism:20251031", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "297ddea07c230dfd246a8ab07337dd7abcd8e5e2a2b2e26590cb0a8af779839c", + "environment/Dockerfile": "c357ea41e4a4013c5c5980b8fef04af6a5ae250ccc6697cf3bad3127401a931f", + "instruction.md": "989bfcf0c54baf6c8cdc391743dbb2f9e858c5779dd85112536875d4d1b2dcd2", + "solution/solve.sh": "787198e4178d88b887697aefdd29819f29435ffdfe356545d92f58a30fb38b70", + "task.toml": "8b4f9f525c08afbbc4eca9168e46238a61ccb903486773e9ca4da64bb7ffe857", + "tests/test.sh": "cc9fd28b6590993fc37e4a99da771cbf04726c8c62fe308e802612241da69dea", + "tests/test_outputs.py": "23574246a407163a8e6581c84287cc2f1eb745efbaad2a317a5a28c39958c64f" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/torch-tensor-parallelism:20251031", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "4c1b3c3be30548f1f94c6f5503b1bc74b006af7f192dc4cd026f411df1a54604", + "environment/Dockerfile": "5f264ff0bf62a49aee3c6398447de5108df141e7a2ad35c6de8ef39fa62049c6", + "instruction.md": "13bde2ab165d72a311e2bf066d5d6c3c8761bf9d3dd39d1b23eda23bc546b910", + "solution/solve.sh": "d083b7cbf9dbb6cc2ff38c8f2acc150f0059c04aea42d6c2183aeaecc5a19385", + "task.toml": "72fd2a5500d1fe8795dcae76cccb1a29fb25894881a7475f2fa519c87d425a8d", + "tests/test.sh": "a7e6c9408b2401e4e7448371f7ddc72f3284caccfb56277d3cd4e98051868cdf", + "tests/test_outputs.py": "4ae4285e15fba1fcf13d7b7c0638ac8c2e2b0797c937abab1fb6b136fd508290" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e", + "native_agent_timeout_seconds": 900, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/write-compressor:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "603015a28500a28f0838fea2c43a5168613746bfd07737920f4bae40b5a326f7", + "environment/Dockerfile": "9a583d29d46818f6ff2be7c6a0879d03343106e5376fd5a48eb5926b402bb93b", + "environment/data.txt": "0fc483d0fb31128dfa41ed11adca56c88688888189826ef9daeafd7559b4fa4e", + "environment/decomp.c": "7f8d9ef66943f912a217aa48f4ef5efc967b44c2b21d0f56df029b5ab413f62a", + "environment/main.rs": "fa53d6572fca95903e30d2d953f1e428bb8b63513ff218ebee9bc60a425ab389", + "instruction.md": "c4fc3e73e44deeac95f1cd4a34a2d87e853b22ceec2b2824767f5f087b470bb2", + "solution/solve.sh": "01550b7de7ab1320b1898eae055226bb7c73bee259c75102d2240c974a730409", + "task.toml": "6066a509066433c97c417fea68c4e3e73996f698dcdcbabe07718835eea355a6", + "tests/original-data.txt": "0fc483d0fb31128dfa41ed11adca56c88688888189826ef9daeafd7559b4fa4e", + "tests/original-decomp.c": "7f8d9ef66943f912a217aa48f4ef5efc967b44c2b21d0f56df029b5ab413f62a", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "809f486053e3ffbce852111dee1f63d80ab31297bd33172389e50278bed384e2" + }, + "prior_nanopy_work_budget_seconds": 720, + "pi_agent_timeout_seconds": 900 + } + ], + "preflight": [ + { + "arm": "pi", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/install-only.json", + "config_sha256": "246429e25d0b54aff9db85f2f93435e80e714be666595b2f4e7fa32a14f11795", + "job_name": "pi-flash-dev34-install-20261024" + } + ], + "queue": [ + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/adaptive-rejection-sampler.json", + "config_sha256": "e737e23d0d9e761c13d934411a79055c35cf4a7c13502d79717dca7ce790a9a5", + "job_name": "pi-flash-dev34-adaptive-rejection-sampler-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/build-cython-ext", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-cython-ext.json", + "config_sha256": "da458c548221b2a1066d4aa30ddc778ad6e3cc3c33c8602e2e1a9cb12df6fb23", + "job_name": "pi-flash-dev34-build-cython-ext-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/cancel-async-tasks", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cancel-async-tasks.json", + "config_sha256": "cfb498953fe328ee6f005d8783e385616b69c3434fb1c8ecd1aab178bd4c0dce", + "job_name": "pi-flash-dev34-cancel-async-tasks-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/cobol-modernization", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/cobol-modernization.json", + "config_sha256": "96d36cedda0a82b9f07a76459212f7ed0c8488b71092ae684773312c4b3917b6", + "job_name": "pi-flash-dev34-cobol-modernization-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/db-wal-recovery", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/db-wal-recovery.json", + "config_sha256": "c7942215a12854d5f9ee54ca7340ba3f9e89190856efab054742bf2fdb4b305c", + "job_name": "pi-flash-dev34-db-wal-recovery-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/fix-code-vulnerability", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/fix-code-vulnerability.json", + "config_sha256": "6bd358f64d0b58b38784faac7b784f678bf5d679f25e44759fc0440e57ab1651", + "job_name": "pi-flash-dev34-fix-code-vulnerability-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/git-multibranch.json", + "config_sha256": "ec21d2f29cc528fce2f36897b2f1a82262fbb44a818e7195856d791725d5e928", + "job_name": "pi-flash-dev34-git-multibranch-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/kv-store-grpc", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/kv-store-grpc.json", + "config_sha256": "3251f511205c9f21a6390e904392661352addee3db92d8498c6d1fb90548b4fe", + "job_name": "pi-flash-dev34-kv-store-grpc-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/log-summary-date-ranges", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/log-summary-date-ranges.json", + "config_sha256": "7c859a4ba515eb7a3d4ca332b5a72acb099e4d8146c7412b25be532084068351", + "job_name": "pi-flash-dev34-log-summary-date-ranges-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/merge-diff-arc-agi-task", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/merge-diff-arc-agi-task.json", + "config_sha256": "44fb82b39850f58dfdb7d3d4da903f043dd37e5c83bc48312798612870df1f92", + "job_name": "pi-flash-dev34-merge-diff-arc-agi-task-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/model-extraction-relu-logits", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/model-extraction-relu-logits.json", + "config_sha256": "89451efa0da9e693d359be60cbcfca12ab8148c5d081c6da8acd2fd6f933d3d5", + "job_name": "pi-flash-dev34-model-extraction-relu-logits-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/multi-source-data-merger.json", + "config_sha256": "c168fada695ee17b8da44d005693b0d701be6cc843dd1a037e4feac9c24621de", + "job_name": "pi-flash-dev34-multi-source-data-merger-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/nginx-request-logging", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/nginx-request-logging.json", + "config_sha256": "6a9adf41ab2bf874c0ea037085d9ccd42573589b7010d5d3a56f0b8fe9b356b4", + "job_name": "pi-flash-dev34-nginx-request-logging-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/openssl-selfsigned-cert", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/openssl-selfsigned-cert.json", + "config_sha256": "f9f0ec331c08d2db293109d7d40012f92249cfc206b0873b3ae9c0230089d813", + "job_name": "pi-flash-dev34-openssl-selfsigned-cert-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/pypi-server", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pypi-server.json", + "config_sha256": "e4238419099c1eaf7da9f3bbcf8cc72a8d382e1a27ce2bdc95182c62af110943", + "job_name": "pi-flash-dev34-pypi-server-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-cli.json", + "config_sha256": "f84653ea98195696e9377fa5504362b20a7ebab746e4a44eac469133399d1226", + "job_name": "pi-flash-dev34-pytorch-model-cli-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/pytorch-model-recovery", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-recovery.json", + "config_sha256": "ca8d05456eefa36a767610781835c0e3ebfa6a587c12aa814ad4a940d507dd12", + "job_name": "pi-flash-dev34-pytorch-model-recovery-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/qemu-alpine-ssh", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/qemu-alpine-ssh.json", + "config_sha256": "eba0397cf677116056fe2c64cfd3883252288c1263ce2b65a9c51657a8de9f9c", + "job_name": "pi-flash-dev34-qemu-alpine-ssh-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/query-optimize", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/query-optimize.json", + "config_sha256": "9f33dd57e5beba6e14b9ff9e1e163d1bf9cd6742402b386aaca792965636c482", + "job_name": "pi-flash-dev34-query-optimize-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/regex-log", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-log.json", + "config_sha256": "b150765adf9dd3eda13a9befc40abeaa807479653f8df9eb157e894283e23269", + "job_name": "pi-flash-dev34-regex-log-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/torch-pipeline-parallelism", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-pipeline-parallelism.json", + "config_sha256": "b8b9f74d9ec270eaa8215b69528e09214d33a3d565a9b7e5301de44342773347", + "job_name": "pi-flash-dev34-torch-pipeline-parallelism-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/torch-tensor-parallelism", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-tensor-parallelism.json", + "config_sha256": "c66e31078c9ac76e056984c16b19338004c203afc6cb700ed6fa219979362737", + "job_name": "pi-flash-dev34-torch-tensor-parallelism-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/write-compressor", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/write-compressor.json", + "config_sha256": "4c3398cec611bc726de21f2833be71a6d8c5332b37fbed155b72eea43bc73ac5", + "job_name": "pi-flash-dev34-write-compressor-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/constraints-scheduling", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/constraints-scheduling.json", + "config_sha256": "5b7ad367963a6894d17eef2ae698f6f4ba329ac779072abacb5f8f8eb5f13f5b", + "job_name": "pi-flash-dev34-constraints-scheduling-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/custom-memory-heap-crash", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/custom-memory-heap-crash.json", + "config_sha256": "02f32088faca67cd0b81860022c5c03861f41f772058b1c18e72b903439fee59", + "job_name": "pi-flash-dev34-custom-memory-heap-crash-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/dna-assembly", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dna-assembly.json", + "config_sha256": "1e21697089756a67a11683a935a00b1f4f0373c8d9e5977cd73ea9ad6754db86", + "job_name": "pi-flash-dev34-dna-assembly-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/llm-inference-batching-scheduler", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/llm-inference-batching-scheduler.json", + "config_sha256": "990ffdb0670445825f8ce37707ca3c18625e37608fcd22ebd8f9f8c30b90cbf6", + "job_name": "pi-flash-dev34-llm-inference-batching-scheduler-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/path-tracing", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/path-tracing.json", + "config_sha256": "e6aef64f083166c6eb4c506a1f7c1ea01d406f4c6e2a9dd87d5fbff5c24bd92f", + "job_name": "pi-flash-dev34-path-tracing-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/schemelike-metacircular-eval", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/schemelike-metacircular-eval.json", + "config_sha256": "8c25426015609bd39dc1002c9da30aca4c2979731701c2683a320dbeffc6eb20", + "job_name": "pi-flash-dev34-schemelike-metacircular-eval-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/caffe-cifar-10", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/caffe-cifar-10.json", + "config_sha256": "ef0fa08a4a2ffb6f7652dd1b918ab0a1e397e4082a96c8518cb64261e787791a", + "job_name": "pi-flash-dev34-caffe-cifar-10-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/circuit-fibsqrt", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/circuit-fibsqrt.json", + "config_sha256": "141731de25773e5e87c6c4157968f2526820430dac6d153411d961fdfbd017ca", + "job_name": "pi-flash-dev34-circuit-fibsqrt-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/mteb-leaderboard", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/mteb-leaderboard.json", + "config_sha256": "712201f8c47c3995c2e3f553b08c1b3883df7b4cde6078bd6fb450675e92c6ad", + "job_name": "pi-flash-dev34-mteb-leaderboard-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/regex-chess", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-chess.json", + "config_sha256": "f8402e08659eff542793bd3e9f95960f47f659265f1424ec22e2c19d32e1260b", + "job_name": "pi-flash-dev34-regex-chess-20261024" + }, + { + "arm": "pi", + "repeat": 1, + "task": "terminal-bench/build-pov-ray", + "config": "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/build-pov-ray.json", + "config_sha256": "2c77147b66971c37fbafa1597046389864cee4d3137940c9fd05f3f33dedafbf", + "job_name": "pi-flash-dev34-build-pov-ray-20261024" + } + ], + "additional_file_sha256": { + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/selection.json": "4572b1cc432a16b9c5d1dd554c8d2189ae80414f386a90bf03829bc4d0c3da85", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-distribution.json": "0e1e2cdbef4ede58b2beed0f4c3655099fb2d6f7e3f6d2ca369879fa24546eb1", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json": "8751fac554abb9f363b188c102045f38e36693d52956f984e0e427b31998b299", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json": "47a2c7370d7e5359c401a74efbc4fef63168229f54e164e88a7974e00af94ce1", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/prior-image-digests.json": "fd9abd24bcc397d172ac0ad4e3faa875a2da8c482b35651a2ab296e04c3adcc4", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/run_comparison.py": "b5f16a23b4495ab7e4a7028eb697649c0b8b22cbef6b03a4493437ba0f5204c7", + "benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/image-digests.json": "7f21855951ff5fff62cb16e686d93e4aae726be117dad049641c67956ab00d46" + }, + "model_policy": { + "model": "deepseek/deepseek-flash", + "api": "openai-completions", + "base_url": "https://api.deepseek.com", + "thinking": "high", + "model_config": "pi-models.json", + "model_metadata_max_tokens": 384000, + "pricing": "Peak-rate estimates from the registered Pi model config; not billing reconciliation. Official pricing also has off-peak discounts.", + "alias": "Official documentation checked 2026-10-24 maps deepseek-flash to DeepSeek-V4.1-Flash-1017; the alias is mutable." + }, + "budget_policy": "Original task-native agent timeouts, original resources, and 1800-second setup. Stock Pi loop and generation behavior: no injected 100-turn cap or 180-second finalization reminder. Model metadata maxTokens=384000 is preserved from the local provider configuration. This differs from the historical Nano baseline (100 replies, 65536 requested output tokens, native timeout minus 180 seconds for work), so this is an agent baseline rather than a matched-budget causal comparison. Temperature unset.", + "verifier_policy": "Original official tests and native per-attempt timeouts. Only verifier timeouts retry, at most three total attempts on the same output. No dependency-error recovery or whole-trial retry. Preserve setup errors and verifier errors separately from scored failures.", + "preflight_policy": "Fresh install-only checks for all 34 tasks before any model trial. QEMU has the registered qemu-alpine-ssh-v1 profile; the other 33 explicitly report not_configured. Binary installation success does not certify unsupported verifier dependencies.", + "queue_policy": "Ascending native agent timeout then task name; consecutive pair barriers; at most two task trials at once; exactly one fresh output per task.", + "information_policy": "All 34 tasks are the existing development pool with prior exposure. No outcome-based task replacement. Review only operational metadata while running; freeze all new outputs before inspecting new trajectories or hidden tests. Keep the remaining 55 tasks unopened.", + "status_at_registration": "not_started: insufficient free system disk; no installation preflight or model trial has started", + "runtime_environment": "Use UV_PROJECT_ENVIRONMENT=/tmp/nanopy-pi-dev34-harbor-venv. Do not reuse or modify another worktree virtualenv." +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/fix-code-vulnerability.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/fix-code-vulnerability.json new file mode 100644 index 0000000..94fed72 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/fix-code-vulnerability.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/git-multibranch.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/git-multibranch.json new file mode 100644 index 0000000..d241f16 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/git-multibranch.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/image-digests.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/image-digests.json new file mode 100644 index 0000000..c5661be --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/image-digests.json @@ -0,0 +1,312 @@ +{ + "observed_at": "2026-10-24T14:07:09.186521+00:00", + "images": [ + { + "task": "terminal-bench/adaptive-rejection-sampler", + "image": "alexgshaw/adaptive-rejection-sampler:20251031", + "image_id": "sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f", + "repo_digests": [ + "alexgshaw/adaptive-rejection-sampler@sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f" + ], + "size_bytes": 29731756 + }, + { + "task": "terminal-bench/build-cython-ext", + "image": "alexgshaw/build-cython-ext:20251031", + "image_id": "sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594", + "repo_digests": [ + "alexgshaw/build-cython-ext@sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594" + ], + "size_bytes": 303222492 + }, + { + "task": "terminal-bench/cancel-async-tasks", + "image": "alexgshaw/cancel-async-tasks:20251031", + "image_id": "sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d", + "repo_digests": [ + "alexgshaw/cancel-async-tasks@sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d" + ], + "size_bytes": 44171676 + }, + { + "task": "terminal-bench/cobol-modernization", + "image": "alexgshaw/cobol-modernization:20251031", + "image_id": "sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0", + "repo_digests": [ + "alexgshaw/cobol-modernization@sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0" + ], + "size_bytes": 162020401 + }, + { + "task": "terminal-bench/db-wal-recovery", + "image": "alexgshaw/db-wal-recovery:20251031", + "image_id": "sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "repo_digests": [ + "alexgshaw/db-wal-recovery@sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e" + ], + "size_bytes": 175276145 + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "image": "alexgshaw/fix-code-vulnerability:20251031", + "image_id": "sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0", + "repo_digests": [ + "alexgshaw/fix-code-vulnerability@sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0" + ], + "size_bytes": 190696453 + }, + { + "task": "terminal-bench/git-multibranch", + "image": "alexgshaw/git-multibranch:20251031", + "image_id": "sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "repo_digests": [ + "alexgshaw/git-multibranch@sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5" + ], + "size_bytes": 222134454 + }, + { + "task": "terminal-bench/kv-store-grpc", + "image": "alexgshaw/kv-store-grpc:20251031", + "image_id": "sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791", + "repo_digests": [ + "alexgshaw/kv-store-grpc@sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791" + ], + "size_bytes": 44171671 + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "image": "alexgshaw/log-summary-date-ranges:20251031", + "image_id": "sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51", + "repo_digests": [ + "alexgshaw/log-summary-date-ranges@sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51" + ], + "size_bytes": 44866158 + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "image": "alexgshaw/merge-diff-arc-agi-task:20251031", + "image_id": "sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4", + "repo_digests": [ + "alexgshaw/merge-diff-arc-agi-task@sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4" + ], + "size_bytes": 29728285 + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "image": "alexgshaw/model-extraction-relu-logits:20251031", + "image_id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "repo_digests": [ + "alexgshaw/model-extraction-relu-logits@sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce" + ], + "size_bytes": 83698971 + }, + { + "task": "terminal-bench/multi-source-data-merger", + "image": "alexgshaw/multi-source-data-merger:20251031", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "repo_digests": [ + "alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e" + ], + "size_bytes": 220880128 + }, + { + "task": "terminal-bench/nginx-request-logging", + "image": "alexgshaw/nginx-request-logging:20251031", + "image_id": "sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d", + "repo_digests": [ + "alexgshaw/nginx-request-logging@sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d" + ], + "size_bytes": 67285778 + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "image": "alexgshaw/openssl-selfsigned-cert:20251031", + "image_id": "sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12", + "repo_digests": [ + "alexgshaw/openssl-selfsigned-cert@sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12" + ], + "size_bytes": 44173158 + }, + { + "task": "terminal-bench/pypi-server", + "image": "alexgshaw/pypi-server:20251031", + "image_id": "sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276", + "repo_digests": [ + "alexgshaw/pypi-server@sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276" + ], + "size_bytes": 83726049 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "image": "alexgshaw/pytorch-model-cli:20251031", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "repo_digests": [ + "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67" + ], + "size_bytes": 44349611 + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "image": "alexgshaw/pytorch-model-recovery:20260430", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "repo_digests": [ + "alexgshaw/pytorch-model-recovery@sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e" + ], + "size_bytes": 269661811 + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "image": "alexgshaw/qemu-alpine-ssh:20251031", + "image_id": "sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964", + "repo_digests": [ + "alexgshaw/qemu-alpine-ssh@sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964" + ], + "size_bytes": 1308553944 + }, + { + "task": "terminal-bench/query-optimize", + "image": "alexgshaw/query-optimize:20251031", + "image_id": "sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "repo_digests": [ + "alexgshaw/query-optimize@sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14" + ], + "size_bytes": 120494963 + }, + { + "task": "terminal-bench/regex-log", + "image": "alexgshaw/regex-log:20251031", + "image_id": "sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9", + "repo_digests": [ + "alexgshaw/regex-log@sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9" + ], + "size_bytes": 29725514 + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "image": "alexgshaw/torch-pipeline-parallelism:20251031", + "image_id": "sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b", + "repo_digests": [ + "alexgshaw/torch-pipeline-parallelism@sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b" + ], + "size_bytes": 29725531 + }, + { + "task": "terminal-bench/torch-tensor-parallelism", + "image": "alexgshaw/torch-tensor-parallelism:20251031", + "image_id": "sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690", + "repo_digests": [ + "alexgshaw/torch-tensor-parallelism@sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690" + ], + "size_bytes": 29725529 + }, + { + "task": "terminal-bench/write-compressor", + "image": "alexgshaw/write-compressor:20251031", + "image_id": "sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1", + "repo_digests": [ + "alexgshaw/write-compressor@sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1" + ], + "size_bytes": 291927726 + }, + { + "task": "terminal-bench/constraints-scheduling", + "image": "alexgshaw/constraints-scheduling:20251031", + "image_id": "sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "repo_digests": [ + "alexgshaw/constraints-scheduling@sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0" + ], + "size_bytes": 49852952 + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "image": "alexgshaw/custom-memory-heap-crash:20251031", + "image_id": "sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557", + "repo_digests": [ + "alexgshaw/custom-memory-heap-crash@sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557" + ], + "size_bytes": 1267999268 + }, + { + "task": "terminal-bench/dna-assembly", + "image": "alexgshaw/dna-assembly:20251031", + "image_id": "sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569", + "repo_digests": [ + "alexgshaw/dna-assembly@sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569" + ], + "size_bytes": 29727501 + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "image": "alexgshaw/llm-inference-batching-scheduler:20251031", + "image_id": "sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774", + "repo_digests": [ + "alexgshaw/llm-inference-batching-scheduler@sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774" + ], + "size_bytes": 44188631 + }, + { + "task": "terminal-bench/path-tracing", + "image": "alexgshaw/path-tracing:20251031", + "image_id": "sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482", + "repo_digests": [ + "alexgshaw/path-tracing@sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482" + ], + "size_bytes": 399390468 + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "image": "alexgshaw/schemelike-metacircular-eval:20251031", + "image_id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "repo_digests": [ + "alexgshaw/schemelike-metacircular-eval@sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db" + ], + "size_bytes": 44188336 + }, + { + "task": "terminal-bench/caffe-cifar-10", + "image": "alexgshaw/caffe-cifar-10:20260403", + "image_id": "sha256:929a6d631b592c82e0d0f5d4a65cf03c946dc554cfc9bb09916e82076e7d3215", + "repo_digests": [ + "alexgshaw/caffe-cifar-10@sha256:929a6d631b592c82e0d0f5d4a65cf03c946dc554cfc9bb09916e82076e7d3215" + ], + "size_bytes": 213171982 + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "image": "alexgshaw/circuit-fibsqrt:20251031", + "image_id": "sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a", + "repo_digests": [ + "alexgshaw/circuit-fibsqrt@sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a" + ], + "size_bytes": 149136738 + }, + { + "task": "terminal-bench/mteb-leaderboard", + "image": "alexgshaw/mteb-leaderboard:20260430", + "image_id": "sha256:a8538f4882ff132e0a20a75be3fffc07164318a7425ad19a6b4bf5268107feb4", + "repo_digests": [ + "alexgshaw/mteb-leaderboard@sha256:a8538f4882ff132e0a20a75be3fffc07164318a7425ad19a6b4bf5268107feb4" + ], + "size_bytes": 663825427 + }, + { + "task": "terminal-bench/regex-chess", + "image": "alexgshaw/regex-chess:20251031", + "image_id": "sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c", + "repo_digests": [ + "alexgshaw/regex-chess@sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c" + ], + "size_bytes": 55625584 + }, + { + "task": "terminal-bench/build-pov-ray", + "image": "alexgshaw/build-pov-ray:20251031", + "image_id": "sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b", + "repo_digests": [ + "alexgshaw/build-pov-ray@sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b" + ], + "size_bytes": 163066497 + } + ], + "matches_prior_registration": true +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/install-only.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/install-only.json new file mode 100644 index 0000000..046cd04 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/install-only.json @@ -0,0 +1,165 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "install_only": true, + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12" + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf" + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1" + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3" + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f" + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67" + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d" + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c" + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302" + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c" + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78" + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a" + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb" + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684" + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/kv-store-grpc.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/kv-store-grpc.json new file mode 100644 index 0000000..b360136 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/kv-store-grpc.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/llm-inference-batching-scheduler.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/llm-inference-batching-scheduler.json new file mode 100644 index 0000000..f1a3025 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/llm-inference-batching-scheduler.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/log-summary-date-ranges.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/log-summary-date-ranges.json new file mode 100644 index 0000000..f85d2c0 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/log-summary-date-ranges.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/merge-diff-arc-agi-task.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/merge-diff-arc-agi-task.json new file mode 100644 index 0000000..8cf8e87 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/merge-diff-arc-agi-task.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/model-extraction-relu-logits.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/model-extraction-relu-logits.json new file mode 100644 index 0000000..4412126 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/model-extraction-relu-logits.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/mteb-leaderboard.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/mteb-leaderboard.json new file mode 100644 index 0000000..2132bee --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/mteb-leaderboard.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/multi-source-data-merger.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/multi-source-data-merger.json new file mode 100644 index 0000000..7d2144b --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/multi-source-data-merger.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/nginx-request-logging.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/nginx-request-logging.json new file mode 100644 index 0000000..49100e6 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/nginx-request-logging.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/openssl-selfsigned-cert.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/openssl-selfsigned-cert.json new file mode 100644 index 0000000..98b75b7 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/openssl-selfsigned-cert.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/path-tracing.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/path-tracing.json new file mode 100644 index 0000000..a0910e8 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/path-tracing.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-distribution.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-distribution.json new file mode 100644 index 0000000..bc28496 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-distribution.json @@ -0,0 +1,852 @@ +{ + "version": "1.4.0", + "source": "Existing local upstream binary distribution; no rebuild or source modification. Its registered file hashes pin the local bytes; an upstream archive checksum has not been independently verified.", + "original_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "files": { + "CHANGELOG.md": { + "sha256": "b0eac55f99b20254f81cf925a52f054ec87c1320454a3fd7ee624683e585add6", + "mode": 420 + }, + "README.md": { + "sha256": "fbc7abec7d0829ac8bc4b217e950794e1ba4d8d1b3e0850f970da318ad72e863", + "mode": 420 + }, + "assets/clankolas.png": { + "sha256": "169acd0dfe6fbb8d8742ed24a3fc654fd0b2e2d4223c733249c5493723f1b72d", + "mode": 420 + }, + "docs/cli-integration.md": { + "sha256": "edf8e723a70b4c3c40275db0391219e55d30597936212a9cfdef33963dd37f93", + "mode": 420 + }, + "docs/cli.md": { + "sha256": "c45e59f31574de71770a71c9a7c9b6ac03bf9cad4928aa31bb6614d661733679", + "mode": 420 + }, + "docs/codemode.md": { + "sha256": "3a96ee10da36395bbd22beb60ac029f8ca8a859f2fd3c3104cfc38f9d5a1f95a", + "mode": 420 + }, + "docs/compaction.md": { + "sha256": "b5b23d10bb86d8ab32b67f80a7e57febd953222c6da6fd422ae91d232ef28faf", + "mode": 420 + }, + "docs/configuration.md": { + "sha256": "fc48d1ce7d885c5253222415fb7a4b46d75f80e6431eb924174657ff775099eb", + "mode": 420 + }, + "docs/containerization.md": { + "sha256": "e309fcb9a0a7592c0dba000ef8ee90f2d7ef1e01e6a1ac2f0cd9760dd63dee0c", + "mode": 420 + }, + "docs/custom-provider.md": { + "sha256": "fe926ed6194756c85ab6b56b7bfd12e81abe385acf39e0bb2a470eb6c70b8c59", + "mode": 420 + }, + "docs/docs.json": { + "sha256": "fc0bccd6bf66716ad1067d4fbf7258f674d89c89a12cb0b2587a027b9d7fdc49", + "mode": 420 + }, + "docs/environment-variables.md": { + "sha256": "097f2101e285f51f509be0b176d458825b83b929a635ae29d2a43703168170dd", + "mode": 420 + }, + "docs/extensions.md": { + "sha256": "46fc0baeb00db8ca8db399e33e0f506e86e425693b291258357e8063e2f3353c", + "mode": 420 + }, + "docs/how-pi-works.md": { + "sha256": "7d43ab8e2b886cd1047dcee503588a1c828b92333dd4f177526b89e6db8aaa8b", + "mode": 420 + }, + "docs/images/doom-extension.png": { + "sha256": "c3b89b5e1fba711a5bcfa188e13d8695a84f7cdda12d9e127bb45b36d13cbb27", + "mode": 420 + }, + "docs/images/exy.png": { + "sha256": "a8f253347440aed01ff659f41f573d340bcfd6010d829726ae92fe18abf601de", + "mode": 420 + }, + "docs/images/interactive-mode.png": { + "sha256": "4078001e39978151bd66b1d0a20123e7ca2ec2d1b572dfd70f552be5e88c0692", + "mode": 420 + }, + "docs/images/tree-view.png": { + "sha256": "ca61e7bbf300dfb3840cc191dc66dc4e9f5166b2f1f4089e6fb13fd04a2c3c66", + "mode": 420 + }, + "docs/index.md": { + "sha256": "c107426e88eac570e9ecbea325cb45c2a6b778a324fd6e424bd032380a65614d", + "mode": 420 + }, + "docs/json.md": { + "sha256": "f103191816fdcf3627dfc98b89acad8a28eee737697130ddaeb2581131213543", + "mode": 420 + }, + "docs/keybindings.md": { + "sha256": "413aa1c5e95a934d216aa5aba20052fa2cf35940ba5f02d1f4b22ff53c6ec678", + "mode": 420 + }, + "docs/llama-cpp.md": { + "sha256": "dccba80951463657c57a831aa2da3a576b92c05455f9b73610b208c942dfc8f4", + "mode": 420 + }, + "docs/mcp.md": { + "sha256": "23b9cca35d1340e8adce6bed8b495da0ebb58931179a937db87d610a1b40b9bf", + "mode": 420 + }, + "docs/message-types.md": { + "sha256": "b832893fb762f87fdbbfd694241d484592942823791f3e81a6c4d915421ff5db", + "mode": 420 + }, + "docs/models-configuration.md": { + "sha256": "1eb60b0f1faad6f9fffd62ea257b545dbc983dbe4c5decc4803b1a241d604155", + "mode": 420 + }, + "docs/models.md": { + "sha256": "c9914f5411db5ddd158b5391ec1da76c1f65788cfaea3a90e1e1555451b3d12e", + "mode": 420 + }, + "docs/nvidia-rtx-llama-cpp.md": { + "sha256": "bcd12ebfd283d8f528cf206ee4ec74f981ab36d3604419acc434c595decbebdb", + "mode": 420 + }, + "docs/packages.md": { + "sha256": "5fe4a3009d7260fe1e2f54b64086fc3c8b4d678232bfa2dbc7e91fe588fc9ea5", + "mode": 420 + }, + "docs/prompt-templates.md": { + "sha256": "ad5113cce0d05ef0ac77b3c71f9ebe41ea68c072275f9273b9616e4924feda42", + "mode": 420 + }, + "docs/providers.md": { + "sha256": "bc8798e4cc3209e455e3dc50c6652ae73c057d40122a4a741d8b5b65a041c2f1", + "mode": 420 + }, + "docs/quickstart.md": { + "sha256": "10c69b394304f6e7d6409b6bde02b0165f5723ff4acfd49cfa8f7414f977266a", + "mode": 420 + }, + "docs/rpc-commands.md": { + "sha256": "4deeee367852f4633ef594bc079f9e9fb14640a4e3ba8a69b7c89b6f54740838", + "mode": 420 + }, + "docs/rpc-extension-ui.md": { + "sha256": "3c6f2c6edfc8b1ace6642343727aef97a7e9234c85509850921b52d6ff09a48e", + "mode": 420 + }, + "docs/rpc.md": { + "sha256": "37a03d180115460a0a275022f4aeabef1e5345458df6a9b6e8a6de18474de720", + "mode": 420 + }, + "docs/sdk.md": { + "sha256": "d9034ccbf2219a58712d8dd4fcdf47e687d77131e71dd5d119d8a71e9bd2a2e8", + "mode": 420 + }, + "docs/security.md": { + "sha256": "3b1e40a8d28b17c9eb7e496a7ba88fbe04665e3e6ae02873a58b943e325ad504", + "mode": 420 + }, + "docs/session-format.md": { + "sha256": "4f8fc42179b332bcf71fe4c85cc95d914853ff7486d68742954a45316b512a4a", + "mode": 420 + }, + "docs/sessions.md": { + "sha256": "152d6489f61a6de9349c8ceaade95d5498fbc74f21771f41ba8e87ce0e5d8936", + "mode": 420 + }, + "docs/settings.md": { + "sha256": "2915d26f91f4c46ab1b92e63cbc46b8d48888fa0e491168642a5a90f973bd814", + "mode": 420 + }, + "docs/shell-aliases.md": { + "sha256": "ebb55456870455b4da91cfd054ae2ac9b3a417e6ff48a009587c33e68ecd7807", + "mode": 420 + }, + "docs/skills.md": { + "sha256": "5b9805525b194474ef1fe5455a0917ef15ed7f710aa9bec2eee11879556469e5", + "mode": 420 + }, + "docs/slash-commands.md": { + "sha256": "d7e1db36e9186ff146a6c27de31fe169671f52cb6f7b93e1bf8d7fe76209a79c", + "mode": 420 + }, + "docs/terminal-setup.md": { + "sha256": "583381fb9dc466135a865ca406a3be299a7a8da2c9d810c5b73f401912a0a357", + "mode": 420 + }, + "docs/termux.md": { + "sha256": "1b0687157f2384385d07800ec23e6c225ef28fbf6648f55453cc7ce62f4c64ea", + "mode": 420 + }, + "docs/themes.md": { + "sha256": "6f11eb43ca6dad4c71bcfa2fc72ac13c013e3cb9383688b88923e0139b2d097c", + "mode": 420 + }, + "docs/tmux.md": { + "sha256": "9ec0abdbf74c57d51d4a93f308fb4aec4d8a3574a24cbfd9ba481d2e39f1273c", + "mode": 420 + }, + "docs/tui.md": { + "sha256": "20d3695ebd11e179a194f535f73870247544f655ebf22e42726c3be559c6d786", + "mode": 420 + }, + "docs/usage.md": { + "sha256": "6dfe7446c81d601d5ebef5769523115229f802da364fe90fcdf96e322ae9fbcd", + "mode": 420 + }, + "docs/virtual-models.md": { + "sha256": "5ab04dee3b3712b317522bb0784e28a3fef2e6b7fe1aeb1c1ca27778685a38f7", + "mode": 420 + }, + "docs/windows.md": { + "sha256": "e5a51792381a9eec859f034153c67f9ff52c36d3cff4c7a1faa0651f77f627c3", + "mode": 420 + }, + "examples/README.md": { + "sha256": "294836cfa47cd2cff1cab628533aefc5fe2d888621e43fe9f4101c09c2917a98", + "mode": 420 + }, + "examples/extensions/README.md": { + "sha256": "b51e1095d338ed8bc6cb0a40dca12da66ffd12d6f4ec7bfd57815fac84d87662", + "mode": 420 + }, + "examples/extensions/auto-commit-on-exit.ts": { + "sha256": "b89b5cb4c6f1f07b0b4291ab85a3c5f025d7d31ebb285821716502452b46af62", + "mode": 420 + }, + "examples/extensions/bash-spawn-hook.ts": { + "sha256": "c79c87653e20ac6bd9f4813aa4bb01a483b3637700fa2edda220703958532662", + "mode": 420 + }, + "examples/extensions/bookmark.ts": { + "sha256": "a71d9449d415ad75c9f3d0d13afde129b9bea2d695b889566db17ecfca256d79", + "mode": 420 + }, + "examples/extensions/border-status-editor.ts": { + "sha256": "ccbbfddd4ff402432293b7adb21f58e869f37a5d12d4da81cea10cc5fc2311cd", + "mode": 420 + }, + "examples/extensions/built-in-tool-renderer.ts": { + "sha256": "4bfd0953e3dbf96ee0cdb26989a60ec91d4850323852ef09eafe5c2efdf23c00", + "mode": 420 + }, + "examples/extensions/claude-rules.ts": { + "sha256": "8a935f14f62c1c1a3e937f5c912c2215e25aa5fa591716e23c86224ce1dc34cd", + "mode": 420 + }, + "examples/extensions/commands.ts": { + "sha256": "36716b53da169936c7e1360a4fde1e2fc0c3356a6505f177235f09c5538c6e4f", + "mode": 420 + }, + "examples/extensions/confirm-destructive.ts": { + "sha256": "55502f82109df07e922ef3ec41af1b1a5ed33d4d6d56076654ab2bc88e2e0888", + "mode": 420 + }, + "examples/extensions/custom-compaction.ts": { + "sha256": "ea797b3508fd8600526f381466e8b08bb84f3d234932a9d6ce512698527d0aa9", + "mode": 420 + }, + "examples/extensions/custom-footer.ts": { + "sha256": "1e5f17bec95a48d407fbcc092faf7e1f6021c30cee4a66ec09bc120412292cb2", + "mode": 420 + }, + "examples/extensions/custom-header.ts": { + "sha256": "5de09cf861d3965201991d6e837d6b68e8a2e80cb89279ee9177cc05d31f723d", + "mode": 420 + }, + "examples/extensions/custom-provider-anthropic/.gitignore": { + "sha256": "4d56952b0fb13bf8f9b6c13a6d4c34a075bac3af447636a1df4335d7576e2f97", + "mode": 420 + }, + "examples/extensions/custom-provider-anthropic/index.ts": { + "sha256": "52bb733e1a25d9e175a644e6a6e17bbf61071beecf0f2903869baa9579a05ab3", + "mode": 420 + }, + "examples/extensions/custom-provider-anthropic/package-lock.json": { + "sha256": "4c1ccae817de13a0d0eee610fb12ee699b3c1f97cf28cbe83ec50cab0f338a77", + "mode": 420 + }, + "examples/extensions/custom-provider-anthropic/package.json": { + "sha256": "b2ceede9b12cef79f37d02c8e850132afdab1b3316c16d622c0c8343244e85c3", + "mode": 420 + }, + "examples/extensions/custom-provider-gitlab-duo/.gitignore": { + "sha256": "4d56952b0fb13bf8f9b6c13a6d4c34a075bac3af447636a1df4335d7576e2f97", + "mode": 420 + }, + "examples/extensions/custom-provider-gitlab-duo/index.ts": { + "sha256": "0c4adac82eff25fef23223af77e11cd53535834fece9731009b125446e213158", + "mode": 420 + }, + "examples/extensions/custom-provider-gitlab-duo/package.json": { + "sha256": "4af767a958f402efc19fc6157c75c209edc45c8ab6103b60f13973369ba41990", + "mode": 420 + }, + "examples/extensions/custom-provider-gitlab-duo/test.ts": { + "sha256": "58dfa6fdb63521ccad551b7463ff56d4aecf735074934a8e9fc34b71e7083d00", + "mode": 420 + }, + "examples/extensions/debug-provider.ts": { + "sha256": "a915f86a26a0257537939701797b1f613072275e76bdd8a27dd941bbf519643a", + "mode": 420 + }, + "examples/extensions/dirty-repo-guard.ts": { + "sha256": "0037a69abdf4cf98181db799bef5bdee8981861315eb82ddab78126c2b0f13c4", + "mode": 420 + }, + "examples/extensions/doom-overlay/.gitignore": { + "sha256": "b26aa26693c610d46d9c8b387b28d20dc5196a73b979f5522e24dd689aab7bc8", + "mode": 420 + }, + "examples/extensions/doom-overlay/README.md": { + "sha256": "e17bc73b76c7977ffeef99120c7896ff992cf5938a60c4d7427a7b19fc180d97", + "mode": 420 + }, + "examples/extensions/doom-overlay/doom/build/doom.js": { + "sha256": "c3147cf4881c88b4210c017c26828f52aff83f0c4db27d7dad3fe6cda9ade1fb", + "mode": 420 + }, + "examples/extensions/doom-overlay/doom/build/doom.wasm": { + "sha256": "571d161956593508cf4ade732ae93753f00484bb526667a8676571cca14dec7d", + "mode": 493 + }, + "examples/extensions/doom-overlay/doom/build.sh": { + "sha256": "ab09c9b6dc98ea7c1fa8e74c13981ce69b698c01aec77309c5c7d40ad114d38c", + "mode": 493 + }, + "examples/extensions/doom-overlay/doom/doomgeneric_pi.c": { + "sha256": "98210909e1e7dee71cbbdba64f1db4bebd88b504e861ee36ea42093c59b251a1", + "mode": 420 + }, + "examples/extensions/doom-overlay/doom-component.ts": { + "sha256": "d95dc47c92db0fdc5b0e0b56a277c67f277012d3f7296df745d8a93f64d5a42e", + "mode": 420 + }, + "examples/extensions/doom-overlay/doom-engine.ts": { + "sha256": "706e43f70bd374b73928e5a44d196d1dd5c03d93979d08bdfc01d049b5083855", + "mode": 420 + }, + "examples/extensions/doom-overlay/doom-keys.ts": { + "sha256": "e8bd99a4ce90dd44ad120c58f3b2ba7bc0b0c662bcdc4ed4994d059130be9fdd", + "mode": 420 + }, + "examples/extensions/doom-overlay/index.ts": { + "sha256": "f89818dd75c7d8970458b1dbb9c5e85fea9b86534fd740c898c1f6f9c9f9ceba", + "mode": 420 + }, + "examples/extensions/doom-overlay/wad-finder.ts": { + "sha256": "20254888262081bf96932253bfd346971deefa48d8df011fd3baa7aa2d653b7d", + "mode": 420 + }, + "examples/extensions/dynamic-resources/SKILL.md": { + "sha256": "f8f34b0d81aa791a2d56501756fe0ace21fb78ca2754025d093f41d3b0ccaf69", + "mode": 420 + }, + "examples/extensions/dynamic-resources/dynamic.json": { + "sha256": "2bf0164779b4ea3c2572f33b1e979c427a4839193998fda19bfc5cba2a4692c5", + "mode": 420 + }, + "examples/extensions/dynamic-resources/dynamic.md": { + "sha256": "e61e2c971d02d628a58f7c36b9b37bd3e636e4ad60618a7f65e2b41410c38f35", + "mode": 420 + }, + "examples/extensions/dynamic-resources/index.ts": { + "sha256": "dfbc49418fe46ebc9696fa335709df2ed0d4c4703da8c1ad0eb317d11fce8433", + "mode": 420 + }, + "examples/extensions/dynamic-tools.ts": { + "sha256": "4394098c11f98ffc26e2487acda2e09b9467ef9ff5867def7273cb0c82a2b6c0", + "mode": 420 + }, + "examples/extensions/entry-renderer.ts": { + "sha256": "e4c01fcf382972d570ebf37d044e32065eae062fe70987b1b885d75d46aa0b11", + "mode": 420 + }, + "examples/extensions/event-bus.ts": { + "sha256": "1185942b7afd83692d9e8da029e8594d702157398c693ec8b3ddb02b0780828a", + "mode": 420 + }, + "examples/extensions/file-trigger.ts": { + "sha256": "6f5fc3dc8fa387d8ca6973786c521feb94da58838b0e171a881fb6b3a5243ac0", + "mode": 420 + }, + "examples/extensions/git-checkpoint.ts": { + "sha256": "b60c85f797091a1f21ccb2a998027c6229fe995bdd62dbb1cdcc22946768f40b", + "mode": 420 + }, + "examples/extensions/git-merge-and-resolve.ts": { + "sha256": "a3c077c87544122abe83b275f3d527f60c80c87f460ea2499d66bde9b4880133", + "mode": 420 + }, + "examples/extensions/github-issue-autocomplete.ts": { + "sha256": "4646ef360e57aa98c787ba21ca2f2e8acb3db30c4db0503e64cdbea4a4c2ac65", + "mode": 420 + }, + "examples/extensions/gondolin/.gitignore": { + "sha256": "4d56952b0fb13bf8f9b6c13a6d4c34a075bac3af447636a1df4335d7576e2f97", + "mode": 420 + }, + "examples/extensions/gondolin/index.ts": { + "sha256": "9a9897639de8127a2abe6bb0d00f0005cd9e9cf9d51fd15893a827e67326bf62", + "mode": 420 + }, + "examples/extensions/gondolin/package-lock.json": { + "sha256": "256d3e45b7c6d293bdfd88a3c4884d8bb38ecdbc4064c290255d2b4d193ea535", + "mode": 420 + }, + "examples/extensions/gondolin/package.json": { + "sha256": "010f1a949005f706239d788a2481c355028eb4c366b4ea16c87577e63caab905", + "mode": 420 + }, + "examples/extensions/handoff.ts": { + "sha256": "ea07b6f680357b677d0f43f0016961ee8cb1331d37ae113e9466a3acffb2e381", + "mode": 420 + }, + "examples/extensions/hello.ts": { + "sha256": "0aa4e9800c2526914d4c1edb00b2cfa9bd9dd6da5218289994cccd5f5bfa4934", + "mode": 420 + }, + "examples/extensions/hidden-thinking-label.ts": { + "sha256": "179dc3704b5c563b49a7f4577222bf9bbaa5becf98b85c75e56ab26c001c5b22", + "mode": 420 + }, + "examples/extensions/inline-bash.ts": { + "sha256": "69b9dfc34e7b1ada9afe10cda3384b0b1a6f3c3a47da5fb7d16f914399b023d4", + "mode": 420 + }, + "examples/extensions/input-transform-streaming.ts": { + "sha256": "5c251ebda6163c8279e3c6d5f0aee56421ed150255dbb954f08ca7d2280b3ad2", + "mode": 420 + }, + "examples/extensions/input-transform.ts": { + "sha256": "cf0f65d610631ca75d18aae1c8f1408139702f674cb2835f8c009b943776474b", + "mode": 420 + }, + "examples/extensions/interactive-shell.ts": { + "sha256": "36a7efb5b132e666c643129ce09c3f83f56aed9924563e6b7269e0e126611ecc", + "mode": 420 + }, + "examples/extensions/jev-router.ts": { + "sha256": "cf2c9ef4baf9233faa133e99255786b00518cb4977d068c811a11b3c96f0a2d6", + "mode": 420 + }, + "examples/extensions/mac-system-theme.ts": { + "sha256": "d7349bae999a798c71b283165e06d0142d41b0264bfdad11ddcb1ec8b62f8710", + "mode": 420 + }, + "examples/extensions/message-renderer.ts": { + "sha256": "bedc198df5215c0a758e0743224599003630741a4ff72d4327837ce309e5ec8c", + "mode": 420 + }, + "examples/extensions/minimal-mode.ts": { + "sha256": "040cbc7619b2e903888c8626e2fa55ca5e3f820535dfd1aadcb2060e156a8770", + "mode": 420 + }, + "examples/extensions/modal-editor.ts": { + "sha256": "4742283930949fad1920359540aa9d42c833d2d24eaa49564bfc08f90e328b0d", + "mode": 420 + }, + "examples/extensions/model-status.ts": { + "sha256": "b6c073c0bdf26ac17e2fed01d02b6447fd4e2763b7b6ca2f2b493574ced25247", + "mode": 420 + }, + "examples/extensions/notify.ts": { + "sha256": "70e4333e09ce00d546c116fd2e918abf7616c70b5da6e88ba8ee21a326afd483", + "mode": 420 + }, + "examples/extensions/overlay-qa-tests.ts": { + "sha256": "4b1778e9e247c9eb24fe48e13fff0272f6f78b8102b4ed085d03598c3b98773f", + "mode": 420 + }, + "examples/extensions/overlay-test.ts": { + "sha256": "801b7c2eeec57c3569187b4b4d851f2159223ec6e9772e406ec65ed728ca6db9", + "mode": 420 + }, + "examples/extensions/permission-gate.ts": { + "sha256": "e32086ba9dd9b4ba0fd308b65241d9affa0b5ecd3f90ef7ea1b9f2b30cab8cc1", + "mode": 420 + }, + "examples/extensions/pirate.ts": { + "sha256": "dd6ce684bbe7630e4a749fe133ef8b8b875adbc7e31284b17fe0b8157d51e013", + "mode": 420 + }, + "examples/extensions/plan-mode/README.md": { + "sha256": "1ff8d611b8fbafef4ca1e3dbc72136fbcc166461b69f859b5e961b15147bff39", + "mode": 420 + }, + "examples/extensions/plan-mode/index.ts": { + "sha256": "bef2bf0712aea7b7d0f73d174e24ac5309679e6cc143f7bcbf1a9e0121222af8", + "mode": 420 + }, + "examples/extensions/plan-mode/utils.ts": { + "sha256": "12ea28d57cc55e68e8d52a2f1d16df8b2e7f2749cd694d41453028002a2bdad9", + "mode": 420 + }, + "examples/extensions/preset.ts": { + "sha256": "7942b7f933f97bcc29024a87f8526969413ba6f87b0bec537648a2926a0a4e29", + "mode": 420 + }, + "examples/extensions/project-trust.ts": { + "sha256": "78781c1f2a1613cf31c02a29ff6a6abd86b60bdf34343465e46cd85c01491578", + "mode": 420 + }, + "examples/extensions/prompt-customizer.ts": { + "sha256": "429546f46795bccda4078deace5729b8230f6103807d424eaf71817c6e07da0a", + "mode": 420 + }, + "examples/extensions/protected-paths.ts": { + "sha256": "9ca66b1f3b1b9a61cc5ddc660f92cca2ea66c4264a1bdb6b00e0da5844399fe2", + "mode": 420 + }, + "examples/extensions/provider-payload.ts": { + "sha256": "e0016077798098f528b1129954c30a28809a1513924d9c2e0efda9328d503a49", + "mode": 420 + }, + "examples/extensions/qna.ts": { + "sha256": "a639a65a486fd05a42afcf14aa59dc32e505e41ada43869167ddfd789a3c7da6", + "mode": 420 + }, + "examples/extensions/question.ts": { + "sha256": "a1625d2c72a65d859154a7e66bef222d48ca9e3654a8bb910829ad350378a4c4", + "mode": 420 + }, + "examples/extensions/questionnaire.ts": { + "sha256": "3f1f719d81152ff013f5d36a97cc3976388d43f2d89bff0f19827d3443d903e6", + "mode": 420 + }, + "examples/extensions/rainbow-editor.ts": { + "sha256": "f95f17ba8a0376180bd23510c20151e83aadc74142e989f14227bf5c4196883f", + "mode": 420 + }, + "examples/extensions/reload-runtime.ts": { + "sha256": "911f26b872860a551572fa1d94d251ddd476d4ea9483b6052e9658d2784b38fc", + "mode": 420 + }, + "examples/extensions/rpc-demo.ts": { + "sha256": "3d54a396eee12eb04ea12d0ea03716a6edd3e8743502b6a529e3b8d429ae640d", + "mode": 420 + }, + "examples/extensions/sandbox/.gitignore": { + "sha256": "16d30e4462189fb14dd611bdb708c510630c576a1f35b9383e89a4352da36c97", + "mode": 420 + }, + "examples/extensions/sandbox/index.ts": { + "sha256": "b5e527aac27254002f27b8bb9de7bc62e242e39eeb7c963683096039e0aa173e", + "mode": 420 + }, + "examples/extensions/sandbox/package-lock.json": { + "sha256": "cd4e34dbd8a9d4b0efdb58becb6c4201eb2ad9e44228b9a272b6853bfb54539d", + "mode": 420 + }, + "examples/extensions/sandbox/package.json": { + "sha256": "4b4bfe0ebeecf2af747a3e023a5edc143cba9c36137d10051e1124d1c1ad9732", + "mode": 420 + }, + "examples/extensions/send-user-message.ts": { + "sha256": "9261e433c782f354530105e1c9723a19798868f72fd1f5d68af3a4624ad7e446", + "mode": 420 + }, + "examples/extensions/session-name.ts": { + "sha256": "a0b9fd94e2cc827931150cc409238b473da635ab1a1cddce633cae5937a3ae24", + "mode": 420 + }, + "examples/extensions/shutdown-command.ts": { + "sha256": "68dc7f9561aa9b1c0f39cae19c0f41d56171a99875972e5998de3caa75f96e09", + "mode": 420 + }, + "examples/extensions/snake.ts": { + "sha256": "3ef506ab52927e866fad096f83b7cbfd0eb50e1504dcabf1b1eb28a06745c937", + "mode": 420 + }, + "examples/extensions/space-invaders.ts": { + "sha256": "43281ae25bea645d44db01234f24df80e2f866771d02de0b416ab32fb3883a1f", + "mode": 420 + }, + "examples/extensions/ssh.ts": { + "sha256": "c03bdef919457013e3798d10250f34255742fd3dc79885ac3d321b34f6a1149d", + "mode": 420 + }, + "examples/extensions/status-line.ts": { + "sha256": "26e27b1158e980551f63f46e55dad055ac613779c2b8c39df4cddc11be26103d", + "mode": 420 + }, + "examples/extensions/structured-output.ts": { + "sha256": "72a3a665ce3de1419c837db6eb8a0e7a890310c6e02e4d59c454e5c9514c15ec", + "mode": 420 + }, + "examples/extensions/subagent/README.md": { + "sha256": "6a888330bf974657751e09f5d375a9039872d59d6606fc217c5105c7eca51b1a", + "mode": 420 + }, + "examples/extensions/subagent/agents/planner.md": { + "sha256": "452f21f65957753cec3b0772004209c108071be168e627fffd7191234c0ce64e", + "mode": 420 + }, + "examples/extensions/subagent/agents/reviewer.md": { + "sha256": "b5c3f13b67207e8e32de631f8cd52a373d9b956028e2f8c1fd8b737b5f239265", + "mode": 420 + }, + "examples/extensions/subagent/agents/scout.md": { + "sha256": "3b84e8e01c6e05c8dfbff1c845c0c6371f4f16350735d4d1dbd02bfa98b1d4e6", + "mode": 420 + }, + "examples/extensions/subagent/agents/worker.md": { + "sha256": "ca2b2298fcbd8bfafcfe61977dc8075001a64b9c4d8c289c3b5eaa896efe6238", + "mode": 420 + }, + "examples/extensions/subagent/agents.ts": { + "sha256": "cc85ecd33b89aa575bea3d2a9a9da7654a6305da4de02e5041aa5342a1de29b4", + "mode": 420 + }, + "examples/extensions/subagent/index.ts": { + "sha256": "1d11858d163e2a3a258af51c1923fc55ad825813acd6224462fd25ad61f5401b", + "mode": 420 + }, + "examples/extensions/subagent/prompts/implement-and-review.md": { + "sha256": "019ba4bf42d596ea03fc4564be5b2cc55a06227c95db19a4d5a5bb96697a75f9", + "mode": 420 + }, + "examples/extensions/subagent/prompts/implement.md": { + "sha256": "65462de33c97ac1372cfed270dad9af164605b05028c069dd4c013ff9792675b", + "mode": 420 + }, + "examples/extensions/subagent/prompts/scout-and-plan.md": { + "sha256": "9506399391e0d2c71826c45f26cc332175dd964465f99a8bb880e109f8ed89f4", + "mode": 420 + }, + "examples/extensions/summarize.ts": { + "sha256": "9c75e838071773d5bacce27ac8c5305160c56958018884f960c3db96b3d09368", + "mode": 420 + }, + "examples/extensions/system-prompt-header.ts": { + "sha256": "aa97f1add4fe347b3fc37cd3709950a105b0db5d6cdaf19325b58a9fbfd05223", + "mode": 420 + }, + "examples/extensions/tic-tac-toe.ts": { + "sha256": "479597f4e76ef1eae3dae71ec9d5c26978c5b9221f950e3e134078c5b532314b", + "mode": 420 + }, + "examples/extensions/timed-confirm.ts": { + "sha256": "c35304599e6f07238ef5472efe9c3f2fef2a022f801750ed792ef9061c7de10f", + "mode": 420 + }, + "examples/extensions/titlebar-spinner.ts": { + "sha256": "ec56e7edf954966b5991bcad63141be1a09d17753cdd0c59efca1b9e649dadf2", + "mode": 420 + }, + "examples/extensions/todo.ts": { + "sha256": "e46824d00217e25242c186d41837cc84ca81b23f978500323448502a9a424ee2", + "mode": 420 + }, + "examples/extensions/tool-override.ts": { + "sha256": "5d3cc796311a16132db810fcd9e437c16abd46978b723b9120734e57cc545726", + "mode": 420 + }, + "examples/extensions/tools.ts": { + "sha256": "5eaebc40cffe885fee25c95e3d8aec79cfbd595b6eac8ac5a6de6c4afec3d5d8", + "mode": 420 + }, + "examples/extensions/trigger-compact.ts": { + "sha256": "92c3b831f9f7bf01a3e0a2df11f4e404f5ac5843000b94e8340a5d78f3cb3add", + "mode": 420 + }, + "examples/extensions/truncated-tool.ts": { + "sha256": "4ed5fbeb6da53eab8d1e04721d77961b8d1b3bf5e11d1512a17fc5a3f58f9955", + "mode": 420 + }, + "examples/extensions/widget-placement.ts": { + "sha256": "ffabd33cf58efb908de2025aac4cb41daf13dbeee24b5ad8dcd0361b43eff6d7", + "mode": 420 + }, + "examples/extensions/with-deps/.gitignore": { + "sha256": "4d56952b0fb13bf8f9b6c13a6d4c34a075bac3af447636a1df4335d7576e2f97", + "mode": 420 + }, + "examples/extensions/with-deps/index.ts": { + "sha256": "8d4dbc7028c891eac7f0386d25eecea3e9ac0de7fd841b5bb08137e2ec5ab443", + "mode": 420 + }, + "examples/extensions/with-deps/package-lock.json": { + "sha256": "a4863bc4e63ff31dd43a6b469835dd0057f8792d940af0a349ec35187ad03663", + "mode": 420 + }, + "examples/extensions/with-deps/package.json": { + "sha256": "1cedd74eb4891d548ff4ae0b90adb3ce73086fa21b11fe489e0d8b1ead1c5267", + "mode": 420 + }, + "examples/extensions/working-indicator.ts": { + "sha256": "e88295d765b78fd6aac0e93db92b183ee9b7103dc21577e3c5a42665dfad032c", + "mode": 420 + }, + "examples/extensions/working-message-test.ts": { + "sha256": "91ee09378f13cce142a4314bec1bfe9784e10484c33463c4fe3139c391c117a0", + "mode": 420 + }, + "examples/rpc-client.ts": { + "sha256": "b46b19c291fbcb59f8f9c539c2dbe12a43eae25e4d7528084844bbb79d27b6f7", + "mode": 420 + }, + "examples/rpc-extension-ui.ts": { + "sha256": "09bc33c28fdc9032e3752a487d659ac21c6776c6f589c5b1ae0967ef4ee5f127", + "mode": 420 + }, + "examples/sdk/01-minimal.ts": { + "sha256": "ffe5215e36f750f5ca7146cc7dfa5be36461292036d55b8db652b4d26331f351", + "mode": 420 + }, + "examples/sdk/02-custom-model.ts": { + "sha256": "dccc8dbdb1742ea00beeed87d302b8a23ac7f7e36a0b6a4c43b283c21be0ee55", + "mode": 420 + }, + "examples/sdk/03-custom-prompt.ts": { + "sha256": "d6db1f36a8e04466d9cbcc24ffb6fe0697858aa6c0e5013d5d90f12798314ef5", + "mode": 420 + }, + "examples/sdk/04-skills.ts": { + "sha256": "4cbf189bf88d14155281349d4d39cc236066d98095aae2d184d4ba7267ab5e42", + "mode": 420 + }, + "examples/sdk/05-tools.ts": { + "sha256": "5df5f9577b343d22d072d839341916536299c8191dc66c6c801a8bbddda23544", + "mode": 420 + }, + "examples/sdk/06-extensions.ts": { + "sha256": "8e01ee409200b6516e9a6a0d9f3b3f61d2ad78aaaa18bd2ef9a8f77820d149b2", + "mode": 420 + }, + "examples/sdk/07-context-files.ts": { + "sha256": "37bd18a284e961903d646e0cd1abdeda72cd968dbe4f40892e8d345a66916b36", + "mode": 420 + }, + "examples/sdk/08-prompt-templates.ts": { + "sha256": "a1b314988f1f5b2e61c80c19c79b68630a2f3c443008a200376286710a2744ed", + "mode": 420 + }, + "examples/sdk/09-api-keys-and-oauth.ts": { + "sha256": "d1d2b1bd4e57a3b2ab6f910068e8382759d12a94fbecf1332e96a9ebe4683eb4", + "mode": 420 + }, + "examples/sdk/10-settings.ts": { + "sha256": "558f82b9d0655935e6a28ec09f6c35f0f9593f8b034f25d45400c02bc77a778a", + "mode": 420 + }, + "examples/sdk/11-sessions.ts": { + "sha256": "538c7028bf6823c84be930eb6745731f2c87ba05a45df2d86d93e66f01674576", + "mode": 420 + }, + "examples/sdk/12-full-control.ts": { + "sha256": "3cf794d470ddbde4504ae965d24a23c42378d1ac86033522e46907ac05518343", + "mode": 420 + }, + "examples/sdk/13-session-runtime.ts": { + "sha256": "0e9f408720a561a62b2630eccb18ece49f9dba567dc65494757da0a91608827a", + "mode": 420 + }, + "examples/sdk/14-codemode-mcp.ts": { + "sha256": "02ae7714e22753a76b0f44033366d43529185b325b92d47bb5d1d0eb5ae34f5e", + "mode": 420 + }, + "examples/sdk/README.md": { + "sha256": "c1f9d8f58c300195d03a420222d2d5370d2884a18a4190a1d8c8ea9371c016ff", + "mode": 420 + }, + "export-html/ansi-to-html.d.ts": { + "sha256": "57d23f45aef05ba2b1a52a8e6b0e1f1ba51b55ab0b54e8d42938a438b088977a", + "mode": 420 + }, + "export-html/ansi-to-html.d.ts.map": { + "sha256": "2847dc4f48c9ec5c8cd7aeacf1a6083ab4fc44f4b7e6667547b1549215c68576", + "mode": 420 + }, + "export-html/ansi-to-html.js": { + "sha256": "44e5fd3585104e5e8cc0c7bea91a10efa7cb7e3b5bc5e2fb9090d5be4419df14", + "mode": 420 + }, + "export-html/ansi-to-html.js.map": { + "sha256": "d24d23d27ad31a21a4fb133c6e7ba2dc2c876f28656ff1254b1af3da234dc51c", + "mode": 420 + }, + "export-html/index.d.ts": { + "sha256": "cd83b5d85e893b8ca18742b28038883dd7cf06c98a203e00d74040f78a6a656d", + "mode": 420 + }, + "export-html/index.d.ts.map": { + "sha256": "1c31d2cbab7deb811274f473316e1fed48bc2a93707f2f964db86ba2f200d522", + "mode": 420 + }, + "export-html/index.js": { + "sha256": "9b7a893c46341d2557ba79dd88307797054577c8789665281d1dfb4134d347a3", + "mode": 420 + }, + "export-html/index.js.map": { + "sha256": "f2bf1920c337f74e83454665e5a1af9621cd9e67fbcca9aade7231eb624bf9a7", + "mode": 420 + }, + "export-html/template.css": { + "sha256": "8ee19851f8e583277ed396cbb76496687aa556707fdfc1f78d1118bff87c740a", + "mode": 420 + }, + "export-html/template.html": { + "sha256": "47d208235b066c856dae69c60281a3b99d164b99be9b81153d880375c9327105", + "mode": 420 + }, + "export-html/template.js": { + "sha256": "946bd9e3e8255c1ebb40f806def18007aa9397816291a7bf2ac0e4e99cec07a4", + "mode": 420 + }, + "export-html/tool-renderer.d.ts": { + "sha256": "ea982244e87c3568ad7bbb28427b83a19506136b5a498c7b496454338483ffd4", + "mode": 420 + }, + "export-html/tool-renderer.d.ts.map": { + "sha256": "2fab18f196e020f0b896b27db7cf790c233f0d0af55de04194e60b5bff137fce", + "mode": 420 + }, + "export-html/tool-renderer.js": { + "sha256": "8d3bd1f1b068996fe8f7d003bdb6623284043e1393a2db92ff45aeb9d6c33b9a", + "mode": 420 + }, + "export-html/tool-renderer.js.map": { + "sha256": "7bead30f87d1b345c70c66953a517fb51d1e1092089158ad5e92280cc97f6ea8", + "mode": 420 + }, + "export-html/vendor/highlight.min.js": { + "sha256": "837a6fa5b0c736b52bbde2b2b6190f305da3fc9ed41681db5321507057b5c846", + "mode": 420 + }, + "export-html/vendor/marked-cjk-friendly.min.js": { + "sha256": "93d2c938754cb24eef5b9ffecc268a98ef6beeb75dad614b0e874176aeeba565", + "mode": 420 + }, + "export-html/vendor/marked.min.js": { + "sha256": "438eedfcf932a414d0d0bfeea32dc365c063563b8ca713b4687fc8f8b501e5e4", + "mode": 420 + }, + "native/linux/prebuilds/linux-x64/linux-platform-x11.node": { + "sha256": "522d2550f7fa5d0d5bd204f45983f786ce2be9cf80f27f00179640d31d0689d6", + "mode": 493 + }, + "package.json": { + "sha256": "df6883685382d6a914e5797232744c26579fe7db5576ea0e9c859d018ca2d54e", + "mode": 420 + }, + "photon_rs_bg.wasm": { + "sha256": "10468181565c56004c867f3a4af96f89a0ef5a63a72f2b5fb12c1f1992a3615c", + "mode": 420 + }, + "pi": { + "sha256": "0d7611d0b86d38ec8f65dda8e8bee9c5e907c7bae381e9cfff9161b21e8087d0", + "mode": 493 + }, + "theme/dark.json": { + "sha256": "e1f96982f8cb7d2a48eca7db03c9cb282cb1e14a243680cd855638ecd6a251b6", + "mode": 420 + }, + "theme/light.json": { + "sha256": "2dee6a4be7f765e7fd7728166b7ebf8c29e4ce313f8ec6b38ae50699e3abc2f3", + "mode": 420 + } + } +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json new file mode 100644 index 0000000..32fd1b7 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json @@ -0,0 +1,42 @@ +{ + "providers": { + "deepseek": { + "api": "openai-completions", + "baseUrl": "https://api.deepseek.com", + "models": [ + { + "id": "deepseek-flash", + "name": "DeepSeek V4.1 Flash", + "reasoning": true, + "input": [ + "text", + "image" + ], + "thinkingLevelMap": { + "minimal": null, + "low": "low", + "medium": null, + "high": "high", + "max": "max" + }, + "contextWindow": 1000000, + "maxTokens": 384000, + "cost": { + "input": 0.3, + "output": 1.2, + "cacheRead": 0.006, + "cacheWrite": 0 + }, + "compat": { + "supportsStore": false, + "supportsDeveloperRole": false, + "maxTokensField": "max_tokens", + "requiresReasoningContentOnAssistantMessages": true, + "thinkingFormat": "deepseek" + } + } + ], + "apiKey": "${DEEPSEEK_API_KEY}" + } + } +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/prior-image-digests.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/prior-image-digests.json new file mode 100644 index 0000000..722c18c --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/prior-image-digests.json @@ -0,0 +1,311 @@ +{ + "observed_at": "2026-10-19T13:14:02.324661+00:00", + "images": [ + { + "task": "terminal-bench/adaptive-rejection-sampler", + "image": "alexgshaw/adaptive-rejection-sampler:20251031", + "image_id": "sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f", + "repo_digests": [ + "alexgshaw/adaptive-rejection-sampler@sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f" + ], + "size_bytes": 29731756 + }, + { + "task": "terminal-bench/build-cython-ext", + "image": "alexgshaw/build-cython-ext:20251031", + "image_id": "sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594", + "repo_digests": [ + "alexgshaw/build-cython-ext@sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594" + ], + "size_bytes": 303222492 + }, + { + "task": "terminal-bench/cancel-async-tasks", + "image": "alexgshaw/cancel-async-tasks:20251031", + "image_id": "sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d", + "repo_digests": [ + "alexgshaw/cancel-async-tasks@sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d" + ], + "size_bytes": 44171676 + }, + { + "task": "terminal-bench/cobol-modernization", + "image": "alexgshaw/cobol-modernization:20251031", + "image_id": "sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0", + "repo_digests": [ + "alexgshaw/cobol-modernization@sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0" + ], + "size_bytes": 162020401 + }, + { + "task": "terminal-bench/db-wal-recovery", + "image": "alexgshaw/db-wal-recovery:20251031", + "image_id": "sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "repo_digests": [ + "alexgshaw/db-wal-recovery@sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e" + ], + "size_bytes": 175276145 + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "image": "alexgshaw/fix-code-vulnerability:20251031", + "image_id": "sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0", + "repo_digests": [ + "alexgshaw/fix-code-vulnerability@sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0" + ], + "size_bytes": 190696453 + }, + { + "task": "terminal-bench/git-multibranch", + "image": "alexgshaw/git-multibranch:20251031", + "image_id": "sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "repo_digests": [ + "alexgshaw/git-multibranch@sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5" + ], + "size_bytes": 222134454 + }, + { + "task": "terminal-bench/kv-store-grpc", + "image": "alexgshaw/kv-store-grpc:20251031", + "image_id": "sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791", + "repo_digests": [ + "alexgshaw/kv-store-grpc@sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791" + ], + "size_bytes": 44171671 + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "image": "alexgshaw/log-summary-date-ranges:20251031", + "image_id": "sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51", + "repo_digests": [ + "alexgshaw/log-summary-date-ranges@sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51" + ], + "size_bytes": 44866158 + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "image": "alexgshaw/merge-diff-arc-agi-task:20251031", + "image_id": "sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4", + "repo_digests": [ + "alexgshaw/merge-diff-arc-agi-task@sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4" + ], + "size_bytes": 29728285 + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "image": "alexgshaw/model-extraction-relu-logits:20251031", + "image_id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "repo_digests": [ + "alexgshaw/model-extraction-relu-logits@sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce" + ], + "size_bytes": 83698971 + }, + { + "task": "terminal-bench/multi-source-data-merger", + "image": "alexgshaw/multi-source-data-merger:20251031", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "repo_digests": [ + "alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e" + ], + "size_bytes": 220880128 + }, + { + "task": "terminal-bench/nginx-request-logging", + "image": "alexgshaw/nginx-request-logging:20251031", + "image_id": "sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d", + "repo_digests": [ + "alexgshaw/nginx-request-logging@sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d" + ], + "size_bytes": 67285778 + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "image": "alexgshaw/openssl-selfsigned-cert:20251031", + "image_id": "sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12", + "repo_digests": [ + "alexgshaw/openssl-selfsigned-cert@sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12" + ], + "size_bytes": 44173158 + }, + { + "task": "terminal-bench/pypi-server", + "image": "alexgshaw/pypi-server:20251031", + "image_id": "sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276", + "repo_digests": [ + "alexgshaw/pypi-server@sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276" + ], + "size_bytes": 83726049 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "image": "alexgshaw/pytorch-model-cli:20251031", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "repo_digests": [ + "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67" + ], + "size_bytes": 44349611 + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "image": "alexgshaw/pytorch-model-recovery:20260430", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "repo_digests": [ + "alexgshaw/pytorch-model-recovery@sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e" + ], + "size_bytes": 269661811 + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "image": "alexgshaw/qemu-alpine-ssh:20251031", + "image_id": "sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964", + "repo_digests": [ + "alexgshaw/qemu-alpine-ssh@sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964" + ], + "size_bytes": 1308553944 + }, + { + "task": "terminal-bench/query-optimize", + "image": "alexgshaw/query-optimize:20251031", + "image_id": "sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "repo_digests": [ + "alexgshaw/query-optimize@sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14" + ], + "size_bytes": 120494963 + }, + { + "task": "terminal-bench/regex-log", + "image": "alexgshaw/regex-log:20251031", + "image_id": "sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9", + "repo_digests": [ + "alexgshaw/regex-log@sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9" + ], + "size_bytes": 29725514 + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "image": "alexgshaw/torch-pipeline-parallelism:20251031", + "image_id": "sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b", + "repo_digests": [ + "alexgshaw/torch-pipeline-parallelism@sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b" + ], + "size_bytes": 29725531 + }, + { + "task": "terminal-bench/torch-tensor-parallelism", + "image": "alexgshaw/torch-tensor-parallelism:20251031", + "image_id": "sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690", + "repo_digests": [ + "alexgshaw/torch-tensor-parallelism@sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690" + ], + "size_bytes": 29725529 + }, + { + "task": "terminal-bench/write-compressor", + "image": "alexgshaw/write-compressor:20251031", + "image_id": "sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1", + "repo_digests": [ + "alexgshaw/write-compressor@sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1" + ], + "size_bytes": 291927726 + }, + { + "task": "terminal-bench/constraints-scheduling", + "image": "alexgshaw/constraints-scheduling:20251031", + "image_id": "sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "repo_digests": [ + "alexgshaw/constraints-scheduling@sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0" + ], + "size_bytes": 49852952 + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "image": "alexgshaw/custom-memory-heap-crash:20251031", + "image_id": "sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557", + "repo_digests": [ + "alexgshaw/custom-memory-heap-crash@sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557" + ], + "size_bytes": 1267999268 + }, + { + "task": "terminal-bench/dna-assembly", + "image": "alexgshaw/dna-assembly:20251031", + "image_id": "sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569", + "repo_digests": [ + "alexgshaw/dna-assembly@sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569" + ], + "size_bytes": 29727501 + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "image": "alexgshaw/llm-inference-batching-scheduler:20251031", + "image_id": "sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774", + "repo_digests": [ + "alexgshaw/llm-inference-batching-scheduler@sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774" + ], + "size_bytes": 44188631 + }, + { + "task": "terminal-bench/path-tracing", + "image": "alexgshaw/path-tracing:20251031", + "image_id": "sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482", + "repo_digests": [ + "alexgshaw/path-tracing@sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482" + ], + "size_bytes": 399390468 + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "image": "alexgshaw/schemelike-metacircular-eval:20251031", + "image_id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "repo_digests": [ + "alexgshaw/schemelike-metacircular-eval@sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db" + ], + "size_bytes": 44188336 + }, + { + "task": "terminal-bench/caffe-cifar-10", + "image": "alexgshaw/caffe-cifar-10:20260403", + "image_id": "sha256:929a6d631b592c82e0d0f5d4a65cf03c946dc554cfc9bb09916e82076e7d3215", + "repo_digests": [ + "alexgshaw/caffe-cifar-10@sha256:929a6d631b592c82e0d0f5d4a65cf03c946dc554cfc9bb09916e82076e7d3215" + ], + "size_bytes": 213171982 + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "image": "alexgshaw/circuit-fibsqrt:20251031", + "image_id": "sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a", + "repo_digests": [ + "alexgshaw/circuit-fibsqrt@sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a" + ], + "size_bytes": 149136738 + }, + { + "task": "terminal-bench/mteb-leaderboard", + "image": "alexgshaw/mteb-leaderboard:20260430", + "image_id": "sha256:a8538f4882ff132e0a20a75be3fffc07164318a7425ad19a6b4bf5268107feb4", + "repo_digests": [ + "alexgshaw/mteb-leaderboard@sha256:a8538f4882ff132e0a20a75be3fffc07164318a7425ad19a6b4bf5268107feb4" + ], + "size_bytes": 663825427 + }, + { + "task": "terminal-bench/regex-chess", + "image": "alexgshaw/regex-chess:20251031", + "image_id": "sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c", + "repo_digests": [ + "alexgshaw/regex-chess@sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c" + ], + "size_bytes": 55625584 + }, + { + "task": "terminal-bench/build-pov-ray", + "image": "alexgshaw/build-pov-ray:20251031", + "image_id": "sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b", + "repo_digests": [ + "alexgshaw/build-pov-ray@sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b" + ], + "size_bytes": 163066497 + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pypi-server.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pypi-server.json new file mode 100644 index 0000000..4c292ab --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pypi-server.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-cli.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-cli.json new file mode 100644 index 0000000..1490c16 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-cli.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-recovery.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-recovery.json new file mode 100644 index 0000000..28cdb6b --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pytorch-model-recovery.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/qemu-alpine-ssh.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/qemu-alpine-ssh.json new file mode 100644 index 0000000..65b8e75 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/qemu-alpine-ssh.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/query-optimize.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/query-optimize.json new file mode 100644 index 0000000..1e7a3f7 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/query-optimize.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-chess.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-chess.json new file mode 100644 index 0000000..5bd156e --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-chess.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-log.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-log.json new file mode 100644 index 0000000..3281d02 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/regex-log.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/schemelike-metacircular-eval.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/schemelike-metacircular-eval.json new file mode 100644 index 0000000..c024b76 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/schemelike-metacircular-eval.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/selection.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/selection.json new file mode 100644 index 0000000..e4bf64e --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/selection.json @@ -0,0 +1,150 @@ +{ + "schema_version": 1, + "selected_at": "2026-10-12T13:52:35.779627+00:00", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "catalog_sha256": "7fc7d95a2077ede76c4f47890a09de93c2fe558252a122ef830288ad293ceaaa", + "selection_method": "Entire existing 34-task development pool, explicitly requested by the user. No selection by latest score and no outcome-based replacements.", + "prior_selection_file": "../dev12-baseline-20261011/selection.json", + "prior_selection_sha256": "cd51acea6592c01ce9ec8cde34cd46a4965a4164950194f2de7e8ca54649bd29", + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12" + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf" + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1" + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3" + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f" + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67" + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d" + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c" + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302" + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c" + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78" + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a" + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb" + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684" + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e" + } + ], + "prior_exposure": "All tasks are development tasks with previous outcomes and some reviewed traces/tests. This is not a held-out capability evaluation. No fresh trace or hidden-test content is reviewed until every new agent output is frozen.", + "remaining_unopened_tasks": 55 +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-pipeline-parallelism.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-pipeline-parallelism.json new file mode 100644 index 0000000..17a2cfc --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-pipeline-parallelism.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-tensor-parallelism.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-tensor-parallelism.json new file mode 100644 index 0000000..6a9b720 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/torch-tensor-parallelism.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684" + } + ] +} diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/write-compressor.json b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/write-compressor.json new file mode 100644 index 0000000..a442618 --- /dev/null +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/write-compressor.json @@ -0,0 +1,32 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "retry": { + "max_retries": 0 + }, + "agents": [ + { + "name": "harbor_adapter:PiCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "distribution_path": "/home/minix/.local/share/mise/installs/pi/1.4.0/pi", + "distribution_sha256": "60a810111c1fd9b34b59a815fb6399c6b861017e1a2278ae881bb2a563177dd9", + "model_config_path": "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/pi-models.json", + "thinking": "high" + } + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/pi-deepseek-flash-dev34-20261024/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/dependency-cache.compose.json" + ] + }, + "tasks": [ + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e" + } + ] +} From bb219c34ba018cb3773852685bd884067ad9c092 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Tue, 27 Oct 2026 07:41:21 -0700 Subject: [PATCH 03/13] fix(benchmarks): bound docker image storage for pi baseline --- benchmarks/harbor/README.md | 4 + benchmarks/harbor/README.zh-CN.md | 3 + .../README.md | 44 +- .../experiment-v2.json | 1513 +++++++++++++++++ .../adaptive-rejection-sampler.json | 32 + .../storage-v2/build-cython-ext.json | 32 + .../storage-v2/build-pov-ray.json | 32 + .../storage-v2/caffe-cifar-10.json | 32 + .../storage-v2/cancel-async-tasks.json | 32 + .../storage-v2/circuit-fibsqrt.json | 32 + .../storage-v2/cobol-modernization.json | 32 + .../storage-v2/constraints-scheduling.json | 32 + .../storage-v2/custom-memory-heap-crash.json | 32 + .../storage-v2/db-wal-recovery.json | 32 + .../storage-v2/dependency-cache.compose.json | 100 ++ .../storage-v2/dna-assembly.json | 32 + .../storage-v2/fix-code-vulnerability.json | 32 + .../storage-v2/git-multibranch.json | 32 + .../storage-v2/install-pair-01.json | 37 + .../storage-v2/install-pair-02.json | 37 + .../storage-v2/install-pair-03.json | 37 + .../storage-v2/install-pair-04.json | 37 + .../storage-v2/install-pair-05.json | 37 + .../storage-v2/install-pair-06.json | 37 + .../storage-v2/install-pair-07.json | 37 + .../storage-v2/install-pair-08.json | 37 + .../storage-v2/install-pair-09.json | 37 + .../storage-v2/install-pair-10.json | 37 + .../storage-v2/install-pair-11.json | 37 + .../storage-v2/install-pair-12.json | 37 + .../storage-v2/install-pair-13.json | 37 + .../storage-v2/install-pair-14.json | 37 + .../storage-v2/install-pair-15.json | 37 + .../storage-v2/install-pair-16.json | 37 + .../storage-v2/install-pair-17.json | 37 + .../storage-v2/kv-store-grpc.json | 32 + .../llm-inference-batching-scheduler.json | 32 + .../storage-v2/log-summary-date-ranges.json | 32 + .../storage-v2/merge-diff-arc-agi-task.json | 32 + .../model-extraction-relu-logits.json | 32 + .../storage-v2/mteb-leaderboard.json | 32 + .../storage-v2/multi-source-data-merger.json | 32 + .../storage-v2/nginx-request-logging.json | 32 + .../storage-v2/openssl-selfsigned-cert.json | 32 + .../storage-v2/path-tracing.json | 32 + .../storage-v2/pypi-server.json | 32 + .../storage-v2/pytorch-model-cli.json | 32 + .../storage-v2/pytorch-model-recovery.json | 32 + .../storage-v2/qemu-alpine-ssh.json | 32 + .../storage-v2/query-optimize.json | 32 + .../storage-v2/regex-chess.json | 32 + .../storage-v2/regex-log.json | 32 + .../schemelike-metacircular-eval.json | 32 + .../torch-pipeline-parallelism.json | 32 + .../storage-v2/torch-tensor-parallelism.json | 32 + .../storage-v2/write-compressor.json | 32 + benchmarks/harbor/scripts/run_comparison.py | 88 +- benchmarks/harbor/tests/test_comparison.py | 55 + 58 files changed, 3513 insertions(+), 11 deletions(-) create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment-v2.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/adaptive-rejection-sampler.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/build-cython-ext.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/build-pov-ray.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/caffe-cifar-10.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/cancel-async-tasks.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/circuit-fibsqrt.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/cobol-modernization.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/constraints-scheduling.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/custom-memory-heap-crash.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/db-wal-recovery.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/dependency-cache.compose.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/dna-assembly.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/fix-code-vulnerability.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/git-multibranch.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-01.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-02.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-03.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-04.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-05.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-06.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-07.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-08.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-09.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-10.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-11.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-12.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-13.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-14.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-15.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-16.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/install-pair-17.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/kv-store-grpc.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/llm-inference-batching-scheduler.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/log-summary-date-ranges.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/merge-diff-arc-agi-task.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/model-extraction-relu-logits.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/mteb-leaderboard.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/multi-source-data-merger.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/nginx-request-logging.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/openssl-selfsigned-cert.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/path-tracing.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/pypi-server.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/pytorch-model-cli.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/pytorch-model-recovery.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/qemu-alpine-ssh.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/query-optimize.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/regex-chess.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/regex-log.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/schemelike-metacircular-eval.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/torch-pipeline-parallelism.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/torch-tensor-parallelism.json create mode 100644 benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/storage-v2/write-compressor.json diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index c387aab..f12d08f 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -116,6 +116,10 @@ followed by the model queue. A registered `minimum_free_disk_bytes` stops new jobs before they start when disk space is insufficient. Existing jobs and failed attempts are preserved. Agent-specific budget differences must be declared in the experiment instead of assuming native agent defaults match. +For constrained Docker storage, an optional `batch_images` plan restores each +batch by registered repository digest, verifies image identities, and evicts +only image references absent from the run's initial inventory. It checks the +actual Docker storage volume and never forces image or container deletion. ### Cache original task images diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 6ffb09f..828f9e4 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -91,6 +91,9 @@ Pi 目前不生成 ATIF trajectory。 比较运行器支持 Pi arm，以及先安装检查、再执行模型队列的 `--phase all`。 注册的 `minimum_free_disk_bytes` 会在空间不足时阻止启动新任务。 已有任务和失败尝试均保留。各 agent 的预算差异必须在实验中说明，不能假设原生默认值相同。 +Docker 空间受限时，可以通过 `batch_images` 计划按批恢复已注册 repository digest +对应的镜像、验证镜像身份，并仅清理本轮初始清单中不存在的镜像引用。 +运行器会检查 Docker 实际存储分区，且不会强制删除镜像或容器。 ### 缓存原始任务镜像 diff --git a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md index 7944263..83e4c6a 100644 --- a/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md +++ b/benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/README.md @@ -1,10 +1,13 @@ # Pi + DeepSeek Flash dev34 agent baseline -**Status: prepared, not started.** The system disk had approximately 824 MiB -free at the last preparation check. No container installation preflight, model -trial, or official verifier has run for this experiment. The runner requires -at least 15 GiB free before starting new jobs. Existing recovery containers are -preserved pending a storage decision. +**Status: starting native installation preflight on October 27.** After the +repository relocation, the project volume has approximately 69 GiB free and +Docker's system volume approximately 20 GiB. The active +[v2 plan](experiment-v2.json) restores and verifies the original images one +batch at a time, then removes only image references absent from the initial +Docker inventory. Existing recovery containers and their images are preserved. +Model work remains gated on successful native installation checks for all tasks. +The [original registration](experiment.json) remains unchanged and unexecuted. This experiment establishes the first external-agent baseline on the existing 34-task development pool. The [selection](selection.json) is byte-identical to @@ -45,7 +48,7 @@ native sessions and completed JSON events rather than generating an ATIF trace. ## Task, budget, and infrastructure policy -The [experiment plan](experiment.json) registers all 34 fresh model trials, +The [active experiment plan](experiment-v2.json) registers all 34 fresh model trials, their fixed order, configurations, source revision, distribution, infrastructure hashes, and installation checks. Maximum concurrency is two, with consecutive pair barriers and no whole-trial retries. Preserve every failed setup or launch @@ -76,17 +79,17 @@ preserves upstream filenames, sizes, SHA256 digests, and native package checks. The worktree's ignored `.cache` link points to the repository's shared cache. No task images, official tests, or package origins are substituted. -## Run after resolving disk space +## Execute From this experiment worktree's root, provide `DEEPSEEK_API_KEY` in the process environment and use an isolated virtual environment. The command below first runs all installation checks; models start only if those checks succeed. ```bash -export UV_PROJECT_ENVIRONMENT=/tmp/nanopy-pi-dev34-harbor-venv +export UV_PROJECT_ENVIRONMENT="$PWD/jobs/harbor-runtime-venv" uv run --frozen --project jobs/runtime-source/benchmarks/harbor \ python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment.json \ + --plan benchmarks/harbor/configs/pi-deepseek-flash-dev34-20261024/experiment-v2.json \ --phase all ``` @@ -98,7 +101,7 @@ uv run --project benchmarks/harbor python -m harbor_adapter run --config