From 72657a0a2611ab38f0f6c5164d1af8bd94f2c7a8 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 19 Oct 2026 06:14:39 -0700 Subject: [PATCH 01/12] chore(benchmarks): register current-main dev34 flash 1017 baseline --- .../dev34-flash1017-main-20261019/README.md | 66 + .../adaptive-rejection-sampler.json | 35 + .../apt-cache.conf | 8 + .../build-cython-ext.json | 35 + .../build-pov-ray.json | 35 + .../caffe-cifar-10.json | 35 + .../cancel-async-tasks.json | 35 + .../circuit-fibsqrt.json | 35 + .../cobol-modernization.json | 35 + .../constraints-scheduling.json | 35 + .../custom-memory-heap-crash.json | 35 + .../db-wal-recovery.json | 35 + .../dependency-cache.compose.json | 100 ++ .../dependency-release-manifest.json | 58 + .../dna-assembly.json | 35 + .../experiment.json | 1376 +++++++++++++++++ .../fix-code-vulnerability.json | 35 + .../git-multibranch.json | 35 + .../image-digests.json | 311 ++++ .../install-only.json | 168 ++ .../kv-store-grpc.json | 35 + .../llm-inference-batching-scheduler.json | 35 + .../log-summary-date-ranges.json | 35 + .../main-prompt.txt | 9 + .../merge-diff-arc-agi-task.json | 35 + .../model-extraction-relu-logits.json | 35 + .../mteb-leaderboard.json | 35 + .../multi-source-data-merger.json | 35 + .../nginx-request-logging.json | 35 + .../openssl-selfsigned-cert.json | 35 + .../path-tracing.json | 35 + .../pypi-server.json | 35 + .../pytorch-model-cli.json | 35 + .../pytorch-model-recovery.json | 35 + .../qemu-alpine-ssh.json | 35 + .../query-optimize.json | 35 + .../regex-chess.json | 35 + .../regex-log.json | 35 + .../schemelike-metacircular-eval.json | 35 + .../selection.json | 150 ++ .../torch-pipeline-parallelism.json | 35 + .../torch-tensor-parallelism.json | 35 + .../write-compressor.json | 35 + 43 files changed, 3436 insertions(+) create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/adaptive-rejection-sampler.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-cython-ext.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-pov-ray.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/caffe-cifar-10.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/cancel-async-tasks.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/circuit-fibsqrt.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/cobol-modernization.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/constraints-scheduling.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/custom-memory-heap-crash.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/db-wal-recovery.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/dna-assembly.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/fix-code-vulnerability.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/image-digests.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/install-only.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/llm-inference-batching-scheduler.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/main-prompt.txt create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/merge-diff-arc-agi-task.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/model-extraction-relu-logits.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/mteb-leaderboard.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/multi-source-data-merger.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/nginx-request-logging.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/openssl-selfsigned-cert.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/path-tracing.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/pypi-server.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-cli.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-recovery.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/qemu-alpine-ssh.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/query-optimize.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-chess.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-log.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/schemelike-metacircular-eval.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/selection.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-pipeline-parallelism.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-tensor-parallelism.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/write-compressor.json diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md new file mode 100644 index 0000000..c37eadd --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md @@ -0,0 +1,66 @@ +# Current-main dev34 baseline with DeepSeek Flash 1017 + +This registration runs all 34 existing development tasks once with fresh model +outputs from current main `ce28b1e826a292d796ba64f13e79199b66f65758`. The experiment branch and its independent +runtime checkout are separate, so recording results cannot change the evaluated +source. Historical scores and outputs remain separate. + +## Frozen inputs and limits + +- [Plan](experiment.json): source, model mapping, task and infrastructure hashes, + budgets, installation preflight, and the complete fixed queue. +- [Selection](selection.json): the unchanged dev34 pool and pinned task revisions. +- [Prompt](main-prompt.txt): current main's 385-word headless prompt. +- [Images](image-digests.json): image identities checked before execution. +- Runtime checkout: `jobs/main-source`, detached at `ce28b1e826a292d796ba64f13e79199b66f65758`. + +The model is `deepseek/deepseek-flash` at the official DeepSeek Anthropic endpoint. +[Official documentation](https://api-docs.deepseek.com/quick_start/agent_integrations/) +was checked on 2026-10-19 and maps this alias to DeepSeek-V4.1-Flash-1017. The +alias does not pin an immutable checkpoint. Temperature and reasoning effort are +unset. Each task allows 100 replies, 65,536 output tokens per reply, and its native +agent timeout minus 180 seconds for model work; setup separately allows 1,800 +seconds. Original task images, resources, package sources, official scripts, and +per-attempt verifier limits stay unchanged. + +Run fresh installation checks for all 34 tasks before starting any model work. +Only QEMU has a registered verifier dependency profile; the other 33 report +`not_configured`. Agent installation success does not certify those dependencies. +The verified APT index and original uv/Python archive caches retain upstream +filename, size, and SHA256 checks. The [archive manifest](dependency-release-manifest.json) +is reused unchanged from the October 18 run, including all five original archives. + +The model queue sorts by ascending work budget and task name, with consecutive +pair barriers and maximum concurrency two. Every task generates exactly one new +output. Only verifier timeouts may retry, at most three total attempts with the +same output and original per-attempt limits. Dependency-error recovery remains +disabled. Setup failures and verifier errors are recorded separately from scored +failures; a zero without executed assertions is analytically unscored. + +## Execute + +From this experiment worktree's repository root, supply credentials through the +process environment, set `ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic`, +and leave `ANTHROPIC_MODEL` and `ANTHROPIC_MAX_TOKENS` unset. + +```bash +python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json \ + --phase preflight + +python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json \ + --phase model +``` + +The existing orchestrator calls only the standard +`uv run --project benchmarks/harbor python -m harbor_adapter run --config ...` +entry point from the frozen runtime checkout. Native jobs and input manifests +live under `jobs/main-source/jobs/`; orchestration records live under +`jobs/dev34-1017-main-20261019-orchestration/`. Existing jobs cannot be overwritten. + +Inspect operational stages and completion metadata during execution, then freeze +all 34 outputs before reviewing new trajectories or hidden-test content. Raw logs +remain local; publish derived findings and evidence hashes after review. All 34 +tasks have prior development exposure. The remaining 55 dataset tasks stay +unopened, and this run does not isolate a model-checkpoint effect. diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/adaptive-rejection-sampler.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/adaptive-rejection-sampler.json new file mode 100644 index 0000000..c175e07 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/adaptive-rejection-sampler.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf new file mode 100644 index 0000000..e191ddb --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf @@ -0,0 +1,8 @@ +// Route only public APT HTTP origins through the verified index cache. +Acquire::http::Proxy::deb.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::security.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::archive.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::archive.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::security.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::ports.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::old-releases.ubuntu.com "http://127.0.0.1:18742/"; diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-cython-ext.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-cython-ext.json new file mode 100644 index 0000000..7d4fb78 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-cython-ext.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-pov-ray.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-pov-ray.json new file mode 100644 index 0000000..a0413a7 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-pov-ray.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 11820 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/caffe-cifar-10.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/caffe-cifar-10.json new file mode 100644 index 0000000..11be8a3 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/caffe-cifar-10.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 3420 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/cancel-async-tasks.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/cancel-async-tasks.json new file mode 100644 index 0000000..bd2a94f --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/cancel-async-tasks.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/circuit-fibsqrt.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/circuit-fibsqrt.json new file mode 100644 index 0000000..1fc3a36 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/circuit-fibsqrt.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 3420 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/cobol-modernization.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/cobol-modernization.json new file mode 100644 index 0000000..594d2ce --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/cobol-modernization.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/constraints-scheduling.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/constraints-scheduling.json new file mode 100644 index 0000000..b94a11f --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/constraints-scheduling.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1020 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/custom-memory-heap-crash.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/custom-memory-heap-crash.json new file mode 100644 index 0000000..055f060 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/custom-memory-heap-crash.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1620 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/db-wal-recovery.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/db-wal-recovery.json new file mode 100644 index 0000000..bf64279 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/db-wal-recovery.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json new file mode 100644 index 0000000..ed62f01 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json @@ -0,0 +1,100 @@ +{ + "services": { + "main": { + "volumes": [ + { + "type": "bind", + "source": "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf", + "target": "/etc/apt/apt.conf.d/99nanopy-index-cache", + "read_only": true + } + ], + "environment": { + "UV_INSTALLER_GITHUB_BASE_URL": "http://127.0.0.1:18743", + "UV_PYTHON_INSTALL_MIRROR": "http://127.0.0.1:18743/astral-sh/python-build-standalone/releases/download" + } + }, + "apt-index-cache": { + "image": "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "network_mode": "service:main", + "entrypoint": [ + "python3", + "/proxy/apt_index_proxy.py", + "--bind", + "127.0.0.1", + "--port", + "18742", + "--cache", + "/cache", + "--workers", + "16" + ], + "user": "1000:1000", + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "mem_limit": "256m", + "cpus": 0.5, + "volumes": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/scripts/apt_index_proxy.py:/proxy/apt_index_proxy.py:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/scripts/cache_image.py:/proxy/cache_image.py:ro", + "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/apt-indexes:/cache" + ], + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + "import socket; socket.create_connection((\"127.0.0.1\",18742), timeout=2).close()" + ], + "interval": "1s", + "timeout": "3s", + "retries": 30 + } + }, + "uv-release-cache": { + "image": "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "network_mode": "service:main", + "entrypoint": [ + "python3", + "/proxy/uv_release_cache.py", + "serve", + "--cache", + "/cache", + "--manifest", + "/manifest.json" + ], + "user": "1000:1000", + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cpus": 0.5, + "mem_limit": "256m", + "volumes": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/scripts/uv_release_cache.py:/proxy/uv_release_cache.py:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/scripts/cache_image.py:/proxy/cache_image.py:ro", + "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/uv-releases:/cache:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json:/manifest.json:ro" + ], + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + "import socket; socket.create_connection((\"127.0.0.1\",18743), timeout=2).close()" + ], + "interval": "1s", + "timeout": "3s", + "retries": 30 + } + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json new file mode 100644 index 0000000..e762efb --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json @@ -0,0 +1,58 @@ +{ + "registered_at": "2026-10-18T13:25:43.131474+00:00", + "metadata_source": "Original release metadata preserved by the source manifests; no versions or payloads substituted.", + "source_manifests": { + "benchmarks/harbor/configs/dev34-main-baseline-20261012/dependency-release-manifest-v5.json": "bdf3d3ef8b7743700c5294c4a511c686a3928c178a77fd4abcba90ee4557ffcc", + "python-agent-release-manifest.json": "78aecd7c397c469196f9d246a160bf1fc99ad2d51cb71320b7d26ebf839647ce" + }, + "assets": [ + { + "version": "0.9.11", + "filename": "uv-x86_64-unknown-linux-gnu.tar.gz", + "url": "https://github.com/astral-sh/uv/releases/download/0.9.11/uv-x86_64-unknown-linux-gnu.tar.gz", + "digest": "sha256:817c0722b437b4b45b9a7e0231616a09db76bab1b8d178ba7a9680c690db19f0", + "size": 21699161, + "asset_id": 318993196, + "release_metadata_sha256": "c1a1378ece65ce1aa8343e57bdbd305289a8fc6d3d9180e8bb6b0f69acd91413" + }, + { + "version": "0.9.5", + "filename": "uv-x86_64-unknown-linux-gnu.tar.gz", + "url": "https://github.com/astral-sh/uv/releases/download/0.9.5/uv-x86_64-unknown-linux-gnu.tar.gz", + "digest": "sha256:2cf10babba653310606f8b49876cfb679928669e7ddaa1fb41fb00ce73e64f66", + "size": 21370871, + "asset_id": 306970260, + "release_metadata_sha256": "1921b17d3370aafb690b29d1d121ee8719ef3d3dbc0c7cf3b948c3059a927a5d" + }, + { + "version": "20251014", + "filename": "cpython-3.13.9+20251014-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251014/cpython-3.13.9%2B20251014-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:c0ccda275948c79996e46993c2c5476ff5cca606dee530f1dea712179131b348", + "size": 33517986, + "asset_id": 304305340, + "release_metadata_sha256": "8e5db21cfcc2bcafcb2127f269f00cbfdd1f74842a685acd2a14e3c1ad6f8394", + "repository": "astral-sh/python-build-standalone" + }, + { + "version": "20251120", + "filename": "cpython-3.14.0+20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251120/cpython-3.14.0%2B20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:66f385305ae0eefd6b65e2cf942bc91d943a61e26fb7581e029f760a2b04f393", + "size": 35376131, + "asset_id": 318881850, + "release_metadata_sha256": "e3a737862d2dddf11c73bfef6c2f2579e9d4ce5bd271cca53790c998590dee58", + "repository": "astral-sh/python-build-standalone" + }, + { + "version": "20251120", + "filename": "cpython-3.13.9+20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251120/cpython-3.13.9%2B20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:1bc9a94ab2425fb6ae58e45f69cab61647759b57895649701390822d976fa923", + "size": 34257108, + "asset_id": 318881608, + "release_metadata_sha256": "705690ab2fd466e90b4c868635c529f3c7fd0cb597d399e350f399e3f3e35e6c", + "repository": "astral-sh/python-build-standalone" + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dna-assembly.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dna-assembly.json new file mode 100644 index 0000000..bed3d40 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dna-assembly.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1620 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json new file mode 100644 index 0000000..6efd410 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json @@ -0,0 +1,1376 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-19T13:13:43.549137+00:00", + "experiment": "dev34-flash1017-main-20261019", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-19T13:13:43.549137+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "main": { + "source_root": "jobs/main-source", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/adaptive-rejection-sampler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "scientific-computing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "95408217b9c484ffc18a02a91b7706323b6902919c72b24866a32e27a23543e2", + "environment/Dockerfile": "e644a3343e339303230f1441b7383f076d21c69a5a7aa6430b4f8a9d27f15c8b", + "environment/protected.tar.gz.enc": "7fc9bf1347bb97da8dede04b2c58638f225070913e222d76627068aebd1b1df2", + "instruction.md": "e6d28c28acd76bef61817427c74c2a29e3e8d75f535cbd708afcbfd02c54013f", + "solution/solve.sh": "fe443a8082b5183ba1623311ba0ee958888aefa633eff93766b9ae14e9bdd84d", + "task.toml": "6ed7b37f74beb46e9d47b0f5fad53025667c9548baff4e96d467b56d429fab1a", + "tests/test.sh": "38b43560d173cc2b952c3a3e17b8a480216d84e33450515e047bcb0d806b1e0a", + "tests/test_outputs.py": "547dc6e107f034f41703722aeceb6d0236e3fb69116fc3f2fbaba11884de352f" + } + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/build-cython-ext:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d5f5113941c3963292bae396baa9369571026f46e59437f98ddcc93475dd7729", + "environment/Dockerfile": "cb6cf421cb9b69d43bd25cd491e49bda982fbb865d47515178d017eda727194e", + "instruction.md": "7e52874e45f88505bd21540229719de6639a401477557d814489981d472f0e34", + "solution/solve.sh": "41bbdde88617070366a70e518eedca466a4ea1acd7e96899b2fb7da988e40abc", + "task.toml": "c472f0e7522c128fb7f03e00b50c8cdf2e5e6474daa1e094e7441f4b54e770b2", + "tests/test.sh": "0156c9c4085258b696ce424cf1ce96cba44d1263c5d73bdf54083fd80237f915", + "tests/test_outputs.py": "6549f1a8e04b6163ec08877e6f6b9868bf94563f3a3c72ac1fba9c512c31689e" + } + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/cancel-async-tasks:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9adf86b1279cc44f83fe16e6d340595aa992240f1d87243d204cae32e453693b", + "environment/Dockerfile": "1ee7bdb4a7536f5f114c11a3ac6ff42790abfea901fcba6b2a8ea8685f19140e", + "instruction.md": "23d22f7039b50682941652c716f96dac314b8585c32c133614cac77dc067f47e", + "solution/solve.sh": "4938ca4ad5b7dcd36b414b42a1abb10c058ecefce7462b678f9bd05c9fae1844", + "task.toml": "14db0675e04408f2133b0067e8dbddf66a1d6e7baf6b8d38bef8ecccfc4e58a8", + "tests/test.py": "f533c0a64db068f582b4733c5b81f2e66b8fa52fcd3efa65752e6bd6d24780aa", + "tests/test.sh": "79600a6b863def31b4daa008297423010cd2d6243e689d67503e118e4dc53397", + "tests/test_outputs.py": "d6f510b51fafc81b4e9b5cc95535aa13491c8753e0e4171db42c45f4de460ee7" + } + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/cobol-modernization:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "c89318cf59efcc7040e3d946022e2616161b6733931a080cdeea7b1918e7ecc4", + "environment/Dockerfile": "8b175c83fc03ec7d16d6101516bc5100230b592f480e435279f4fe051c9280b2", + "environment/data/ACCOUNTS.DAT": "46f63eb29442ebc6716bcd8a0c36f4367910f2783f6b84a307ed9a61ccd1046e", + "environment/data/BOOKS.DAT": "db04da646207cd87f8e9c5b6f0b57d95953899f3eef22de1a81423f06ef4a9fd", + "environment/data/TRANSACTIONS.DAT": "1fa4cc5420ded94d585c563e8196b85166ebbae2bb8924b20014717fb2fb9ee3", + "environment/src/INPUT.DAT": "52fc984f0cf14af1495285d7efe02b9bae13159eea59255560ee938c4f78afee", + "environment/src/program.cbl": "27cb0e56f65f52fa75f346720b37a180b8e034ebe7d7622b979f195087bd6ebb", + "instruction.md": "a87074f1141b7345394cd88053d0debb0dc8f7de95308882c6538ee9a3b359da", + "solution/solve.sh": "7d4a0504e1686c1790bfbc074d4e598f4c631f54c2bc5f3d5fd786574b4810b9", + "task.toml": "4b620211dc3d6b1b989f0210c5ecd2f606974581c8bce27bd78659d48cf0cfbd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "009180a7092583fc8143bb3ffc3957923aa0d61d0aac3a471f76eeb4aae54a88" + } + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/db-wal-recovery:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "file-operations", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "600762031a1a02ecc3b46d70f7683f3ff930b47c8a3d0e27cc31b146158c3126", + "environment/Dockerfile": "63a260531c62f4ae72060db17f295c0aa03cfc6321f8162c324b7aa6b3f81302", + "environment/main.db": "68cb288eb29e76d814b4e7ee6887b37c415675e644f88173bc191348e290d119", + "environment/main.db-wal.encrypted": "68753bdce26196a53194260d001544cc6bb8f9f38e2f2f6826d09798e31e6e26", + "instruction.md": "5a20375162921b8911c67035d1905df7fd335d1bed03044271e608b2c3c93b26", + "solution/solve.sh": "ed8530c94103bf19783a1d41bc63fe4db478cdd9629f22fee25c45a45699f5e8", + "task.toml": "1cdc2c615e259bc10265efbe71b518332070c1c002e226566399e9788f18d0aa", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "88dc5c2a3abba75bca0e0f9136223ae5bb585d5984b8d9eba51a7829c1d2ba49" + } + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/fix-code-vulnerability:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "security", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "87f28dbad183c3a634887696d328ca68a8a0858f132bc48cb7341c22267d38f7", + "environment/Dockerfile": "0917470986224df4daf4854a71302ae81c73ba5dd6e25be1232d29b758a5b3ab", + "instruction.md": "89b1abdf0af19399f720233b5cdf6b2e8e0ff7b6e3fe3e2a3af0d4d298a8641a", + "solution/solve.sh": "7a0131b369d83ae8546ecc3f9b4e23d38240f1bd1f0941e4430fa85b5182ee03", + "task.toml": "9aa2bd14ce9408289d199d0608bc3fa65b8b5e23374c9641020de62197dbf1e2", + "tests/test.sh": "f1611da693cb4cfc5e8482698c6a1858a5c67220ae9db89b50aed66584e9b17f", + "tests/test_outputs.py": "c006a5de95b69bd78055aa26cc055734dbd50718ede25d331f767bfee4ef7f26" + } + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + } + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/kv-store-grpc:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "51fdc2ad06f0adb1270164f247ea41799b2ee2ad15490e48ddbf5e79d3767aaa", + "environment/Dockerfile": "47c9982789092437f950736110b2bd18899c39b8fd20f3af3c95b848e564a197", + "instruction.md": "f4b3facc199d5b1c78db6e603ffa29ec1b93a77ff7d291e1a92b42e6bf582bed", + "solution/solve.sh": "6bee2938ac9731d1156b0ca02b28c3d5793bf2cbe5f995c863d397093af01cb7", + "task.toml": "6ff0f30a526bfff86a67b0b53439b9965229c0f368167dc1a5d05f34d82299a3", + "tests/test.sh": "c348467a076bf614d0a850e25b2263f85bbf7b4c7a8d344fbd5454a0611a0df8", + "tests/test_outputs.py": "115de2e964bbe689d07930527113422ef8a9e4be4eb958ad84594e3fcd2b1b0e" + } + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/log-summary-date-ranges:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "026000415cb577f3d3d5132902670262f7d5d8f69841903ec06a882292f756b4", + "environment/Dockerfile": "b479f6e987b63a6ec118fa1691b8bb85c437b2a6a0fe63feab5ed7154957fdfc", + "environment/log_generator_deterministic.py": "beb6eff59979e9aa0be63ce8a0f1b0df1178f92871f04ba9f76fa70cf8beb81c", + "instruction.md": "c1d4516ad4ec2238209740eaf01291df9ab48b0028a3fc5df383c6f831453484", + "solution/solve.sh": "9f6a8ee8947bdf1c0f31b530b6194ae998d66841a3b785fa2e246ee840350ab6", + "task.toml": "9a9593673d6805ef3a301ea734e9c4dbaf80da1b37c0fe9d1f2ceea2a8c57e18", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2fc06a6ce1944ad82f84fb437395fa75ed8c0eadd385b92e375c27e5bc9cb2b4" + } + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/merge-diff-arc-agi-task:20251031", + "resources": { + "cpus": 1, + "memory_mb": 4096, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d6b48ed0890b12eb68f72c12cd02689bd6bfae2191dd2688949cde6fe77f6a94", + "environment/Dockerfile": "4a03ae6f0c60b9c2f45c33f0413b5aa75c9811870254e3ff1968fd0119173fc6", + "environment/task-deps/bundle1.bundle": "a759f9b6ffc1a2eee50778d72a1899ad7d724becab84fa30e7aac71ee9e443a5", + "environment/task-deps/bundle2.bundle": "75e3e3916a01c980a03715de67b16c729e03661d94a694b5f4223d229bd6593a", + "environment/task-deps/examples.json": "5426569e4ea078bbb74039ec712a2d4d35f1f9fa3a5f26bf2a0de4a2c8ef5db8", + "instruction.md": "2d826479873e5ac1c94c1f802763bf85c22bb04624384685ebdf6b153fe0278e", + "solution/solve.sh": "48d4becc41fda5b6063fe96947474ffe778a29d6924d7b2bb235eb61e5ce555d", + "task.toml": "6ced40e06b43b169857509765eb7f07791db558f9437cc7367d2536771c2ca5b", + "tests/test.json": "20f116a6b718bbf4874bbe203d15a5c0f85a1d88cb622e2d7529c7e4fb2d4c28", + "tests/test.sh": "c9052954cd6ce25c4a375a2fb48e765b1efa5b39f7991b93993cc793af8ab9d2", + "tests/test_outputs.py": "2754ba6c5c5f75c9db5ed2c5729d9f5d7e21cb7fdb18aa0c65abdc3e7e85c149" + } + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/model-extraction-relu-logits:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "mathematics", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9465accba6c57e5de9a196eed776a16eeb0852a8afc18fc3f8979e2de71b0c7b", + "environment/Dockerfile": "598b1f80d3b7194e97cbf02c8d245de30e303909e0cbe6465125d1d3179fddb5", + "environment/forward.py": "af1a23efd8e4390d16a29777c28116b2f8623ae3f7b628fe14db9f94dac26f5b", + "instruction.md": "c9e179dd099065a34c362a865b0eb9fab5f7d96be582cf117e7519580c036ebe", + "solution/solve.sh": "4cea5094f074ff21453d4423aa1b415a104ed90caa55a4acbb4f09941a3994ea", + "task.toml": "eb64dfaf8ae88868911ae28855cdc4f767de464001e2704919015b595be6dc29", + "tests/test.sh": "3b2d8f5e0353c2bab3515d6387468a863a4f9f241ee131c73f545f3fdf9e2d52", + "tests/test_outputs.py": "e5189c7eb9942ede55e1b48a80388590b80a486b75497613f257b3373ad9b2c9" + } + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/nginx-request-logging:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d79997586d1c14ed8dbfea050e3580b25500e04c8345acfd3f294152433c795a", + "environment/Dockerfile": "43003466d89ab83484a3fa618e35b58cdb74a002b1e8716ef30f12b3ac8302d8", + "instruction.md": "26a7ac98aced6107f147206790ada77bbf8ed3c25fb360cf323d5fc6889edf99", + "solution/solve.sh": "2454bbbf7c201a9a778165f286373ba2cb991a0eb1d026a8005d57ae71250dd9", + "task.toml": "b7a78d00be13656f75069197100b7b8c4fd268decf6f78929c853b3222a2729d", + "tests/test.sh": "27761195f53cbfe5941fe39b573631f6c926869e64b7c43220377e1e116528e7", + "tests/test_outputs.py": "045cc716c14efde3b0dcff5fc7c85ec5d18bfc6ce66f8b40a418fa2a3a4acda0" + } + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/openssl-selfsigned-cert:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "security", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "5dfa88651890cda06fa2a25babd3b813818767635e88bbabecc8305d5bb3942f", + "environment/Dockerfile": "f590b523038be43afcaf5291d7b9e8ed1333c48d9994e77b8dee2f2c352f0f40", + "instruction.md": "38054a3371f7c2c853478a2f9af0bff858437d5b28a9a07483238f06e566921f", + "solution/solve.sh": "58cf3c844e907745cd81fb4bd75efb3d080e2f0e82b589c56c954ae6e1756510", + "task.toml": "75c8913f0fa15890786c11999cde920b8bfc10d4ce8a14edf1e76985a2ac5800", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "90970bb3c372e8e7fb6bb9ac5f649ba73eb38e1a5ea2e922a9057a72b6b4192d" + } + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pypi-server:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "906a096a2cc7b9afc738ff5ad5fbd605be65c103f5069c6673915223b9b187e3", + "environment/Dockerfile": "17a4e67580e4f2c6a151d04b9d74eb020666e7eec8e5259c4a5411f9e93858d0", + "instruction.md": "1c537e4a311a8e71155e121f57bfc03f742fbfb99d05314cd03e433996a28a31", + "solution/solve.sh": "1143c0eade4932ec3aeaa5013fb5e23bbcccca15dead294aa03b6af88750775a", + "task.toml": "07c39d7a733c89adadf74bb63a3e906c3694feb6c490d5fc85ac187c003fd447", + "tests/test.sh": "d2b1220e42bcfdcce87907ac5bbd69ac210a3c94926545ca2439636786e7784e", + "tests/test_outputs.py": "82c6351722e9aa984ff1d701bf57df3150ce51466df2b70afa1aa62d2650be4c" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-recovery:20260430", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e080308a05703b40edd16f53aea41a0e6b9874750ecee35bb432029d0d44bf53", + "environment/Dockerfile": "8a6b7edbb9e7c241e447f15a71e13e5d043b25169be2fd9802b84f5be3d56eb6", + "environment/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "environment/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "instruction.md": "2bef07dc28be3ab2c35ae389584034c2e4dde92c2c31b3eb6bd829628c8c3f4a", + "solution/solve.sh": "175a7802777b43e0fa4f629d71f7c68fb40706813c342aaddc72108d89bd5b2c", + "task.toml": "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd", + "tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + } + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/qemu-alpine-ssh:20251031", + "resources": { + "cpus": 1, + "memory_mb": 4096, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "qemu-alpine-ssh-v1", + "task_file_sha256": { + "README.md": "72f1c83fa1e3e6381ef85dd82fd109ee18cdeac090311cf3212a0a8cff0a32a5", + "environment/Dockerfile": "015ce0601ff54758f66b33f4952e0d3a577d84c09aed8f3ec5945a570e8d168a", + "instruction.md": "8348c60490cb90bc690cba259dfe98996da4c3294c6d42bba1119b00c64de4de", + "solution/solve.sh": "1cd31186c427b91bb00499d22aa4f0214fc1ca93b40b6e4cfa969611b848db2a", + "task.toml": "10a43ec6ee732eeba852c377f12e23178f7fa5a4868143f3699a15565ffb64c7", + "tests/test.sh": "3b244479776c4ee3639631112799f2cde22cbb80ac10511715e3a8f32909c289", + "tests/test_outputs.py": "83ccb4bdcd259b342212bebefe395f289858b4dc55b2ee7ae0dab238ad2e9718" + } + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/query-optimize:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-science", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a82c53e333a0ac0b0b410b5e26cb8e7305dece31db84d340248ffb13d5a45160", + "environment/Dockerfile": "064818771f200e0517d22068f87229a7a95d0a82b0778c84ca7203bf1926a0b8", + "environment/task-deps/my-sql-query.sql": "43382792f0f438625d643e34abe144febd5669518f4ab1ab550a15c6d9314a21", + "instruction.md": "473e12d320d2732f679fb631c2a4dab4a01deea6775686718ae0c417b8a39447", + "solution/solve.sh": "0d95f3cedb287ff9d47c796cbaf812c300d1222c10fa80c49e8acb0e81eb6978", + "task.toml": "6e88f2a86df699594bcb1bf81df4ed694505f79c9420f671335b02524bbe3868", + "tests/golden.sql": "4da1885d56b3ffa7b7b22b422996d018795611f6963499d551582881b74b84fd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2068ae80c57af8d52a7ae510f0666732c057806e556b480970f36f14bae683f6" + } + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/regex-log:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "29ffd98c55dbb6f4844f1b9d8c1769b2155c117ee475e0df6bbc5ef330d239b3", + "environment/Dockerfile": "5f264ff0bf62a49aee3c6398447de5108df141e7a2ad35c6de8ef39fa62049c6", + "instruction.md": "4f7ac05e70cf9220ea0f1e5a052c5f908cd0fa884e847d80b0bd51bae2e96f9c", + "solution/solve.sh": "7e670d4f2b4bccb1e4db38f2a173e085ceda028c38167912b466b0a84fcc0999", + "task.toml": "6bc98525852755caf1fe34c174c2f10f4328adb4ce6c1968dcafe1c877e4e160", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "345c3bd09ab6f6fe8c8361a58c0a47bf0a13b3fcb38a5ac7824e44ff855e8f72" + } + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/torch-pipeline-parallelism:20251031", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "297ddea07c230dfd246a8ab07337dd7abcd8e5e2a2b2e26590cb0a8af779839c", + "environment/Dockerfile": "c357ea41e4a4013c5c5980b8fef04af6a5ae250ccc6697cf3bad3127401a931f", + "instruction.md": "989bfcf0c54baf6c8cdc391743dbb2f9e858c5779dd85112536875d4d1b2dcd2", + "solution/solve.sh": "787198e4178d88b887697aefdd29819f29435ffdfe356545d92f58a30fb38b70", + "task.toml": "8b4f9f525c08afbbc4eca9168e46238a61ccb903486773e9ca4da64bb7ffe857", + "tests/test.sh": "cc9fd28b6590993fc37e4a99da771cbf04726c8c62fe308e802612241da69dea", + "tests/test_outputs.py": "23574246a407163a8e6581c84287cc2f1eb745efbaad2a317a5a28c39958c64f" + } + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/torch-tensor-parallelism:20251031", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "4c1b3c3be30548f1f94c6f5503b1bc74b006af7f192dc4cd026f411df1a54604", + "environment/Dockerfile": "5f264ff0bf62a49aee3c6398447de5108df141e7a2ad35c6de8ef39fa62049c6", + "instruction.md": "13bde2ab165d72a311e2bf066d5d6c3c8761bf9d3dd39d1b23eda23bc546b910", + "solution/solve.sh": "d083b7cbf9dbb6cc2ff38c8f2acc150f0059c04aea42d6c2183aeaecc5a19385", + "task.toml": "72fd2a5500d1fe8795dcae76cccb1a29fb25894881a7475f2fa519c87d425a8d", + "tests/test.sh": "a7e6c9408b2401e4e7448371f7ddc72f3284caccfb56277d3cd4e98051868cdf", + "tests/test_outputs.py": "4ae4285e15fba1fcf13d7b7c0638ac8c2e2b0797c937abab1fb6b136fd508290" + } + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/write-compressor:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "603015a28500a28f0838fea2c43a5168613746bfd07737920f4bae40b5a326f7", + "environment/Dockerfile": "9a583d29d46818f6ff2be7c6a0879d03343106e5376fd5a48eb5926b402bb93b", + "environment/data.txt": "0fc483d0fb31128dfa41ed11adca56c88688888189826ef9daeafd7559b4fa4e", + "environment/decomp.c": "7f8d9ef66943f912a217aa48f4ef5efc967b44c2b21d0f56df029b5ab413f62a", + "environment/main.rs": "fa53d6572fca95903e30d2d953f1e428bb8b63513ff218ebee9bc60a425ab389", + "instruction.md": "c4fc3e73e44deeac95f1cd4a34a2d87e853b22ceec2b2824767f5f087b470bb2", + "solution/solve.sh": "01550b7de7ab1320b1898eae055226bb7c73bee259c75102d2240c974a730409", + "task.toml": "6066a509066433c97c417fea68c4e3e73996f698dcdcbabe07718835eea355a6", + "tests/original-data.txt": "0fc483d0fb31128dfa41ed11adca56c88688888189826ef9daeafd7559b4fa4e", + "tests/original-decomp.c": "7f8d9ef66943f912a217aa48f4ef5efc967b44c2b21d0f56df029b5ab413f62a", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "809f486053e3ffbce852111dee1f63d80ab31297bd33172389e50278bed384e2" + } + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "native_agent_timeout_seconds": 1200, + "work_budget_seconds": 1020, + "verifier_timeout_seconds": 1200.0, + "docker_image": "alexgshaw/constraints-scheduling:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "personal-assistant", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d54a690970149e1216bd4fc066b45069c7948c13e3b1ef2a213a74342805f209", + "environment/Dockerfile": "7d9aba072a4a907229728b9739c8cf821fc2c98450babd64452682caf3759c84", + "environment/inputs/alice_calendar.ics": "5a5d45804bffd339c2479d49e76fe817b38244d14d389afe1de583443808ecbc", + "environment/inputs/bob_calendar.ics": "c97e85dbcdd756758f8a80f4db5fd5c1dcacee79298e4d2262f46de5ff680b50", + "environment/inputs/carol_calendar.ics": "32a0f8687637df0a12b8e5c56d49c4192bc3e790856943632d70f237c7c1363b", + "instruction.md": "342def78177f9bc41bea4d11af679bc385a9dda2ac8a1e425be384a5b7f937b6", + "solution/solve.sh": "802d81ec064fad3050bbade95a2f847ac24e0b7f80fa69b0bcee0347dc92fe77", + "task.toml": "7b9842de16d74e8faf8c7c0aa03b0b81145941cb12dd322d81ae23fc48607250", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "1422b345ba3aacdfc4184e962811b6379988435896c50d8cf62903e70990c5d1" + } + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/custom-memory-heap-crash:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a06ed23a2c467aa36d0d4a88e5c020b1076cc25b8d483517239658e9c596dde9", + "environment/Dockerfile": "6ee81e5a4fa2540443d7f27d577ba840728fc6aab9f301ab7919103673c78080", + "environment/patches/locale_classes.tcc.patch": "16951f4a4df6bf253886ac75f5a39fdbc328e415194493f63cb7589a364aa817", + "environment/patches/locale_init.cc.patch": "e97f133ceebd86f8881f972e0e45b0ec0a13a9bfa428d8032008707683d54292", + "environment/program/main.cpp": "b94538c76887cc2b2ab739b5ac4731cddd0faa8f4a9892a6265c062bbbeede14", + "environment/program/user.cpp": "92b203786fed1bf0b3aa1675c9cd2d82b2651170675064da8ffee73b98ef14f9", + "environment/program/user.h": "f43d155995a56c22b9cd1417daef7c3b5a976cdc662a98eef0feb438ae1eee77", + "instruction.md": "53c68ae14a3c3af9a97c40bfd9bdc30a5d98b37eaa2f69f763475745f70f108d", + "solution/solve.sh": "b0b1d10bea1cd5be14a82b35d522df616df1e67028f90d2a6bef90aaa98511a1", + "task.toml": "ed474be1983e212913aa8866ebb66216acbe36aef2339ac375d232aa46617481", + "tests/test.sh": "70a2b55f0c5757f6b123a216dfd202be4a646680bced28dddcd31fd8959a058f", + "tests/test_outputs.py": "4e1cb33f777fa7fb5e1536f9bea9eace6deccd515c67d4f87570adbe692914ba" + } + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/dna-assembly:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "scientific-computing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e7855c23d9822f688eec4dd89ca0f1caf91c6a69e1c8f3c9eb544b4ab2a94821", + "environment/Dockerfile": "26e9e43a98f0b1d132bb638f44b7dc882b0d0ca32d56fa33048adeee00ab600d", + "environment/sequences.fasta": "ebdc361b5fba28eb0a2c7a8a8a444ccb12cc842c8cbc6df5a2c24254f130736f", + "instruction.md": "88998c98beabdd966ab46c04ede822bbc74796696aa1bf87c567a9131210533c", + "solution/solve.sh": "c976cb118c8ef1befd846ba05fd5cadb1b1f529bc76b9f2ea9cb3049e6e11a47", + "task.toml": "7b571544b64d9be1c91526837e5d805c2181afe0e6fd57a7f0676bba36c868d7", + "tests/test.sh": "271c8317c98539e3b49233d017fe51fb0e5fba8114e76830fe698565499b2b1d", + "tests/test_outputs.py": "5c2d0fd2e5f6303263e9750f67acda7db491ef04841832b4fc62bcfb61b08924" + } + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/llm-inference-batching-scheduler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "machine-learning", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "ecc9dd635357c184742cf72fc1e452a8946e207b183fc9a7eebbcf07133b911f", + "environment/Dockerfile": "10a6193904ee0022ac098504182989893b196dfc74e626a3ca48534a8eacde48", + "environment/task_file/input_data/requests_bucket_1.jsonl": "8b3cf1e2b06074162038064dd2c114abe308acd32cec7d365ee63ee81978f7de", + "environment/task_file/input_data/requests_bucket_2.jsonl": "e15e2d4c4d55f875c42a6042610d6282f6c0657c39d3e0d0f087268252525cb5", + "environment/task_file/scripts/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "environment/task_file/scripts/baseline_packer.py": "547d230d5e6d93197803480cb81cab0ec6ac63fcfa0a24b246f549523f216c4e", + "environment/task_file/scripts/cost_model.py": "0cf417fd5ff0112defcd46970d37a290c0763c5a9092e91cb3ebb86c3f20e6ce", + "instruction.md": "0817374767533fbf51bb93345ee1bba266263c216c6cc55c096e71b4a1d37dfc", + "solution/solve.sh": "cb9efb586d76a893c3027e62029e286eb6c7583dc92dbf1840a3adc236a926d7", + "task.toml": "52aaac942f917e6a91ff658e0272195bd1cd842d41d71e302414c52603cc588b", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/cost_model_for_tests.py": "d3e810a99d40327fb8317f6512d879c52c6aeebdf5dcdc0c5e117c7df61c9ab9", + "tests/test.sh": "005f980332355e2fe143d4ff4aac11828adf070594bbe7055399c238f06953ca", + "tests/test_outputs.py": "80b06b7c4ba0db28832b767d48be74bb462d7f5df5798d9cb58c4f1160d96ffa" + } + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/path-tracing:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3e4fc952b3f9a860c7ee7837b3f3c686f1283ee7821b014265dba7c94d7fc849", + "environment/Dockerfile": "7aca02e4e20b04ed2269d627a29dd4e7337a02764a9652b641452bf09836a3d5", + "environment/orig.c": "665c1bde0907740890ac114d6b36af1f3fd00449fa287583817f778bb5da200c", + "instruction.md": "9f36e02d636f5dc0946619bc518f2e51c24fa75251da9d1f0ca9eca612cb9572", + "solution/solve.sh": "d107abde101eecee6838da80cc52a3d480c93956fff3a9343a299ea01b3fe9e6", + "task.toml": "cf47013f69bf1b43242da984f231d4daf364fdf44d3845f2eea2ca2cb9308c46", + "tests/test.sh": "a719dbb1ba7ae76df095e9d815941dfa4c55edaf11f173b9c3c3097afa838548", + "tests/test_outputs.py": "572e68b2843191d29592faf42a405861b7958d8e894868b74c5a8886b53eb879" + } + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "native_agent_timeout_seconds": 2400, + "work_budget_seconds": 2220, + "verifier_timeout_seconds": 2400.0, + "docker_image": "alexgshaw/schemelike-metacircular-eval:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "095d3ec6dcc0fe0c7a7e92355456dcea6992f91d1f8d397669b8cfe18475f506", + "environment/Dockerfile": "9b3df76222e16aad360fbd34cb0f71a0be8986052c065b10b92c0cdf417b25c7", + "environment/tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "environment/tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "environment/tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "environment/tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "environment/tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "environment/tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "environment/tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "environment/tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "environment/tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "environment/tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "environment/tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "environment/tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "environment/tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "environment/tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "environment/tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "environment/tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "environment/tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "environment/tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "environment/tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "environment/tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "environment/tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "environment/tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "environment/tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "environment/tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "environment/tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "environment/tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "environment/tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "environment/tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "environment/tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "environment/tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "environment/tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "environment/tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "environment/tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "environment/tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "environment/tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "environment/tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "environment/tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "environment/tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "environment/tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "environment/tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "environment/tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "environment/tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "environment/tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "environment/tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "environment/tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "environment/tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "environment/tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "environment/tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "environment/tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "environment/tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "environment/tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "environment/tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "environment/tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "environment/tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "environment/tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "environment/tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "environment/tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "environment/tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "environment/tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "environment/tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "environment/tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "environment/tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "environment/tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "environment/tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "environment/tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "environment/tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed", + "instruction.md": "3af5f9567b62f93683db7518ec6fcb609f83796258aa02166b5a7091c5b69031", + "solution/solve.sh": "e2c5b0ac2bef4801a1891e13c108813381877426c1a7f01bab86b4fc1f712510", + "task.toml": "8dae8eb226183b0ca92331c1106152a7abc93b03b4e5a95291b86a4ddc9b44e0", + "tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed" + } + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 1200.0, + "docker_image": "alexgshaw/caffe-cifar-10:20260403", + "resources": { + "cpus": 4, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "machine-learning", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e918e6852c5c034422b81a6be22d7596fddcee53abe088f56727b2482da32728", + "environment/Dockerfile": "b3b8adc915a1afa035b604273af3bc1aa8b90999d2dfd72763218b34cb87abcd", + "instruction.md": "46907b0ee048f74269c29a15e0e086546b2b1220149597017da8b5ccc05a0cea", + "solution/solve.sh": "771887de1ed557269c7f3160c777519aa9538e1e9b59771b2f6ef38c8952490c", + "task.toml": "2831165ad9d9e8bbc646e6d85c0a2290640dcc4f13749547a2937c7675012a8b", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "e99b9f6284e923388fbc21488b1af99fdde46d4a809d3557b0788dbf1cb6b281" + } + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/circuit-fibsqrt:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "201f7c889dd6f73dd83ca078e347272b33720fa622f8c757ea6d5770453f4cbd", + "environment/Dockerfile": "780c23986778f3e5f842567868af30b1135ed095ca9449e1c90b7f0e0483d3d4", + "environment/gates.txt": "032786f94f16605767c62bb61ce4dbba7ab7d3f1046712c975f01c9a04b6ad29", + "environment/tests/sim.c": "700645ac6156cdc88ec1e210f7c7d8d9969bb42aec969c410597e8efca796173", + "environment/tests/test_outputs.py": "6117b8edb7d567b30e32b16a9244bec4c131ec01e9065d52f6f53f87e36f0438", + "instruction.md": "a7f500266a878d04ac9de39fba76220cfad216d8f6b8063bba93bebb3d4ab9ea", + "solution/solve.sh": "a4ffab4183815a0eb7646abc1de1fc67695014c060d7ff16e8bca9ec94385d0b", + "task.toml": "9f5aa1f7ebbbcec90fd109121e9bd509af3cfff6b4cec701bd91947988f40972", + "tests/sim.c": "700645ac6156cdc88ec1e210f7c7d8d9969bb42aec969c410597e8efca796173", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "6117b8edb7d567b30e32b16a9244bec4c131ec01e9065d52f6f53f87e36f0438" + } + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/mteb-leaderboard:20260430", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-science", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "ed147a5bc31e9865b16848b2f01d2f9b1e677af57a7c632d7ea2311c0553ca76", + "environment/Dockerfile": "271fdb5b34520c0bf2cb6462365363ec52058f127cd9eea3f5da2589df5c1044", + "instruction.md": "06ea44106eca0405c25bab00f5b3cfeb611c872093f6c0f82ed057839d1d13de", + "solution/solve.sh": "4ed68831d4acf5d80d2d73eec1f21efd0896a6f51f23020f572cfc0af6da0410", + "task.toml": "86574bd1c6c4fefd2e64eee9c2caa53d87c4bcf2e51547912440a0ddbd0badb8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "b28f8fc8b7614801305813c402be52f4be1e75de6658ecdfdf9383aa793892cf" + } + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/regex-chess:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9296cbee592fc996c58d228b51fee303647e5ecfae82d5666d42a40cb30ec355", + "environment/Dockerfile": "2616cd775252303839dfda0ee2f047eb862952b053b0103bd6b2460ef90c11b7", + "environment/check.py": "f9c416ae9ea01dcdd5f6a43170c8434b8c0a0b389a4a30876a2048e277d191c7", + "instruction.md": "a6ed3fcc54fc92d71a84af921942422d299ca89b410dcd92cd05a53d0d06436c", + "solution/solve.sh": "b728bf0ea392a3d244afbd0818d36933c8d3e190e47743c1382cd518d518a2e9", + "task.toml": "07fe52636fcc4bc9fff25d5630af145b97a5331219d719cc8decb4ca67ea21d4", + "tests/test.sh": "8f7f00a1c4dde7fcca6c918c768f2aba432b8c4007c15989716c68091950da87", + "tests/test_outputs.py": "4983c44d21c56dd8191381eff0304afc8b87bb714527ea9036b3319c015a30b4" + } + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "native_agent_timeout_seconds": 12000, + "work_budget_seconds": 11820, + "verifier_timeout_seconds": 12000.0, + "docker_image": "alexgshaw/build-pov-ray:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "38380f64bc631916e264f80ce4d37feedbc783310001475b367d03989d114fdf", + "environment/Dockerfile": "44f2b5fc7311d8f336daaaca99dd4216b49355e67f50b494ebacf1b0f7c14e92", + "environment/deps/illum1.pov": "5a1f7f53ee0730c4923fcd21cd4a9c34719024d8557920441551ba7d96e75dc4", + "instruction.md": "56c8ecdd0084a07f8c97986c43af5c37ed03dd5aa994da6735cf59806162d1b2", + "solution/solve.sh": "3da9dbf589656512cd7a7004dc3752d3552a6af939cfe59eff79f91695fa42c7", + "task.toml": "b90a47ec26c1191d3c523cc094ea96b170e42fbfe176ef1f814c57d05390c097", + "tests/reference_illum1.png": "bb084f430bbc9d5b623a656920670dc06cee6d9132a9d550be4a1b99e41ab9e0", + "tests/test.sh": "928d1497ad21aef0c456294e3784f1fddf8493410f23584da89f9e02ec6bd134", + "tests/test_outputs.py": "64d6ce107503714ada346e5606832ee886be9950971cf81523e73d8ba28e8d37" + } + } + ], + "total_model_runs": 34, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/adaptive-rejection-sampler.json", + "config_sha256": "61f6ce0294dca33165eb715ac90dc3b262dc3c29b1f16699e17d72d4749e5274", + "job_name": "dev34-1017-adaptive-rejection-sampler-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-cython-ext", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-cython-ext.json", + "config_sha256": "f1e37ab1ed2f9b4648b449eb3872c8fbbb212b5d254218194c378c9e37a34a5f", + "job_name": "dev34-1017-build-cython-ext-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cancel-async-tasks", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/cancel-async-tasks.json", + "config_sha256": "a6136d04c394a97d37614c5e18d01016e1ca87f92e2280c9161674bf7af05e53", + "job_name": "dev34-1017-cancel-async-tasks-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cobol-modernization", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/cobol-modernization.json", + "config_sha256": "972c8574ba792dc705688ac7fcf6f292e1dc272e8f268cde879bfafc0196b218", + "job_name": "dev34-1017-cobol-modernization-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/db-wal-recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/db-wal-recovery.json", + "config_sha256": "38f02618b07309a96f0e31317443461e53624fb56153888bfe02348eeb88b4dc", + "job_name": "dev34-1017-db-wal-recovery-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/fix-code-vulnerability", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/fix-code-vulnerability.json", + "config_sha256": "85bb7c9285571296a7aa3186e69aa57df10264bd3f1bd3ce2c00d435ab40863b", + "job_name": "dev34-1017-fix-code-vulnerability-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json", + "config_sha256": "09db3a5a292440a611a1da389c422dc404853d21d8be54ad994d135939662f11", + "job_name": "dev34-1017-git-multibranch-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/kv-store-grpc", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json", + "config_sha256": "22c29825f0762fb05628119d5a4a47f96038c7d05058ac61de0e0b5300987e17", + "job_name": "dev34-1017-kv-store-grpc-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/log-summary-date-ranges", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json", + "config_sha256": "4edf8f4c1c500ab2ec690aecdb5481dcaa76d7327b972e07c01a0472dd70e3dc", + "job_name": "dev34-1017-log-summary-date-ranges-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/merge-diff-arc-agi-task", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/merge-diff-arc-agi-task.json", + "config_sha256": "c2409f07940bec07a25e852ed89785e1af02b4d81b2188961e6504bd6a2963e6", + "job_name": "dev34-1017-merge-diff-arc-agi-task-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/model-extraction-relu-logits", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/model-extraction-relu-logits.json", + "config_sha256": "bd4d1341bea166384900f8aafd2e745c54f910db8bf7a0d906f5ffc1da9abf8d", + "job_name": "dev34-1017-model-extraction-relu-logits-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/multi-source-data-merger.json", + "config_sha256": "b708c26001fed1c2fed7ee0bdb2e996d6bd0397586e011e69450ea62c4502c27", + "job_name": "dev34-1017-multi-source-data-merger-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/nginx-request-logging", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/nginx-request-logging.json", + "config_sha256": "d9564cfe19e4351903a14b107ba2512a863144a94ca1cb1c3bdb4e9c43a27ddf", + "job_name": "dev34-1017-nginx-request-logging-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/openssl-selfsigned-cert", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/openssl-selfsigned-cert.json", + "config_sha256": "9ef85ea697e55e2a45385b0c76a661d3f309ec96f5854c99ede5c167c4840182", + "job_name": "dev34-1017-openssl-selfsigned-cert-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pypi-server", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/pypi-server.json", + "config_sha256": "aab0c2c0acc85ab7b1bca03f53e6643aedc8b14f438e4e7e6fc697365da8c207", + "job_name": "dev34-1017-pypi-server-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-cli.json", + "config_sha256": "86d722dc6fc065087c7c9f88039007f7d984f08fc6a21b8754a1964061de22be", + "job_name": "dev34-1017-pytorch-model-cli-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-recovery.json", + "config_sha256": "2d22a0df5b6ea299fcfbf68ece763cab6b048d67a6c3c99a1b03331fe4254dcb", + "job_name": "dev34-1017-pytorch-model-recovery-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/qemu-alpine-ssh", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/qemu-alpine-ssh.json", + "config_sha256": "77920fbee85e0a4109e065171224f9350e73ef9c26906d695a6184068655c3e9", + "job_name": "dev34-1017-qemu-alpine-ssh-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/query-optimize", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/query-optimize.json", + "config_sha256": "339d21d9f22e4a7c90f16042e388556ed4bde0818bf52e8e1515d47a44c01013", + "job_name": "dev34-1017-query-optimize-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/regex-log", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-log.json", + "config_sha256": "7a98a17dc177dcc27b7b610210043f91ea6b33cb965a550690eddbf9ac57c886", + "job_name": "dev34-1017-regex-log-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/torch-pipeline-parallelism", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-pipeline-parallelism.json", + "config_sha256": "5c7b6d41d8c239a00f91d1eab67f710d09c256c9f82657d389b19634579814c2", + "job_name": "dev34-1017-torch-pipeline-parallelism-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/torch-tensor-parallelism", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-tensor-parallelism.json", + "config_sha256": "d7e752886c5972c8c06746709a98276853da6b17a1361815778fba31e45793af", + "job_name": "dev34-1017-torch-tensor-parallelism-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/write-compressor", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/write-compressor.json", + "config_sha256": "e095d60ba13c31cb46eebc64108a3f9046f004609c560040a00438fffaff5857", + "job_name": "dev34-1017-write-compressor-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/constraints-scheduling", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/constraints-scheduling.json", + "config_sha256": "0356b8b7a0c0529ee54194649ff98d12d878cf5c6a87b3fe9b1452d348d31b23", + "job_name": "dev34-1017-constraints-scheduling-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/custom-memory-heap-crash", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/custom-memory-heap-crash.json", + "config_sha256": "e182a58f586ce9a4452e7a5f005b6a0b35a21c0b0bbccf0a288cc561fb8035a1", + "job_name": "dev34-1017-custom-memory-heap-crash-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/dna-assembly", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dna-assembly.json", + "config_sha256": "0fa30910ad30e182bcc8b9d39cd62b364e7e44d092ae5dbfda6cce293f33682e", + "job_name": "dev34-1017-dna-assembly-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/llm-inference-batching-scheduler", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/llm-inference-batching-scheduler.json", + "config_sha256": "8671502ee1568ed2b63a35c6d25ebd85aee25de8df4e7750ceb7644a097e891e", + "job_name": "dev34-1017-llm-inference-batching-scheduler-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/path-tracing", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/path-tracing.json", + "config_sha256": "0b6814bb9b09b23cc77406914897d33182fff8ed0e26c602cc203c5731a1a40d", + "job_name": "dev34-1017-path-tracing-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/schemelike-metacircular-eval", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/schemelike-metacircular-eval.json", + "config_sha256": "ab39b406a1793049af65f8590875c1a63169ec3efbd3214fe7fee0bd81b09f60", + "job_name": "dev34-1017-schemelike-metacircular-eval-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/caffe-cifar-10", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/caffe-cifar-10.json", + "config_sha256": "38e430910a58b740f0ca7a67c9873b675b4f65f1ebd9a70a40fd21a35ff1ade4", + "job_name": "dev34-1017-caffe-cifar-10-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/circuit-fibsqrt", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/circuit-fibsqrt.json", + "config_sha256": "546c8cc4416c6b23ad5fbd87ae1b660893e5225da9bbeff66a989134b4fe16ad", + "job_name": "dev34-1017-circuit-fibsqrt-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/mteb-leaderboard", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/mteb-leaderboard.json", + "config_sha256": "7ed551ce3442a0e98b9986f3b8ba399e2dac0fa5ca3d058ed048658abf1d6289", + "job_name": "dev34-1017-mteb-leaderboard-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/regex-chess", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-chess.json", + "config_sha256": "f82c737563dad901b2991908556193c7a8f3fbe6dd46736625b484f4c2613aef", + "job_name": "dev34-1017-regex-chess-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-pov-ray", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-pov-ray.json", + "config_sha256": "5856142204e35e53e7fd9215b8da5cd3cea24d590f505243c4d61d575b417a43", + "job_name": "dev34-1017-build-pov-ray-main-20261019" + } + ], + "preflight": [ + { + "arm": "main", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/install-only.json", + "config_sha256": "66e161eff814f0f6f4892aed51b9586a812c7cee36cac8fc42bd1df08754f2aa", + "job_name": "dev34-1017-install-only-main-20261019" + } + ], + "selection_file": "selection.json", + "selection_sha256": "0eeba4c1efaaf6fbea1f0a257fe8f00a0033428d97c04d760d33c3cccebe0016", + "image_digests_file": "image-digests.json", + "log_dir": "jobs/dev34-1017-main-20261019-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json": "3970b156e2f4d429d09aea068a3aab2707f21df17902235ca4423c11e881e417", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/run_comparison.py": "04f2106f0099a7b4d77e9206a36767f98a5e92c875ec78dd2de858c4936c12a8" + }, + "budget_policy": "Same as the October 12 baseline: native agent timeout minus 180 seconds, separate 1800-second setup, 100 replies, 65536 output tokens per reply. Native verifier per-attempt limits; only timeouts may retry, at most three total attempts on the same output. Temperature and reasoning effort unset.", + "preflight_policy": "Fresh install-only checks for all 34 tasks before any model task. QEMU has the reviewed qemu-alpine-ssh-v1 verifier profile; the other 33 are explicitly not_configured. Agent installation success does not certify unsupported verifier dependencies.", + "information_policy": "All tasks have previous development exposure. Supply original task instructions and current main prompt only; no earlier diagnoses or fixes. Review operational metadata while running. Freeze all new model outputs before reviewing fresh trajectories or hidden-test content.", + "queue_policy": "Ascending work budget then task name, consecutive pair barriers, maximum concurrency two, one fresh model trial per task, zero whole-trial retries and no result-based replacement.", + "source_policy": "Agent and adapter are evaluated from a detached, unmodified checkout of the main commit observed at registration. The standard entry point independently snapshots sources and verifies each wheel.", + "prior_baseline": "dev34-main-baseline-20261012", + "scope": "Fresh full development-pool baseline on current main and Flash 1017. Previous task outputs and rewards are not reused. This is not a held-out score or a controlled estimate of checkpoint effects.", + "image_digests_sha256": "fd9abd24bcc397d172ac0ad4e3faa875a2da8c482b35651a2ab296e04c3adcc4" +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/fix-code-vulnerability.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/fix-code-vulnerability.json new file mode 100644 index 0000000..c5a9244 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/fix-code-vulnerability.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json new file mode 100644 index 0000000..be0db8b --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/image-digests.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/image-digests.json new file mode 100644 index 0000000..722c18c --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/image-digests.json @@ -0,0 +1,311 @@ +{ + "observed_at": "2026-10-19T13:14:02.324661+00:00", + "images": [ + { + "task": "terminal-bench/adaptive-rejection-sampler", + "image": "alexgshaw/adaptive-rejection-sampler:20251031", + "image_id": "sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f", + "repo_digests": [ + "alexgshaw/adaptive-rejection-sampler@sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f" + ], + "size_bytes": 29731756 + }, + { + "task": "terminal-bench/build-cython-ext", + "image": "alexgshaw/build-cython-ext:20251031", + "image_id": "sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594", + "repo_digests": [ + "alexgshaw/build-cython-ext@sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594" + ], + "size_bytes": 303222492 + }, + { + "task": "terminal-bench/cancel-async-tasks", + "image": "alexgshaw/cancel-async-tasks:20251031", + "image_id": "sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d", + "repo_digests": [ + "alexgshaw/cancel-async-tasks@sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d" + ], + "size_bytes": 44171676 + }, + { + "task": "terminal-bench/cobol-modernization", + "image": "alexgshaw/cobol-modernization:20251031", + "image_id": "sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0", + "repo_digests": [ + "alexgshaw/cobol-modernization@sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0" + ], + "size_bytes": 162020401 + }, + { + "task": "terminal-bench/db-wal-recovery", + "image": "alexgshaw/db-wal-recovery:20251031", + "image_id": "sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "repo_digests": [ + "alexgshaw/db-wal-recovery@sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e" + ], + "size_bytes": 175276145 + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "image": "alexgshaw/fix-code-vulnerability:20251031", + "image_id": "sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0", + "repo_digests": [ + "alexgshaw/fix-code-vulnerability@sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0" + ], + "size_bytes": 190696453 + }, + { + "task": "terminal-bench/git-multibranch", + "image": "alexgshaw/git-multibranch:20251031", + "image_id": "sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "repo_digests": [ + "alexgshaw/git-multibranch@sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5" + ], + "size_bytes": 222134454 + }, + { + "task": "terminal-bench/kv-store-grpc", + "image": "alexgshaw/kv-store-grpc:20251031", + "image_id": "sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791", + "repo_digests": [ + "alexgshaw/kv-store-grpc@sha256:3399400800dcb207634daa42bc1b052e831e285cc9d221eea66c47bc0fc79791" + ], + "size_bytes": 44171671 + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "image": "alexgshaw/log-summary-date-ranges:20251031", + "image_id": "sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51", + "repo_digests": [ + "alexgshaw/log-summary-date-ranges@sha256:cbeb6ba905c2fec294f16cd5e16e3ea7f2e04d38ac2484d51a11de262aa7dc51" + ], + "size_bytes": 44866158 + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "image": "alexgshaw/merge-diff-arc-agi-task:20251031", + "image_id": "sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4", + "repo_digests": [ + "alexgshaw/merge-diff-arc-agi-task@sha256:bfc2a235f2ceea64ffb1ffc5529c2acebf570bff61f3668f0e5669ef570c17d4" + ], + "size_bytes": 29728285 + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "image": "alexgshaw/model-extraction-relu-logits:20251031", + "image_id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "repo_digests": [ + "alexgshaw/model-extraction-relu-logits@sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce" + ], + "size_bytes": 83698971 + }, + { + "task": "terminal-bench/multi-source-data-merger", + "image": "alexgshaw/multi-source-data-merger:20251031", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "repo_digests": [ + "alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e" + ], + "size_bytes": 220880128 + }, + { + "task": "terminal-bench/nginx-request-logging", + "image": "alexgshaw/nginx-request-logging:20251031", + "image_id": "sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d", + "repo_digests": [ + "alexgshaw/nginx-request-logging@sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d" + ], + "size_bytes": 67285778 + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "image": "alexgshaw/openssl-selfsigned-cert:20251031", + "image_id": "sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12", + "repo_digests": [ + "alexgshaw/openssl-selfsigned-cert@sha256:4c948a4e630af2435ae0a19108fc0814a946ac2fa29a512469e0fc77b38c8c12" + ], + "size_bytes": 44173158 + }, + { + "task": "terminal-bench/pypi-server", + "image": "alexgshaw/pypi-server:20251031", + "image_id": "sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276", + "repo_digests": [ + "alexgshaw/pypi-server@sha256:d18bb30f47c7dcaa3acdbc31ac7413c98eadca2ea16540bbd380f2f063b95276" + ], + "size_bytes": 83726049 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "image": "alexgshaw/pytorch-model-cli:20251031", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "repo_digests": [ + "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67" + ], + "size_bytes": 44349611 + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "image": "alexgshaw/pytorch-model-recovery:20260430", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "repo_digests": [ + "alexgshaw/pytorch-model-recovery@sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e" + ], + "size_bytes": 269661811 + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "image": "alexgshaw/qemu-alpine-ssh:20251031", + "image_id": "sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964", + "repo_digests": [ + "alexgshaw/qemu-alpine-ssh@sha256:541e8214a8c4faa7b556531b899d45da21048fa2ca6dd328d8cb6c4cffa8e964" + ], + "size_bytes": 1308553944 + }, + { + "task": "terminal-bench/query-optimize", + "image": "alexgshaw/query-optimize:20251031", + "image_id": "sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "repo_digests": [ + "alexgshaw/query-optimize@sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14" + ], + "size_bytes": 120494963 + }, + { + "task": "terminal-bench/regex-log", + "image": "alexgshaw/regex-log:20251031", + "image_id": "sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9", + "repo_digests": [ + "alexgshaw/regex-log@sha256:90101b2e815323a8da20528a1439bebc407eb9761c9c68a3d557730856c878e9" + ], + "size_bytes": 29725514 + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "image": "alexgshaw/torch-pipeline-parallelism:20251031", + "image_id": "sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b", + "repo_digests": [ + "alexgshaw/torch-pipeline-parallelism@sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b" + ], + "size_bytes": 29725531 + }, + { + "task": "terminal-bench/torch-tensor-parallelism", + "image": "alexgshaw/torch-tensor-parallelism:20251031", + "image_id": "sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690", + "repo_digests": [ + "alexgshaw/torch-tensor-parallelism@sha256:c50ac169e11465f41b49749fe87e4487f24d31a0c39c0906cd3242cc5499c690" + ], + "size_bytes": 29725529 + }, + { + "task": "terminal-bench/write-compressor", + "image": "alexgshaw/write-compressor:20251031", + "image_id": "sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1", + "repo_digests": [ + "alexgshaw/write-compressor@sha256:3618e1f8a997b437c09cd3dbaac736705809c05f6f12f584b65722175970ebe1" + ], + "size_bytes": 291927726 + }, + { + "task": "terminal-bench/constraints-scheduling", + "image": "alexgshaw/constraints-scheduling:20251031", + "image_id": "sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "repo_digests": [ + "alexgshaw/constraints-scheduling@sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0" + ], + "size_bytes": 49852952 + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "image": "alexgshaw/custom-memory-heap-crash:20251031", + "image_id": "sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557", + "repo_digests": [ + "alexgshaw/custom-memory-heap-crash@sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557" + ], + "size_bytes": 1267999268 + }, + { + "task": "terminal-bench/dna-assembly", + "image": "alexgshaw/dna-assembly:20251031", + "image_id": "sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569", + "repo_digests": [ + "alexgshaw/dna-assembly@sha256:d1adf6835f1dd91205ba70e452c699d0aea601010038e5617f370716efb50569" + ], + "size_bytes": 29727501 + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "image": "alexgshaw/llm-inference-batching-scheduler:20251031", + "image_id": "sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774", + "repo_digests": [ + "alexgshaw/llm-inference-batching-scheduler@sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774" + ], + "size_bytes": 44188631 + }, + { + "task": "terminal-bench/path-tracing", + "image": "alexgshaw/path-tracing:20251031", + "image_id": "sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482", + "repo_digests": [ + "alexgshaw/path-tracing@sha256:4b44725102cd627dfa895191a5a30d16dae3c5623bb43da6459f79135db6b482" + ], + "size_bytes": 399390468 + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "image": "alexgshaw/schemelike-metacircular-eval:20251031", + "image_id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "repo_digests": [ + "alexgshaw/schemelike-metacircular-eval@sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db" + ], + "size_bytes": 44188336 + }, + { + "task": "terminal-bench/caffe-cifar-10", + "image": "alexgshaw/caffe-cifar-10:20260403", + "image_id": "sha256:929a6d631b592c82e0d0f5d4a65cf03c946dc554cfc9bb09916e82076e7d3215", + "repo_digests": [ + "alexgshaw/caffe-cifar-10@sha256:929a6d631b592c82e0d0f5d4a65cf03c946dc554cfc9bb09916e82076e7d3215" + ], + "size_bytes": 213171982 + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "image": "alexgshaw/circuit-fibsqrt:20251031", + "image_id": "sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a", + "repo_digests": [ + "alexgshaw/circuit-fibsqrt@sha256:29783439f529eaed2145592f15af7a2281161528860392a382c825489030ae3a" + ], + "size_bytes": 149136738 + }, + { + "task": "terminal-bench/mteb-leaderboard", + "image": "alexgshaw/mteb-leaderboard:20260430", + "image_id": "sha256:a8538f4882ff132e0a20a75be3fffc07164318a7425ad19a6b4bf5268107feb4", + "repo_digests": [ + "alexgshaw/mteb-leaderboard@sha256:a8538f4882ff132e0a20a75be3fffc07164318a7425ad19a6b4bf5268107feb4" + ], + "size_bytes": 663825427 + }, + { + "task": "terminal-bench/regex-chess", + "image": "alexgshaw/regex-chess:20251031", + "image_id": "sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c", + "repo_digests": [ + "alexgshaw/regex-chess@sha256:e3b6d61d2d3930fbbdc5f7bf091a7f8b7f6125e1951ef8a7dbe165b1ef62942c" + ], + "size_bytes": 55625584 + }, + { + "task": "terminal-bench/build-pov-ray", + "image": "alexgshaw/build-pov-ray:20251031", + "image_id": "sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b", + "repo_digests": [ + "alexgshaw/build-pov-ray@sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b" + ], + "size_bytes": 163066497 + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/install-only.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/install-only.json new file mode 100644 index 0000000..9a296d5 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/install-only.json @@ -0,0 +1,168 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12" + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf" + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1" + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3" + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f" + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67" + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d" + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c" + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302" + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c" + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78" + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a" + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb" + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684" + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + }, + "install_only": true +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json new file mode 100644 index 0000000..de195fd --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/llm-inference-batching-scheduler.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/llm-inference-batching-scheduler.json new file mode 100644 index 0000000..b455c56 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/llm-inference-batching-scheduler.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1620 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json new file mode 100644 index 0000000..0631937 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/main-prompt.txt b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/main-prompt.txt new file mode 100644 index 0000000..56116f5 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/main-prompt.txt @@ -0,0 +1,9 @@ +You are nanoPyCodeAgent, a coding agent running non-interactively on a task handed to you up front. There is no user to reply to you: never ask a clarifying question, never stop to wait for confirmation, and never present a plan for approval — decide on your own and carry it out. Work the task through to the end, then check the result with the tools instead of assuming it worked. When it is done, answer with a short summary and no further tool calls: that reply is what ends the run. Runtime budget reminders report remaining time and model replies; when either is running low, prioritize saving the required deliverables, necessary verification, and a final summary. Do not start optional work after the requirements are satisfied. + +Identify the required deliverables, interfaces, and constraints. Treat examples as evidence, not as the full contract unless the task says so. Inspect relevant files, dependencies, and existing checks instead of guessing. Task-supplied workspace files, including source, executables, and tests, are available references unless the task restricts their use. Consult general documentation as needed. Do not seek or use task-specific reference solutions or hidden evaluation tests from outside the supplied materials unless the task explicitly requests them. Derive checks from the stated requirements. + +Verify the requested behavior through the real entry point when feasible. Check the content and meaning of outputs, not only command success or file existence. For logic or numerical work, choose a few representative and boundary cases justified by the task, using an independent reference, invariant, or consistency check when available. Do not derive every expectation from the implementation being tested or weaken requirements to make a check pass. Keep verification proportional to the task and remaining budget. + +When a check fails, inspect the error and relevant output, revise the underlying assumption or implementation, and rerun the affected check. Keep the task's constraints intact. Once appropriate checks support the required behavior, save the deliverables and finish. State what was verified and any remaining limitation without claiming unobserved success. + +Prefer the read tool for viewing files, the edit tool for changing part of an existing file, and the write tool for creating files or rewriting them whole. Use the bash tool to run commands, search with grep, and complete tasks that need real command output instead of guessing. \ No newline at end of file diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/merge-diff-arc-agi-task.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/merge-diff-arc-agi-task.json new file mode 100644 index 0000000..b8929e9 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/merge-diff-arc-agi-task.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/model-extraction-relu-logits.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/model-extraction-relu-logits.json new file mode 100644 index 0000000..24f3f58 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/model-extraction-relu-logits.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/mteb-leaderboard.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/mteb-leaderboard.json new file mode 100644 index 0000000..f9fde41 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/mteb-leaderboard.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 3420 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/multi-source-data-merger.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/multi-source-data-merger.json new file mode 100644 index 0000000..22172d9 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/multi-source-data-merger.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/nginx-request-logging.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/nginx-request-logging.json new file mode 100644 index 0000000..6a716ca --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/nginx-request-logging.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/openssl-selfsigned-cert.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/openssl-selfsigned-cert.json new file mode 100644 index 0000000..0b32251 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/openssl-selfsigned-cert.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/path-tracing.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/path-tracing.json new file mode 100644 index 0000000..381c895 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/path-tracing.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1620 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pypi-server.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pypi-server.json new file mode 100644 index 0000000..4f6a676 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pypi-server.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-cli.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-cli.json new file mode 100644 index 0000000..ba87889 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-cli.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-recovery.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-recovery.json new file mode 100644 index 0000000..3cd0e77 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-recovery.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/qemu-alpine-ssh.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/qemu-alpine-ssh.json new file mode 100644 index 0000000..9511975 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/qemu-alpine-ssh.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/query-optimize.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/query-optimize.json new file mode 100644 index 0000000..561ad59 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/query-optimize.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-chess.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-chess.json new file mode 100644 index 0000000..6e6c5bb --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-chess.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 3420 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-log.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-log.json new file mode 100644 index 0000000..7656d9d --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-log.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/schemelike-metacircular-eval.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/schemelike-metacircular-eval.json new file mode 100644 index 0000000..fec6adb --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/schemelike-metacircular-eval.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 2220 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/selection.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/selection.json new file mode 100644 index 0000000..10658f2 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/selection.json @@ -0,0 +1,150 @@ +{ + "schema_version": 1, + "selected_at": "2026-10-19T13:13:43.549137+00:00", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "catalog_sha256": "7fc7d95a2077ede76c4f47890a09de93c2fe558252a122ef830288ad293ceaaa", + "selection_method": "Entire unchanged 34-task development pool, requested by the user for a fresh baseline on current main with Flash 1017.", + "prior_selection_file": "../dev34-main-baseline-20261012/selection.json", + "prior_selection_sha256": "4572b1cc432a16b9c5d1dd554c8d2189ae80414f386a90bf03829bc4d0c3da85", + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12" + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf" + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1" + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3" + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f" + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67" + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d" + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c" + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302" + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c" + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78" + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a" + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb" + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684" + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e" + } + ], + "prior_exposure": "All tasks are development tasks with previous outcomes and some reviewed traces/tests. This is not a held-out capability evaluation. No fresh trace or hidden-test content is reviewed until every new agent output is frozen.", + "remaining_unopened_tasks": 55 +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-pipeline-parallelism.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-pipeline-parallelism.json new file mode 100644 index 0000000..8a1a017 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-pipeline-parallelism.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-tensor-parallelism.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-tensor-parallelism.json new file mode 100644 index 0000000..0f8b367 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-tensor-parallelism.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/write-compressor.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/write-compressor.json new file mode 100644 index 0000000..8fed68d --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/write-compressor.json @@ -0,0 +1,35 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json" + ] + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": false + } + } +} From a6d41c655a80e894c7289b6d7839753153d7ef15 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 19 Oct 2026 06:45:08 -0700 Subject: [PATCH 02/12] chore(benchmarks): record dev34 installation preflight evidence --- .../dev34-flash1017-main-20261019/README.md | 23 +- .../experiment-v2.json | 1681 +++++++++++++++++ .../preflight-evidence.json | 641 +++++++ 3 files changed, 2342 insertions(+), 3 deletions(-) create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md index c37eadd..6481340 100644 --- a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md @@ -7,7 +7,7 @@ source. Historical scores and outputs remain separate. ## Frozen inputs and limits -- [Plan](experiment.json): source, model mapping, task and infrastructure hashes, +- [Active plan](experiment-v2.json): source, model mapping, task and infrastructure hashes, budgets, installation preflight, and the complete fixed queue. - [Selection](selection.json): the unchanged dev34 pool and pinned task revisions. - [Prompt](main-prompt.txt): current main's 385-word headless prompt. @@ -37,6 +37,23 @@ same output and original per-attempt limits. Dependency-error recovery remains disabled. Setup failures and verifier errors are recorded separately from scored failures; a zero without executed assertions is analytically unscored. +## Installation preflight history + +All 34 tasks now have fresh, successful native installation-only records from +this frozen checkout. The first batch completed 33 installations and recorded +one `NetworkConnectionError`: TLS failed while downloading the unchanged +`https://astral.sh/uv/0.9.11/install.sh` script for the Scheme evaluator. A new, +separately named installation-only trial for that task succeeded. No model or +official verifier ran during either preflight job. + +The [original registration](experiment.json) and all 35 attempts remain intact. +[Native evidence](preflight-evidence.json) binds all results, bootstrap records, +source manifests, and input configurations by hash. The active v2 plan accepts +exactly one successful native record per task through the existing strict +orchestrator checks. Model configurations, task revisions, environment transport, +source, budgets, and timeout-only verifier policy remain unchanged. QEMU's +registered verifier profile passed; the other 33 remain `not_configured`. + ## Execute From this experiment worktree's repository root, supply credentials through the @@ -45,11 +62,11 @@ and leave `ANTHROPIC_MODEL` and `ANTHROPIC_MAX_TOKENS` unset. ```bash python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json \ + --plan benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json \ --phase preflight python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json \ + --plan benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json \ --phase model ``` diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json new file mode 100644 index 0000000..981851b --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json @@ -0,0 +1,1681 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-19T13:44:55.715021+00:00", + "experiment": "dev34-flash1017-main-20261019", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-19T13:13:43.549137+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "main": { + "source_root": "jobs/main-source", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/adaptive-rejection-sampler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "scientific-computing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "95408217b9c484ffc18a02a91b7706323b6902919c72b24866a32e27a23543e2", + "environment/Dockerfile": "e644a3343e339303230f1441b7383f076d21c69a5a7aa6430b4f8a9d27f15c8b", + "environment/protected.tar.gz.enc": "7fc9bf1347bb97da8dede04b2c58638f225070913e222d76627068aebd1b1df2", + "instruction.md": "e6d28c28acd76bef61817427c74c2a29e3e8d75f535cbd708afcbfd02c54013f", + "solution/solve.sh": "fe443a8082b5183ba1623311ba0ee958888aefa633eff93766b9ae14e9bdd84d", + "task.toml": "6ed7b37f74beb46e9d47b0f5fad53025667c9548baff4e96d467b56d429fab1a", + "tests/test.sh": "38b43560d173cc2b952c3a3e17b8a480216d84e33450515e047bcb0d806b1e0a", + "tests/test_outputs.py": "547dc6e107f034f41703722aeceb6d0236e3fb69116fc3f2fbaba11884de352f" + } + }, + { + "name": "terminal-bench/build-cython-ext", + "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/build-cython-ext:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d5f5113941c3963292bae396baa9369571026f46e59437f98ddcc93475dd7729", + "environment/Dockerfile": "cb6cf421cb9b69d43bd25cd491e49bda982fbb865d47515178d017eda727194e", + "instruction.md": "7e52874e45f88505bd21540229719de6639a401477557d814489981d472f0e34", + "solution/solve.sh": "41bbdde88617070366a70e518eedca466a4ea1acd7e96899b2fb7da988e40abc", + "task.toml": "c472f0e7522c128fb7f03e00b50c8cdf2e5e6474daa1e094e7441f4b54e770b2", + "tests/test.sh": "0156c9c4085258b696ce424cf1ce96cba44d1263c5d73bdf54083fd80237f915", + "tests/test_outputs.py": "6549f1a8e04b6163ec08877e6f6b9868bf94563f3a3c72ac1fba9c512c31689e" + } + }, + { + "name": "terminal-bench/cancel-async-tasks", + "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/cancel-async-tasks:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9adf86b1279cc44f83fe16e6d340595aa992240f1d87243d204cae32e453693b", + "environment/Dockerfile": "1ee7bdb4a7536f5f114c11a3ac6ff42790abfea901fcba6b2a8ea8685f19140e", + "instruction.md": "23d22f7039b50682941652c716f96dac314b8585c32c133614cac77dc067f47e", + "solution/solve.sh": "4938ca4ad5b7dcd36b414b42a1abb10c058ecefce7462b678f9bd05c9fae1844", + "task.toml": "14db0675e04408f2133b0067e8dbddf66a1d6e7baf6b8d38bef8ecccfc4e58a8", + "tests/test.py": "f533c0a64db068f582b4733c5b81f2e66b8fa52fcd3efa65752e6bd6d24780aa", + "tests/test.sh": "79600a6b863def31b4daa008297423010cd2d6243e689d67503e118e4dc53397", + "tests/test_outputs.py": "d6f510b51fafc81b4e9b5cc95535aa13491c8753e0e4171db42c45f4de460ee7" + } + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/cobol-modernization:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "c89318cf59efcc7040e3d946022e2616161b6733931a080cdeea7b1918e7ecc4", + "environment/Dockerfile": "8b175c83fc03ec7d16d6101516bc5100230b592f480e435279f4fe051c9280b2", + "environment/data/ACCOUNTS.DAT": "46f63eb29442ebc6716bcd8a0c36f4367910f2783f6b84a307ed9a61ccd1046e", + "environment/data/BOOKS.DAT": "db04da646207cd87f8e9c5b6f0b57d95953899f3eef22de1a81423f06ef4a9fd", + "environment/data/TRANSACTIONS.DAT": "1fa4cc5420ded94d585c563e8196b85166ebbae2bb8924b20014717fb2fb9ee3", + "environment/src/INPUT.DAT": "52fc984f0cf14af1495285d7efe02b9bae13159eea59255560ee938c4f78afee", + "environment/src/program.cbl": "27cb0e56f65f52fa75f346720b37a180b8e034ebe7d7622b979f195087bd6ebb", + "instruction.md": "a87074f1141b7345394cd88053d0debb0dc8f7de95308882c6538ee9a3b359da", + "solution/solve.sh": "7d4a0504e1686c1790bfbc074d4e598f4c631f54c2bc5f3d5fd786574b4810b9", + "task.toml": "4b620211dc3d6b1b989f0210c5ecd2f606974581c8bce27bd78659d48cf0cfbd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "009180a7092583fc8143bb3ffc3957923aa0d61d0aac3a471f76eeb4aae54a88" + } + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/db-wal-recovery:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "file-operations", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "600762031a1a02ecc3b46d70f7683f3ff930b47c8a3d0e27cc31b146158c3126", + "environment/Dockerfile": "63a260531c62f4ae72060db17f295c0aa03cfc6321f8162c324b7aa6b3f81302", + "environment/main.db": "68cb288eb29e76d814b4e7ee6887b37c415675e644f88173bc191348e290d119", + "environment/main.db-wal.encrypted": "68753bdce26196a53194260d001544cc6bb8f9f38e2f2f6826d09798e31e6e26", + "instruction.md": "5a20375162921b8911c67035d1905df7fd335d1bed03044271e608b2c3c93b26", + "solution/solve.sh": "ed8530c94103bf19783a1d41bc63fe4db478cdd9629f22fee25c45a45699f5e8", + "task.toml": "1cdc2c615e259bc10265efbe71b518332070c1c002e226566399e9788f18d0aa", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "88dc5c2a3abba75bca0e0f9136223ae5bb585d5984b8d9eba51a7829c1d2ba49" + } + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/fix-code-vulnerability:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "security", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "87f28dbad183c3a634887696d328ca68a8a0858f132bc48cb7341c22267d38f7", + "environment/Dockerfile": "0917470986224df4daf4854a71302ae81c73ba5dd6e25be1232d29b758a5b3ab", + "instruction.md": "89b1abdf0af19399f720233b5cdf6b2e8e0ff7b6e3fe3e2a3af0d4d298a8641a", + "solution/solve.sh": "7a0131b369d83ae8546ecc3f9b4e23d38240f1bd1f0941e4430fa85b5182ee03", + "task.toml": "9aa2bd14ce9408289d199d0608bc3fa65b8b5e23374c9641020de62197dbf1e2", + "tests/test.sh": "f1611da693cb4cfc5e8482698c6a1858a5c67220ae9db89b50aed66584e9b17f", + "tests/test_outputs.py": "c006a5de95b69bd78055aa26cc055734dbd50718ede25d331f767bfee4ef7f26" + } + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + } + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/kv-store-grpc:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "51fdc2ad06f0adb1270164f247ea41799b2ee2ad15490e48ddbf5e79d3767aaa", + "environment/Dockerfile": "47c9982789092437f950736110b2bd18899c39b8fd20f3af3c95b848e564a197", + "instruction.md": "f4b3facc199d5b1c78db6e603ffa29ec1b93a77ff7d291e1a92b42e6bf582bed", + "solution/solve.sh": "6bee2938ac9731d1156b0ca02b28c3d5793bf2cbe5f995c863d397093af01cb7", + "task.toml": "6ff0f30a526bfff86a67b0b53439b9965229c0f368167dc1a5d05f34d82299a3", + "tests/test.sh": "c348467a076bf614d0a850e25b2263f85bbf7b4c7a8d344fbd5454a0611a0df8", + "tests/test_outputs.py": "115de2e964bbe689d07930527113422ef8a9e4be4eb958ad84594e3fcd2b1b0e" + } + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/log-summary-date-ranges:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "026000415cb577f3d3d5132902670262f7d5d8f69841903ec06a882292f756b4", + "environment/Dockerfile": "b479f6e987b63a6ec118fa1691b8bb85c437b2a6a0fe63feab5ed7154957fdfc", + "environment/log_generator_deterministic.py": "beb6eff59979e9aa0be63ce8a0f1b0df1178f92871f04ba9f76fa70cf8beb81c", + "instruction.md": "c1d4516ad4ec2238209740eaf01291df9ab48b0028a3fc5df383c6f831453484", + "solution/solve.sh": "9f6a8ee8947bdf1c0f31b530b6194ae998d66841a3b785fa2e246ee840350ab6", + "task.toml": "9a9593673d6805ef3a301ea734e9c4dbaf80da1b37c0fe9d1f2ceea2a8c57e18", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2fc06a6ce1944ad82f84fb437395fa75ed8c0eadd385b92e375c27e5bc9cb2b4" + } + }, + { + "name": "terminal-bench/merge-diff-arc-agi-task", + "ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/merge-diff-arc-agi-task:20251031", + "resources": { + "cpus": 1, + "memory_mb": 4096, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d6b48ed0890b12eb68f72c12cd02689bd6bfae2191dd2688949cde6fe77f6a94", + "environment/Dockerfile": "4a03ae6f0c60b9c2f45c33f0413b5aa75c9811870254e3ff1968fd0119173fc6", + "environment/task-deps/bundle1.bundle": "a759f9b6ffc1a2eee50778d72a1899ad7d724becab84fa30e7aac71ee9e443a5", + "environment/task-deps/bundle2.bundle": "75e3e3916a01c980a03715de67b16c729e03661d94a694b5f4223d229bd6593a", + "environment/task-deps/examples.json": "5426569e4ea078bbb74039ec712a2d4d35f1f9fa3a5f26bf2a0de4a2c8ef5db8", + "instruction.md": "2d826479873e5ac1c94c1f802763bf85c22bb04624384685ebdf6b153fe0278e", + "solution/solve.sh": "48d4becc41fda5b6063fe96947474ffe778a29d6924d7b2bb235eb61e5ce555d", + "task.toml": "6ced40e06b43b169857509765eb7f07791db558f9437cc7367d2536771c2ca5b", + "tests/test.json": "20f116a6b718bbf4874bbe203d15a5c0f85a1d88cb622e2d7529c7e4fb2d4c28", + "tests/test.sh": "c9052954cd6ce25c4a375a2fb48e765b1efa5b39f7991b93993cc793af8ab9d2", + "tests/test_outputs.py": "2754ba6c5c5f75c9db5ed2c5729d9f5d7e21cb7fdb18aa0c65abdc3e7e85c149" + } + }, + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/model-extraction-relu-logits:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "mathematics", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9465accba6c57e5de9a196eed776a16eeb0852a8afc18fc3f8979e2de71b0c7b", + "environment/Dockerfile": "598b1f80d3b7194e97cbf02c8d245de30e303909e0cbe6465125d1d3179fddb5", + "environment/forward.py": "af1a23efd8e4390d16a29777c28116b2f8623ae3f7b628fe14db9f94dac26f5b", + "instruction.md": "c9e179dd099065a34c362a865b0eb9fab5f7d96be582cf117e7519580c036ebe", + "solution/solve.sh": "4cea5094f074ff21453d4423aa1b415a104ed90caa55a4acbb4f09941a3994ea", + "task.toml": "eb64dfaf8ae88868911ae28855cdc4f767de464001e2704919015b595be6dc29", + "tests/test.sh": "3b2d8f5e0353c2bab3515d6387468a863a4f9f241ee131c73f545f3fdf9e2d52", + "tests/test_outputs.py": "e5189c7eb9942ede55e1b48a80388590b80a486b75497613f257b3373ad9b2c9" + } + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/nginx-request-logging:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d79997586d1c14ed8dbfea050e3580b25500e04c8345acfd3f294152433c795a", + "environment/Dockerfile": "43003466d89ab83484a3fa618e35b58cdb74a002b1e8716ef30f12b3ac8302d8", + "instruction.md": "26a7ac98aced6107f147206790ada77bbf8ed3c25fb360cf323d5fc6889edf99", + "solution/solve.sh": "2454bbbf7c201a9a778165f286373ba2cb991a0eb1d026a8005d57ae71250dd9", + "task.toml": "b7a78d00be13656f75069197100b7b8c4fd268decf6f78929c853b3222a2729d", + "tests/test.sh": "27761195f53cbfe5941fe39b573631f6c926869e64b7c43220377e1e116528e7", + "tests/test_outputs.py": "045cc716c14efde3b0dcff5fc7c85ec5d18bfc6ce66f8b40a418fa2a3a4acda0" + } + }, + { + "name": "terminal-bench/openssl-selfsigned-cert", + "ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/openssl-selfsigned-cert:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "security", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "5dfa88651890cda06fa2a25babd3b813818767635e88bbabecc8305d5bb3942f", + "environment/Dockerfile": "f590b523038be43afcaf5291d7b9e8ed1333c48d9994e77b8dee2f2c352f0f40", + "instruction.md": "38054a3371f7c2c853478a2f9af0bff858437d5b28a9a07483238f06e566921f", + "solution/solve.sh": "58cf3c844e907745cd81fb4bd75efb3d080e2f0e82b589c56c954ae6e1756510", + "task.toml": "75c8913f0fa15890786c11999cde920b8bfc10d4ce8a14edf1e76985a2ac5800", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "90970bb3c372e8e7fb6bb9ac5f649ba73eb38e1a5ea2e922a9057a72b6b4192d" + } + }, + { + "name": "terminal-bench/pypi-server", + "ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pypi-server:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "906a096a2cc7b9afc738ff5ad5fbd605be65c103f5069c6673915223b9b187e3", + "environment/Dockerfile": "17a4e67580e4f2c6a151d04b9d74eb020666e7eec8e5259c4a5411f9e93858d0", + "instruction.md": "1c537e4a311a8e71155e121f57bfc03f742fbfb99d05314cd03e433996a28a31", + "solution/solve.sh": "1143c0eade4932ec3aeaa5013fb5e23bbcccca15dead294aa03b6af88750775a", + "task.toml": "07c39d7a733c89adadf74bb63a3e906c3694feb6c490d5fc85ac187c003fd447", + "tests/test.sh": "d2b1220e42bcfdcce87907ac5bbd69ac210a3c94926545ca2439636786e7784e", + "tests/test_outputs.py": "82c6351722e9aa984ff1d701bf57df3150ce51466df2b70afa1aa62d2650be4c" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-recovery:20260430", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e080308a05703b40edd16f53aea41a0e6b9874750ecee35bb432029d0d44bf53", + "environment/Dockerfile": "8a6b7edbb9e7c241e447f15a71e13e5d043b25169be2fd9802b84f5be3d56eb6", + "environment/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "environment/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "instruction.md": "2bef07dc28be3ab2c35ae389584034c2e4dde92c2c31b3eb6bd829628c8c3f4a", + "solution/solve.sh": "175a7802777b43e0fa4f629d71f7c68fb40706813c342aaddc72108d89bd5b2c", + "task.toml": "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd", + "tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + } + }, + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/qemu-alpine-ssh:20251031", + "resources": { + "cpus": 1, + "memory_mb": 4096, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "qemu-alpine-ssh-v1", + "task_file_sha256": { + "README.md": "72f1c83fa1e3e6381ef85dd82fd109ee18cdeac090311cf3212a0a8cff0a32a5", + "environment/Dockerfile": "015ce0601ff54758f66b33f4952e0d3a577d84c09aed8f3ec5945a570e8d168a", + "instruction.md": "8348c60490cb90bc690cba259dfe98996da4c3294c6d42bba1119b00c64de4de", + "solution/solve.sh": "1cd31186c427b91bb00499d22aa4f0214fc1ca93b40b6e4cfa969611b848db2a", + "task.toml": "10a43ec6ee732eeba852c377f12e23178f7fa5a4868143f3699a15565ffb64c7", + "tests/test.sh": "3b244479776c4ee3639631112799f2cde22cbb80ac10511715e3a8f32909c289", + "tests/test_outputs.py": "83ccb4bdcd259b342212bebefe395f289858b4dc55b2ee7ae0dab238ad2e9718" + } + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/query-optimize:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-science", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a82c53e333a0ac0b0b410b5e26cb8e7305dece31db84d340248ffb13d5a45160", + "environment/Dockerfile": "064818771f200e0517d22068f87229a7a95d0a82b0778c84ca7203bf1926a0b8", + "environment/task-deps/my-sql-query.sql": "43382792f0f438625d643e34abe144febd5669518f4ab1ab550a15c6d9314a21", + "instruction.md": "473e12d320d2732f679fb631c2a4dab4a01deea6775686718ae0c417b8a39447", + "solution/solve.sh": "0d95f3cedb287ff9d47c796cbaf812c300d1222c10fa80c49e8acb0e81eb6978", + "task.toml": "6e88f2a86df699594bcb1bf81df4ed694505f79c9420f671335b02524bbe3868", + "tests/golden.sql": "4da1885d56b3ffa7b7b22b422996d018795611f6963499d551582881b74b84fd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2068ae80c57af8d52a7ae510f0666732c057806e556b480970f36f14bae683f6" + } + }, + { + "name": "terminal-bench/regex-log", + "ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/regex-log:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "29ffd98c55dbb6f4844f1b9d8c1769b2155c117ee475e0df6bbc5ef330d239b3", + "environment/Dockerfile": "5f264ff0bf62a49aee3c6398447de5108df141e7a2ad35c6de8ef39fa62049c6", + "instruction.md": "4f7ac05e70cf9220ea0f1e5a052c5f908cd0fa884e847d80b0bd51bae2e96f9c", + "solution/solve.sh": "7e670d4f2b4bccb1e4db38f2a173e085ceda028c38167912b466b0a84fcc0999", + "task.toml": "6bc98525852755caf1fe34c174c2f10f4328adb4ce6c1968dcafe1c877e4e160", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "345c3bd09ab6f6fe8c8361a58c0a47bf0a13b3fcb38a5ac7824e44ff855e8f72" + } + }, + { + "name": "terminal-bench/torch-pipeline-parallelism", + "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/torch-pipeline-parallelism:20251031", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "297ddea07c230dfd246a8ab07337dd7abcd8e5e2a2b2e26590cb0a8af779839c", + "environment/Dockerfile": "c357ea41e4a4013c5c5980b8fef04af6a5ae250ccc6697cf3bad3127401a931f", + "instruction.md": "989bfcf0c54baf6c8cdc391743dbb2f9e858c5779dd85112536875d4d1b2dcd2", + "solution/solve.sh": "787198e4178d88b887697aefdd29819f29435ffdfe356545d92f58a30fb38b70", + "task.toml": "8b4f9f525c08afbbc4eca9168e46238a61ccb903486773e9ca4da64bb7ffe857", + "tests/test.sh": "cc9fd28b6590993fc37e4a99da771cbf04726c8c62fe308e802612241da69dea", + "tests/test_outputs.py": "23574246a407163a8e6581c84287cc2f1eb745efbaad2a317a5a28c39958c64f" + } + }, + { + "name": "terminal-bench/torch-tensor-parallelism", + "ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/torch-tensor-parallelism:20251031", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "4c1b3c3be30548f1f94c6f5503b1bc74b006af7f192dc4cd026f411df1a54604", + "environment/Dockerfile": "5f264ff0bf62a49aee3c6398447de5108df141e7a2ad35c6de8ef39fa62049c6", + "instruction.md": "13bde2ab165d72a311e2bf066d5d6c3c8761bf9d3dd39d1b23eda23bc546b910", + "solution/solve.sh": "d083b7cbf9dbb6cc2ff38c8f2acc150f0059c04aea42d6c2183aeaecc5a19385", + "task.toml": "72fd2a5500d1fe8795dcae76cccb1a29fb25894881a7475f2fa519c87d425a8d", + "tests/test.sh": "a7e6c9408b2401e4e7448371f7ddc72f3284caccfb56277d3cd4e98051868cdf", + "tests/test_outputs.py": "4ae4285e15fba1fcf13d7b7c0638ac8c2e2b0797c937abab1fb6b136fd508290" + } + }, + { + "name": "terminal-bench/write-compressor", + "ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/write-compressor:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "603015a28500a28f0838fea2c43a5168613746bfd07737920f4bae40b5a326f7", + "environment/Dockerfile": "9a583d29d46818f6ff2be7c6a0879d03343106e5376fd5a48eb5926b402bb93b", + "environment/data.txt": "0fc483d0fb31128dfa41ed11adca56c88688888189826ef9daeafd7559b4fa4e", + "environment/decomp.c": "7f8d9ef66943f912a217aa48f4ef5efc967b44c2b21d0f56df029b5ab413f62a", + "environment/main.rs": "fa53d6572fca95903e30d2d953f1e428bb8b63513ff218ebee9bc60a425ab389", + "instruction.md": "c4fc3e73e44deeac95f1cd4a34a2d87e853b22ceec2b2824767f5f087b470bb2", + "solution/solve.sh": "01550b7de7ab1320b1898eae055226bb7c73bee259c75102d2240c974a730409", + "task.toml": "6066a509066433c97c417fea68c4e3e73996f698dcdcbabe07718835eea355a6", + "tests/original-data.txt": "0fc483d0fb31128dfa41ed11adca56c88688888189826ef9daeafd7559b4fa4e", + "tests/original-decomp.c": "7f8d9ef66943f912a217aa48f4ef5efc967b44c2b21d0f56df029b5ab413f62a", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "809f486053e3ffbce852111dee1f63d80ab31297bd33172389e50278bed384e2" + } + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "native_agent_timeout_seconds": 1200, + "work_budget_seconds": 1020, + "verifier_timeout_seconds": 1200.0, + "docker_image": "alexgshaw/constraints-scheduling:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "personal-assistant", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d54a690970149e1216bd4fc066b45069c7948c13e3b1ef2a213a74342805f209", + "environment/Dockerfile": "7d9aba072a4a907229728b9739c8cf821fc2c98450babd64452682caf3759c84", + "environment/inputs/alice_calendar.ics": "5a5d45804bffd339c2479d49e76fe817b38244d14d389afe1de583443808ecbc", + "environment/inputs/bob_calendar.ics": "c97e85dbcdd756758f8a80f4db5fd5c1dcacee79298e4d2262f46de5ff680b50", + "environment/inputs/carol_calendar.ics": "32a0f8687637df0a12b8e5c56d49c4192bc3e790856943632d70f237c7c1363b", + "instruction.md": "342def78177f9bc41bea4d11af679bc385a9dda2ac8a1e425be384a5b7f937b6", + "solution/solve.sh": "802d81ec064fad3050bbade95a2f847ac24e0b7f80fa69b0bcee0347dc92fe77", + "task.toml": "7b9842de16d74e8faf8c7c0aa03b0b81145941cb12dd322d81ae23fc48607250", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "1422b345ba3aacdfc4184e962811b6379988435896c50d8cf62903e70990c5d1" + } + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/custom-memory-heap-crash:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "debugging", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a06ed23a2c467aa36d0d4a88e5c020b1076cc25b8d483517239658e9c596dde9", + "environment/Dockerfile": "6ee81e5a4fa2540443d7f27d577ba840728fc6aab9f301ab7919103673c78080", + "environment/patches/locale_classes.tcc.patch": "16951f4a4df6bf253886ac75f5a39fdbc328e415194493f63cb7589a364aa817", + "environment/patches/locale_init.cc.patch": "e97f133ceebd86f8881f972e0e45b0ec0a13a9bfa428d8032008707683d54292", + "environment/program/main.cpp": "b94538c76887cc2b2ab739b5ac4731cddd0faa8f4a9892a6265c062bbbeede14", + "environment/program/user.cpp": "92b203786fed1bf0b3aa1675c9cd2d82b2651170675064da8ffee73b98ef14f9", + "environment/program/user.h": "f43d155995a56c22b9cd1417daef7c3b5a976cdc662a98eef0feb438ae1eee77", + "instruction.md": "53c68ae14a3c3af9a97c40bfd9bdc30a5d98b37eaa2f69f763475745f70f108d", + "solution/solve.sh": "b0b1d10bea1cd5be14a82b35d522df616df1e67028f90d2a6bef90aaa98511a1", + "task.toml": "ed474be1983e212913aa8866ebb66216acbe36aef2339ac375d232aa46617481", + "tests/test.sh": "70a2b55f0c5757f6b123a216dfd202be4a646680bced28dddcd31fd8959a058f", + "tests/test_outputs.py": "4e1cb33f777fa7fb5e1536f9bea9eace6deccd515c67d4f87570adbe692914ba" + } + }, + { + "name": "terminal-bench/dna-assembly", + "ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/dna-assembly:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "scientific-computing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e7855c23d9822f688eec4dd89ca0f1caf91c6a69e1c8f3c9eb544b4ab2a94821", + "environment/Dockerfile": "26e9e43a98f0b1d132bb638f44b7dc882b0d0ca32d56fa33048adeee00ab600d", + "environment/sequences.fasta": "ebdc361b5fba28eb0a2c7a8a8a444ccb12cc842c8cbc6df5a2c24254f130736f", + "instruction.md": "88998c98beabdd966ab46c04ede822bbc74796696aa1bf87c567a9131210533c", + "solution/solve.sh": "c976cb118c8ef1befd846ba05fd5cadb1b1f529bc76b9f2ea9cb3049e6e11a47", + "task.toml": "7b571544b64d9be1c91526837e5d805c2181afe0e6fd57a7f0676bba36c868d7", + "tests/test.sh": "271c8317c98539e3b49233d017fe51fb0e5fba8114e76830fe698565499b2b1d", + "tests/test_outputs.py": "5c2d0fd2e5f6303263e9750f67acda7db491ef04841832b4fc62bcfb61b08924" + } + }, + { + "name": "terminal-bench/llm-inference-batching-scheduler", + "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/llm-inference-batching-scheduler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "machine-learning", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "ecc9dd635357c184742cf72fc1e452a8946e207b183fc9a7eebbcf07133b911f", + "environment/Dockerfile": "10a6193904ee0022ac098504182989893b196dfc74e626a3ca48534a8eacde48", + "environment/task_file/input_data/requests_bucket_1.jsonl": "8b3cf1e2b06074162038064dd2c114abe308acd32cec7d365ee63ee81978f7de", + "environment/task_file/input_data/requests_bucket_2.jsonl": "e15e2d4c4d55f875c42a6042610d6282f6c0657c39d3e0d0f087268252525cb5", + "environment/task_file/scripts/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "environment/task_file/scripts/baseline_packer.py": "547d230d5e6d93197803480cb81cab0ec6ac63fcfa0a24b246f549523f216c4e", + "environment/task_file/scripts/cost_model.py": "0cf417fd5ff0112defcd46970d37a290c0763c5a9092e91cb3ebb86c3f20e6ce", + "instruction.md": "0817374767533fbf51bb93345ee1bba266263c216c6cc55c096e71b4a1d37dfc", + "solution/solve.sh": "cb9efb586d76a893c3027e62029e286eb6c7583dc92dbf1840a3adc236a926d7", + "task.toml": "52aaac942f917e6a91ff658e0272195bd1cd842d41d71e302414c52603cc588b", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/cost_model_for_tests.py": "d3e810a99d40327fb8317f6512d879c52c6aeebdf5dcdc0c5e117c7df61c9ab9", + "tests/test.sh": "005f980332355e2fe143d4ff4aac11828adf070594bbe7055399c238f06953ca", + "tests/test_outputs.py": "80b06b7c4ba0db28832b767d48be74bb462d7f5df5798d9cb58c4f1160d96ffa" + } + }, + { + "name": "terminal-bench/path-tracing", + "ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "docker_image": "alexgshaw/path-tracing:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3e4fc952b3f9a860c7ee7837b3f3c686f1283ee7821b014265dba7c94d7fc849", + "environment/Dockerfile": "7aca02e4e20b04ed2269d627a29dd4e7337a02764a9652b641452bf09836a3d5", + "environment/orig.c": "665c1bde0907740890ac114d6b36af1f3fd00449fa287583817f778bb5da200c", + "instruction.md": "9f36e02d636f5dc0946619bc518f2e51c24fa75251da9d1f0ca9eca612cb9572", + "solution/solve.sh": "d107abde101eecee6838da80cc52a3d480c93956fff3a9343a299ea01b3fe9e6", + "task.toml": "cf47013f69bf1b43242da984f231d4daf364fdf44d3845f2eea2ca2cb9308c46", + "tests/test.sh": "a719dbb1ba7ae76df095e9d815941dfa4c55edaf11f173b9c3c3097afa838548", + "tests/test_outputs.py": "572e68b2843191d29592faf42a405861b7958d8e894868b74c5a8886b53eb879" + } + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "native_agent_timeout_seconds": 2400, + "work_budget_seconds": 2220, + "verifier_timeout_seconds": 2400.0, + "docker_image": "alexgshaw/schemelike-metacircular-eval:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "095d3ec6dcc0fe0c7a7e92355456dcea6992f91d1f8d397669b8cfe18475f506", + "environment/Dockerfile": "9b3df76222e16aad360fbd34cb0f71a0be8986052c065b10b92c0cdf417b25c7", + "environment/tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "environment/tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "environment/tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "environment/tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "environment/tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "environment/tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "environment/tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "environment/tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "environment/tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "environment/tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "environment/tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "environment/tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "environment/tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "environment/tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "environment/tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "environment/tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "environment/tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "environment/tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "environment/tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "environment/tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "environment/tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "environment/tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "environment/tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "environment/tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "environment/tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "environment/tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "environment/tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "environment/tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "environment/tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "environment/tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "environment/tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "environment/tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "environment/tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "environment/tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "environment/tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "environment/tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "environment/tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "environment/tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "environment/tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "environment/tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "environment/tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "environment/tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "environment/tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "environment/tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "environment/tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "environment/tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "environment/tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "environment/tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "environment/tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "environment/tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "environment/tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "environment/tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "environment/tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "environment/tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "environment/tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "environment/tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "environment/tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "environment/tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "environment/tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "environment/tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "environment/tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "environment/tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "environment/tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "environment/tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "environment/tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "environment/tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed", + "instruction.md": "3af5f9567b62f93683db7518ec6fcb609f83796258aa02166b5a7091c5b69031", + "solution/solve.sh": "e2c5b0ac2bef4801a1891e13c108813381877426c1a7f01bab86b4fc1f712510", + "task.toml": "8dae8eb226183b0ca92331c1106152a7abc93b03b4e5a95291b86a4ddc9b44e0", + "tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed" + } + }, + { + "name": "terminal-bench/caffe-cifar-10", + "ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 1200.0, + "docker_image": "alexgshaw/caffe-cifar-10:20260403", + "resources": { + "cpus": 4, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "machine-learning", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e918e6852c5c034422b81a6be22d7596fddcee53abe088f56727b2482da32728", + "environment/Dockerfile": "b3b8adc915a1afa035b604273af3bc1aa8b90999d2dfd72763218b34cb87abcd", + "instruction.md": "46907b0ee048f74269c29a15e0e086546b2b1220149597017da8b5ccc05a0cea", + "solution/solve.sh": "771887de1ed557269c7f3160c777519aa9538e1e9b59771b2f6ef38c8952490c", + "task.toml": "2831165ad9d9e8bbc646e6d85c0a2290640dcc4f13749547a2937c7675012a8b", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "e99b9f6284e923388fbc21488b1af99fdde46d4a809d3557b0788dbf1cb6b281" + } + }, + { + "name": "terminal-bench/circuit-fibsqrt", + "ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/circuit-fibsqrt:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "201f7c889dd6f73dd83ca078e347272b33720fa622f8c757ea6d5770453f4cbd", + "environment/Dockerfile": "780c23986778f3e5f842567868af30b1135ed095ca9449e1c90b7f0e0483d3d4", + "environment/gates.txt": "032786f94f16605767c62bb61ce4dbba7ab7d3f1046712c975f01c9a04b6ad29", + "environment/tests/sim.c": "700645ac6156cdc88ec1e210f7c7d8d9969bb42aec969c410597e8efca796173", + "environment/tests/test_outputs.py": "6117b8edb7d567b30e32b16a9244bec4c131ec01e9065d52f6f53f87e36f0438", + "instruction.md": "a7f500266a878d04ac9de39fba76220cfad216d8f6b8063bba93bebb3d4ab9ea", + "solution/solve.sh": "a4ffab4183815a0eb7646abc1de1fc67695014c060d7ff16e8bca9ec94385d0b", + "task.toml": "9f5aa1f7ebbbcec90fd109121e9bd509af3cfff6b4cec701bd91947988f40972", + "tests/sim.c": "700645ac6156cdc88ec1e210f7c7d8d9969bb42aec969c410597e8efca796173", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "6117b8edb7d567b30e32b16a9244bec4c131ec01e9065d52f6f53f87e36f0438" + } + }, + { + "name": "terminal-bench/mteb-leaderboard", + "ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/mteb-leaderboard:20260430", + "resources": { + "cpus": 1, + "memory_mb": 8192, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-science", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "ed147a5bc31e9865b16848b2f01d2f9b1e677af57a7c632d7ea2311c0553ca76", + "environment/Dockerfile": "271fdb5b34520c0bf2cb6462365363ec52058f127cd9eea3f5da2589df5c1044", + "instruction.md": "06ea44106eca0405c25bab00f5b3cfeb611c872093f6c0f82ed057839d1d13de", + "solution/solve.sh": "4ed68831d4acf5d80d2d73eec1f21efd0896a6f51f23020f572cfc0af6da0410", + "task.toml": "86574bd1c6c4fefd2e64eee9c2caa53d87c4bcf2e51547912440a0ddbd0badb8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "b28f8fc8b7614801305813c402be52f4be1e75de6658ecdfdf9383aa793892cf" + } + }, + { + "name": "terminal-bench/regex-chess", + "ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78", + "native_agent_timeout_seconds": 3600, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "docker_image": "alexgshaw/regex-chess:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9296cbee592fc996c58d228b51fee303647e5ecfae82d5666d42a40cb30ec355", + "environment/Dockerfile": "2616cd775252303839dfda0ee2f047eb862952b053b0103bd6b2460ef90c11b7", + "environment/check.py": "f9c416ae9ea01dcdd5f6a43170c8434b8c0a0b389a4a30876a2048e277d191c7", + "instruction.md": "a6ed3fcc54fc92d71a84af921942422d299ca89b410dcd92cd05a53d0d06436c", + "solution/solve.sh": "b728bf0ea392a3d244afbd0818d36933c8d3e190e47743c1382cd518d518a2e9", + "task.toml": "07fe52636fcc4bc9fff25d5630af145b97a5331219d719cc8decb4ca67ea21d4", + "tests/test.sh": "8f7f00a1c4dde7fcca6c918c768f2aba432b8c4007c15989716c68091950da87", + "tests/test_outputs.py": "4983c44d21c56dd8191381eff0304afc8b87bb714527ea9036b3319c015a30b4" + } + }, + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "native_agent_timeout_seconds": 12000, + "work_budget_seconds": 11820, + "verifier_timeout_seconds": 12000.0, + "docker_image": "alexgshaw/build-pov-ray:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "38380f64bc631916e264f80ce4d37feedbc783310001475b367d03989d114fdf", + "environment/Dockerfile": "44f2b5fc7311d8f336daaaca99dd4216b49355e67f50b494ebacf1b0f7c14e92", + "environment/deps/illum1.pov": "5a1f7f53ee0730c4923fcd21cd4a9c34719024d8557920441551ba7d96e75dc4", + "instruction.md": "56c8ecdd0084a07f8c97986c43af5c37ed03dd5aa994da6735cf59806162d1b2", + "solution/solve.sh": "3da9dbf589656512cd7a7004dc3752d3552a6af939cfe59eff79f91695fa42c7", + "task.toml": "b90a47ec26c1191d3c523cc094ea96b170e42fbfe176ef1f814c57d05390c097", + "tests/reference_illum1.png": "bb084f430bbc9d5b623a656920670dc06cee6d9132a9d550be4a1b99e41ab9e0", + "tests/test.sh": "928d1497ad21aef0c456294e3784f1fddf8493410f23584da89f9e02ec6bd134", + "tests/test_outputs.py": "64d6ce107503714ada346e5606832ee886be9950971cf81523e73d8ba28e8d37" + } + } + ], + "total_model_runs": 34, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/adaptive-rejection-sampler.json", + "config_sha256": "61f6ce0294dca33165eb715ac90dc3b262dc3c29b1f16699e17d72d4749e5274", + "job_name": "dev34-1017-adaptive-rejection-sampler-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-cython-ext", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-cython-ext.json", + "config_sha256": "f1e37ab1ed2f9b4648b449eb3872c8fbbb212b5d254218194c378c9e37a34a5f", + "job_name": "dev34-1017-build-cython-ext-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cancel-async-tasks", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/cancel-async-tasks.json", + "config_sha256": "a6136d04c394a97d37614c5e18d01016e1ca87f92e2280c9161674bf7af05e53", + "job_name": "dev34-1017-cancel-async-tasks-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cobol-modernization", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/cobol-modernization.json", + "config_sha256": "972c8574ba792dc705688ac7fcf6f292e1dc272e8f268cde879bfafc0196b218", + "job_name": "dev34-1017-cobol-modernization-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/db-wal-recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/db-wal-recovery.json", + "config_sha256": "38f02618b07309a96f0e31317443461e53624fb56153888bfe02348eeb88b4dc", + "job_name": "dev34-1017-db-wal-recovery-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/fix-code-vulnerability", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/fix-code-vulnerability.json", + "config_sha256": "85bb7c9285571296a7aa3186e69aa57df10264bd3f1bd3ce2c00d435ab40863b", + "job_name": "dev34-1017-fix-code-vulnerability-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json", + "config_sha256": "09db3a5a292440a611a1da389c422dc404853d21d8be54ad994d135939662f11", + "job_name": "dev34-1017-git-multibranch-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/kv-store-grpc", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json", + "config_sha256": "22c29825f0762fb05628119d5a4a47f96038c7d05058ac61de0e0b5300987e17", + "job_name": "dev34-1017-kv-store-grpc-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/log-summary-date-ranges", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json", + "config_sha256": "4edf8f4c1c500ab2ec690aecdb5481dcaa76d7327b972e07c01a0472dd70e3dc", + "job_name": "dev34-1017-log-summary-date-ranges-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/merge-diff-arc-agi-task", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/merge-diff-arc-agi-task.json", + "config_sha256": "c2409f07940bec07a25e852ed89785e1af02b4d81b2188961e6504bd6a2963e6", + "job_name": "dev34-1017-merge-diff-arc-agi-task-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/model-extraction-relu-logits", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/model-extraction-relu-logits.json", + "config_sha256": "bd4d1341bea166384900f8aafd2e745c54f910db8bf7a0d906f5ffc1da9abf8d", + "job_name": "dev34-1017-model-extraction-relu-logits-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/multi-source-data-merger.json", + "config_sha256": "b708c26001fed1c2fed7ee0bdb2e996d6bd0397586e011e69450ea62c4502c27", + "job_name": "dev34-1017-multi-source-data-merger-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/nginx-request-logging", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/nginx-request-logging.json", + "config_sha256": "d9564cfe19e4351903a14b107ba2512a863144a94ca1cb1c3bdb4e9c43a27ddf", + "job_name": "dev34-1017-nginx-request-logging-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/openssl-selfsigned-cert", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/openssl-selfsigned-cert.json", + "config_sha256": "9ef85ea697e55e2a45385b0c76a661d3f309ec96f5854c99ede5c167c4840182", + "job_name": "dev34-1017-openssl-selfsigned-cert-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pypi-server", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/pypi-server.json", + "config_sha256": "aab0c2c0acc85ab7b1bca03f53e6643aedc8b14f438e4e7e6fc697365da8c207", + "job_name": "dev34-1017-pypi-server-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-cli.json", + "config_sha256": "86d722dc6fc065087c7c9f88039007f7d984f08fc6a21b8754a1964061de22be", + "job_name": "dev34-1017-pytorch-model-cli-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/pytorch-model-recovery.json", + "config_sha256": "2d22a0df5b6ea299fcfbf68ece763cab6b048d67a6c3c99a1b03331fe4254dcb", + "job_name": "dev34-1017-pytorch-model-recovery-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/qemu-alpine-ssh", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/qemu-alpine-ssh.json", + "config_sha256": "77920fbee85e0a4109e065171224f9350e73ef9c26906d695a6184068655c3e9", + "job_name": "dev34-1017-qemu-alpine-ssh-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/query-optimize", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/query-optimize.json", + "config_sha256": "339d21d9f22e4a7c90f16042e388556ed4bde0818bf52e8e1515d47a44c01013", + "job_name": "dev34-1017-query-optimize-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/regex-log", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-log.json", + "config_sha256": "7a98a17dc177dcc27b7b610210043f91ea6b33cb965a550690eddbf9ac57c886", + "job_name": "dev34-1017-regex-log-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/torch-pipeline-parallelism", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-pipeline-parallelism.json", + "config_sha256": "5c7b6d41d8c239a00f91d1eab67f710d09c256c9f82657d389b19634579814c2", + "job_name": "dev34-1017-torch-pipeline-parallelism-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/torch-tensor-parallelism", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/torch-tensor-parallelism.json", + "config_sha256": "d7e752886c5972c8c06746709a98276853da6b17a1361815778fba31e45793af", + "job_name": "dev34-1017-torch-tensor-parallelism-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/write-compressor", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/write-compressor.json", + "config_sha256": "e095d60ba13c31cb46eebc64108a3f9046f004609c560040a00438fffaff5857", + "job_name": "dev34-1017-write-compressor-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/constraints-scheduling", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/constraints-scheduling.json", + "config_sha256": "0356b8b7a0c0529ee54194649ff98d12d878cf5c6a87b3fe9b1452d348d31b23", + "job_name": "dev34-1017-constraints-scheduling-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/custom-memory-heap-crash", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/custom-memory-heap-crash.json", + "config_sha256": "e182a58f586ce9a4452e7a5f005b6a0b35a21c0b0bbccf0a288cc561fb8035a1", + "job_name": "dev34-1017-custom-memory-heap-crash-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/dna-assembly", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dna-assembly.json", + "config_sha256": "0fa30910ad30e182bcc8b9d39cd62b364e7e44d092ae5dbfda6cce293f33682e", + "job_name": "dev34-1017-dna-assembly-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/llm-inference-batching-scheduler", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/llm-inference-batching-scheduler.json", + "config_sha256": "8671502ee1568ed2b63a35c6d25ebd85aee25de8df4e7750ceb7644a097e891e", + "job_name": "dev34-1017-llm-inference-batching-scheduler-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/path-tracing", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/path-tracing.json", + "config_sha256": "0b6814bb9b09b23cc77406914897d33182fff8ed0e26c602cc203c5731a1a40d", + "job_name": "dev34-1017-path-tracing-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/schemelike-metacircular-eval", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/schemelike-metacircular-eval.json", + "config_sha256": "ab39b406a1793049af65f8590875c1a63169ec3efbd3214fe7fee0bd81b09f60", + "job_name": "dev34-1017-schemelike-metacircular-eval-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/caffe-cifar-10", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/caffe-cifar-10.json", + "config_sha256": "38e430910a58b740f0ca7a67c9873b675b4f65f1ebd9a70a40fd21a35ff1ade4", + "job_name": "dev34-1017-caffe-cifar-10-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/circuit-fibsqrt", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/circuit-fibsqrt.json", + "config_sha256": "546c8cc4416c6b23ad5fbd87ae1b660893e5225da9bbeff66a989134b4fe16ad", + "job_name": "dev34-1017-circuit-fibsqrt-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/mteb-leaderboard", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/mteb-leaderboard.json", + "config_sha256": "7ed551ce3442a0e98b9986f3b8ba399e2dac0fa5ca3d058ed048658abf1d6289", + "job_name": "dev34-1017-mteb-leaderboard-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/regex-chess", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/regex-chess.json", + "config_sha256": "f82c737563dad901b2991908556193c7a8f3fbe6dd46736625b484f4c2613aef", + "job_name": "dev34-1017-regex-chess-main-20261019" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-pov-ray", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/build-pov-ray.json", + "config_sha256": "5856142204e35e53e7fd9215b8da5cd3cea24d590f505243c4d61d575b417a43", + "job_name": "dev34-1017-build-pov-ray-main-20261019" + } + ], + "preflight": [], + "selection_file": "selection.json", + "selection_sha256": "0eeba4c1efaaf6fbea1f0a257fe8f00a0033428d97c04d760d33c3cccebe0016", + "image_digests_file": "image-digests.json", + "log_dir": "jobs/dev34-1017-main-20261019-v2-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json": "3970b156e2f4d429d09aea068a3aab2707f21df17902235ca4423c11e881e417", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/run_comparison.py": "04f2106f0099a7b4d77e9206a36767f98a5e92c875ec78dd2de858c4936c12a8", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json": "d5381e807867a54a655c77f5a93c480d0be1d9f592386dc9b285726ea070dd3a", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json": "4a479285131edd49b71d9c31ba5546c1846d7a14b844ba70f37ea1815f207c13" + }, + "budget_policy": "Same as the October 12 baseline: native agent timeout minus 180 seconds, separate 1800-second setup, 100 replies, 65536 output tokens per reply. Native verifier per-attempt limits; only timeouts may retry, at most three total attempts on the same output. Temperature and reasoning effort unset.", + "preflight_policy": "Fresh install-only checks for all 34 tasks before any model task. QEMU has the reviewed qemu-alpine-ssh-v1 verifier profile; the other 33 are explicitly not_configured. Agent installation success does not certify unsupported verifier dependencies.", + "information_policy": "All tasks have previous development exposure. Supply original task instructions and current main prompt only; no earlier diagnoses or fixes. Review operational metadata while running. Freeze all new model outputs before reviewing fresh trajectories or hidden-test content.", + "queue_policy": "Ascending work budget then task name, consecutive pair barriers, maximum concurrency two, one fresh model trial per task, zero whole-trial retries and no result-based replacement.", + "source_policy": "Agent and adapter are evaluated from a detached, unmodified checkout of the main commit observed at registration. The standard entry point independently snapshots sources and verifies each wheel.", + "prior_baseline": "dev34-main-baseline-20261012", + "scope": "Fresh full development-pool baseline on current main and Flash 1017. Previous task outputs and rewards are not reused. This is not a held-out score or a controlled estimate of checkpoint effects.", + "image_digests_sha256": "fd9abd24bcc397d172ac0ad4e3faa875a2da8c482b35651a2ab296e04c3adcc4", + "accepted_preflight_trials": [ + { + "arm": "main", + "task": "terminal-bench/adaptive-rejection-sampler", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/adaptive-rejection-sampler__q2UYcaW/result.json", + "result_sha256": "47f301285fb59b6e09e01ced977a956b08927b5c362dec94ccd60c9de46461a9", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/build-cython-ext", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/build-cython-ext__AcKrSeD/result.json", + "result_sha256": "eb91a96d979392d78cbd190808933119057ac3fe3f880b7c16edb3b4a746ee18", + "bootstrap_sha256": "324a224408951769384fa327b2681ea1d72a8f50bee2f713befaeab399be1f8e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/cancel-async-tasks", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/cancel-async-tasks__abay9aV/result.json", + "result_sha256": "38ad7f1144c033d424a9512f1f75e449b7c11c5237ff51fbdb0fa8322be6b067", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/cobol-modernization", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/cobol-modernization__tUc2bp6/result.json", + "result_sha256": "b15ce7b9d9d3d6d396b7136d53abe936f37af405a1f9575272861089dacf7d7b", + "bootstrap_sha256": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/db-wal-recovery", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/db-wal-recovery__72Euzgt/result.json", + "result_sha256": "11609b4cc04662eb603584f485ab6193fa0ecd981c06f691e476d9cd0c3f785b", + "bootstrap_sha256": "b5e3b1241fffd9cb9c124014d0da1221f3dd38771c762332e33db42dd948a9b8", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/fix-code-vulnerability", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/fix-code-vulnerability__zSiCJCL/result.json", + "result_sha256": "d54ad89483fc5d576816dab4c2d696a02cc8e5ca4f2cac908407b02a48756621", + "bootstrap_sha256": "b8f2c9df149283c7445145220c5cb37c36fc9acfe0b60dc3a92532845b1966cc", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/git-multibranch", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/git-multibranch__d7USbyH/result.json", + "result_sha256": "a234d4227ae620016cd873293619782d1e9aab01bb772f52ce454d1e21cfd20b", + "bootstrap_sha256": "6dc412dac065243b61cef728ad0604dd165c861955efb831afaf2fe1f4aa2feb", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/kv-store-grpc", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/kv-store-grpc__eYejAv7/result.json", + "result_sha256": "da3ba5cacef8f87048e20c8fd9446c20270caa5b663ab1dcc9fc71c9e19a4ce9", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/log-summary-date-ranges", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/log-summary-date-ranges__RSkJxvq/result.json", + "result_sha256": "0b0916e25de82f5009edf6818ab5d2ee93f594f9281b4e0960abe4d410b4bd18", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/merge-diff-arc-agi-task", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/merge-diff-arc-agi-task__pebzZBA/result.json", + "result_sha256": "1d11b6daabd37a2690bbdd77e31b435b3783c4e9ba91673b06872211b47dca91", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/model-extraction-relu-logits", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/model-extraction-relu-logits__wLmRiqE/result.json", + "result_sha256": "586176ca56a1e0f1458eea7174a0dc32870fa9f829166c5da19ca1f217aa4e9b", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/multi-source-data-merger", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/multi-source-data-merger__VEqER8d/result.json", + "result_sha256": "e213c89e2520ba176473e17b8e0990772af42f8657bf184375eca234a75c5f11", + "bootstrap_sha256": "5011ac5d6ab074bd66636cc77aae3a2d966293bcb051b929e7a9cb076f908948", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/nginx-request-logging", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/nginx-request-logging__TK6evE9/result.json", + "result_sha256": "dda3b153fabea4e03d3ceef5e747e15211043055449f5cf1bccb8979e4d9684a", + "bootstrap_sha256": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/openssl-selfsigned-cert", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/openssl-selfsigned-cert__NHLRsUU/result.json", + "result_sha256": "bff0dcb6001fad4c06aba7e225434b0bad9ea469cc25df7793ded1aa35ab9406", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/pypi-server", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pypi-server__DrMvmHf/result.json", + "result_sha256": "b22a835c15ef8571934127f65034c95d750780d270019305556c42ea81e41fad", + "bootstrap_sha256": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-cli", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pytorch-model-cli__479me7S/result.json", + "result_sha256": "ed1348c5f597546163447d4beed18a66989b165f8b4a6efab6c97ecbe74c37fe", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-recovery", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pytorch-model-recovery__QgtL4QH/result.json", + "result_sha256": "d0246b8672e5863312892ec3da55944e67139f4fd6fbdcf029512fb1cd300706", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/qemu-alpine-ssh", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/qemu-alpine-ssh__fRwRKWY/result.json", + "result_sha256": "db70bcc900bcb84947f014b5cb9d80ecfcd26e81aac181b168a41efb03e72afd", + "bootstrap_sha256": "aaaa4fafbe6b8a8077e28cffbce3abddae4c4daa3e8fc813010552542cacc1a2", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/query-optimize", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/query-optimize__PHokY2V/result.json", + "result_sha256": "86dda8afbd1df2bb80caa3bb154571396b9ce83664156badced3a3848650709b", + "bootstrap_sha256": "90bb56ff2a427a339bca3904e95ff8d7b2cfdca6af6af63ab01995c7c52cf240", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/regex-log", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/regex-log__m2ebsyq/result.json", + "result_sha256": "c12f672e7faff3bea9def494f81af8323083a7f9c4472b2b7d2adfa63d0f1014", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/torch-pipeline-parallelism", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/torch-pipeline-parallelism__vSaJRmN/result.json", + "result_sha256": "dee0aa434d2b82027fd8c4532b15deb9720d750654dfc27813deae804616d6f3", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/torch-tensor-parallelism", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/torch-tensor-parallelism__LaLFMf7/result.json", + "result_sha256": "0cdf7bfdc1d92b38bd0d42f344854a60e887c339bc3879f35ac0d3f118f89e53", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/write-compressor", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/write-compressor__TYgm2Z6/result.json", + "result_sha256": "320c9bb44fb28dc1acb45caf1889c66abd1d2973dea945d12ac289414d2cbae5", + "bootstrap_sha256": "031768ca83f80450befe1475e9f6bfc93052b2053d23aae46a188aa390a36aba", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/constraints-scheduling", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/constraints-scheduling__wGKDyi9/result.json", + "result_sha256": "4b6b33f3c5b88cf535767a36a89178ee2cba1a9ec83fde223c3f5a7678c4a45e", + "bootstrap_sha256": "9705c14c20a581a03dadb6e7334925f68eb71992573c8b228c8c79ef0655e9e9", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/custom-memory-heap-crash", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/custom-memory-heap-crash__YkCeR9M/result.json", + "result_sha256": "eb31be14382121d747f2c799be8916e6b317165f8882c0daa992415e5aa2d9e6", + "bootstrap_sha256": "b1cca4c2159275ee7676539abaf5854f9c053ddaa2a5477b3762d3e244f2a04f", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/dna-assembly", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/dna-assembly__oLidH7H/result.json", + "result_sha256": "ce78ce2f2c7327921b9cc91d327e5937f8af5efdbbe8953b23d78a356be4bc66", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/llm-inference-batching-scheduler", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/llm-inference-batching-scheduler__H8UUtuL/result.json", + "result_sha256": "8c57cbcb6700400662fb3d9c7c6178985a6f93fc3409642893fb69060fb470f3", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/path-tracing", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/path-tracing__MDLwYoa/result.json", + "result_sha256": "441be3b43d7ad91c739f06f3aab716a44a69b9e04f52d16b1c359cf5780909bd", + "bootstrap_sha256": "2cdec543c50c5600582cfc3b2ebbe2f2c9d16988e065a02cf6e8e823f3a05542", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/schemelike-metacircular-eval", + "result_path": "jobs/main-source/jobs/dev34-1017-schemelike-preflight-recovery-main-20261019/schemelike-metacircular-eval__agfE5zu/result.json", + "result_sha256": "425babe123990ef719e9d4525782b30b8279fdca68b2a829ade7f94d4ea5afff", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "8d355368404fbe99e2d8cc0baf9d36b0127f957d94d8c3aa318a087c67e019fd", + "input_config_sha256": "43327ff3a8de9865142362a0df3b8c4602a5713119c834843bc5a9acbab44b5c" + }, + { + "arm": "main", + "task": "terminal-bench/caffe-cifar-10", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/caffe-cifar-10__JtiTSgA/result.json", + "result_sha256": "f9b7522ee51557ee07bea2de648d6ece5ecfc9a5aca863cc29114ce6f6047776", + "bootstrap_sha256": "0ebfd791a33d4b0978a4942b1cf9effb2a4347febb1a4ac2efa9520b1f79096e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/circuit-fibsqrt", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/circuit-fibsqrt__SVkt6UK/result.json", + "result_sha256": "58451281e419ba7f796b53ac9577c54bb93d8bc4496992400dd706b6b8d367fe", + "bootstrap_sha256": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/mteb-leaderboard", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/mteb-leaderboard__4fYZMaB/result.json", + "result_sha256": "0b881e62eec5a6ef77aef0961d09c72ffc39646ae886432f6b77e7064b9f1a1d", + "bootstrap_sha256": "a3d2b8905b957adcb3553196f9680346955b8f276e2915b0e3539aab45b5407e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/regex-chess", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/regex-chess__mSWEp4p/result.json", + "result_sha256": "eb3d47204a40434cde47dc3e826107bb8b866c64207ec902af99c624713a14bc", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + }, + { + "arm": "main", + "task": "terminal-bench/build-pov-ray", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/build-pov-ray__YrawpJV/result.json", + "result_sha256": "777bbd00ac72baf20292d65c3c600158323de04380e5b33efb074a95e2c64400", + "bootstrap_sha256": "0ebfd791a33d4b0978a4942b1cf9effb2a4347febb1a4ac2efa9520b1f79096e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + } + ], + "supersedes": "experiment.json", + "amendment": "Accept hash-bound successful native installation records for all 34 tasks: 33 from the original preflight and one fresh installation-only recovery. Preserve its one original TLS setup failure. All model configurations, sources, images, budgets, task revisions, and timeout-only verifier policy are unchanged." +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json new file mode 100644 index 0000000..98ce737 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json @@ -0,0 +1,641 @@ +{ + "collected_at": "2026-10-19T13:44:55.714589+00:00", + "model_started": false, + "attempts": 35, + "successful_installations": 34, + "setup_failures": 1, + "administrative_interruptions": 0, + "recovery_reason": "Fresh installation-only trial after the original uv installer download failed with an astral.sh TLS connection error. Same source, task, environment, budgets, and official sources. No model or verifier execution occurred.", + "trials": [ + { + "arm": "main", + "task": "terminal-bench/adaptive-rejection-sampler", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/adaptive-rejection-sampler__q2UYcaW/result.json", + "result_sha256": "47f301285fb59b6e09e01ced977a956b08927b5c362dec94ccd60c9de46461a9", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:15:12.636428Z", + "finished_at": "2026-10-19T13:16:25.741910Z" + }, + { + "arm": "main", + "task": "terminal-bench/build-cython-ext", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/build-cython-ext__AcKrSeD/result.json", + "result_sha256": "eb91a96d979392d78cbd190808933119057ac3fe3f880b7c16edb3b4a746ee18", + "bootstrap_sha256": "324a224408951769384fa327b2681ea1d72a8f50bee2f713befaeab399be1f8e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:15:12.540236Z", + "finished_at": "2026-10-19T13:16:08.891548Z" + }, + { + "arm": "main", + "task": "terminal-bench/build-pov-ray", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/build-pov-ray__YrawpJV/result.json", + "result_sha256": "777bbd00ac72baf20292d65c3c600158323de04380e5b33efb074a95e2c64400", + "bootstrap_sha256": "0ebfd791a33d4b0978a4942b1cf9effb2a4347febb1a4ac2efa9520b1f79096e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:16:11.491224Z", + "finished_at": "2026-10-19T13:17:04.626533Z" + }, + { + "arm": "main", + "task": "terminal-bench/caffe-cifar-10", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/caffe-cifar-10__JtiTSgA/result.json", + "result_sha256": "f9b7522ee51557ee07bea2de648d6ece5ecfc9a5aca863cc29114ce6f6047776", + "bootstrap_sha256": "0ebfd791a33d4b0978a4942b1cf9effb2a4347febb1a4ac2efa9520b1f79096e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:16:27.100557Z", + "finished_at": "2026-10-19T13:17:22.025668Z" + }, + { + "arm": "main", + "task": "terminal-bench/cancel-async-tasks", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/cancel-async-tasks__abay9aV/result.json", + "result_sha256": "38ad7f1144c033d424a9512f1f75e449b7c11c5237ff51fbdb0fa8322be6b067", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:17:06.074938Z", + "finished_at": "2026-10-19T13:18:04.698437Z" + }, + { + "arm": "main", + "task": "terminal-bench/circuit-fibsqrt", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/circuit-fibsqrt__SVkt6UK/result.json", + "result_sha256": "58451281e419ba7f796b53ac9577c54bb93d8bc4496992400dd706b6b8d367fe", + "bootstrap_sha256": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:17:23.836052Z", + "finished_at": "2026-10-19T13:18:20.535668Z" + }, + { + "arm": "main", + "task": "terminal-bench/cobol-modernization", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/cobol-modernization__tUc2bp6/result.json", + "result_sha256": "b15ce7b9d9d3d6d396b7136d53abe936f37af405a1f9575272861089dacf7d7b", + "bootstrap_sha256": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:18:06.038349Z", + "finished_at": "2026-10-19T13:19:16.468335Z" + }, + { + "arm": "main", + "task": "terminal-bench/constraints-scheduling", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/constraints-scheduling__wGKDyi9/result.json", + "result_sha256": "4b6b33f3c5b88cf535767a36a89178ee2cba1a9ec83fde223c3f5a7678c4a45e", + "bootstrap_sha256": "9705c14c20a581a03dadb6e7334925f68eb71992573c8b228c8c79ef0655e9e9", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:18:22.135295Z", + "finished_at": "2026-10-19T13:19:22.955131Z" + }, + { + "arm": "main", + "task": "terminal-bench/custom-memory-heap-crash", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/custom-memory-heap-crash__YkCeR9M/result.json", + "result_sha256": "eb31be14382121d747f2c799be8916e6b317165f8882c0daa992415e5aa2d9e6", + "bootstrap_sha256": "b1cca4c2159275ee7676539abaf5854f9c053ddaa2a5477b3762d3e244f2a04f", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:19:17.799127Z", + "finished_at": "2026-10-19T13:21:04.686171Z" + }, + { + "arm": "main", + "task": "terminal-bench/db-wal-recovery", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/db-wal-recovery__72Euzgt/result.json", + "result_sha256": "11609b4cc04662eb603584f485ab6193fa0ecd981c06f691e476d9cd0c3f785b", + "bootstrap_sha256": "b5e3b1241fffd9cb9c124014d0da1221f3dd38771c762332e33db42dd948a9b8", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:19:25.570501Z", + "finished_at": "2026-10-19T13:21:06.938778Z" + }, + { + "arm": "main", + "task": "terminal-bench/dna-assembly", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/dna-assembly__oLidH7H/result.json", + "result_sha256": "ce78ce2f2c7327921b9cc91d327e5937f8af5efdbbe8953b23d78a356be4bc66", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:21:07.358307Z", + "finished_at": "2026-10-19T13:22:52.593972Z" + }, + { + "arm": "main", + "task": "terminal-bench/fix-code-vulnerability", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/fix-code-vulnerability__zSiCJCL/result.json", + "result_sha256": "d54ad89483fc5d576816dab4c2d696a02cc8e5ca4f2cac908407b02a48756621", + "bootstrap_sha256": "b8f2c9df149283c7445145220c5cb37c36fc9acfe0b60dc3a92532845b1966cc", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:21:07.596608Z", + "finished_at": "2026-10-19T13:23:26.804966Z" + }, + { + "arm": "main", + "task": "terminal-bench/git-multibranch", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/git-multibranch__d7USbyH/result.json", + "result_sha256": "a234d4227ae620016cd873293619782d1e9aab01bb772f52ce454d1e21cfd20b", + "bootstrap_sha256": "6dc412dac065243b61cef728ad0604dd165c861955efb831afaf2fe1f4aa2feb", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:22:54.595191Z", + "finished_at": "2026-10-19T13:24:20.010352Z" + }, + { + "arm": "main", + "task": "terminal-bench/kv-store-grpc", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/kv-store-grpc__eYejAv7/result.json", + "result_sha256": "da3ba5cacef8f87048e20c8fd9446c20270caa5b663ab1dcc9fc71c9e19a4ce9", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:23:29.233529Z", + "finished_at": "2026-10-19T13:25:35.910272Z" + }, + { + "arm": "main", + "task": "terminal-bench/llm-inference-batching-scheduler", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/llm-inference-batching-scheduler__H8UUtuL/result.json", + "result_sha256": "8c57cbcb6700400662fb3d9c7c6178985a6f93fc3409642893fb69060fb470f3", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:24:23.090832Z", + "finished_at": "2026-10-19T13:26:14.362653Z" + }, + { + "arm": "main", + "task": "terminal-bench/log-summary-date-ranges", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/log-summary-date-ranges__RSkJxvq/result.json", + "result_sha256": "0b0916e25de82f5009edf6818ab5d2ee93f594f9281b4e0960abe4d410b4bd18", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:25:37.965528Z", + "finished_at": "2026-10-19T13:27:08.774851Z" + }, + { + "arm": "main", + "task": "terminal-bench/merge-diff-arc-agi-task", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/merge-diff-arc-agi-task__pebzZBA/result.json", + "result_sha256": "1d11b6daabd37a2690bbdd77e31b435b3783c4e9ba91673b06872211b47dca91", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:26:16.038482Z", + "finished_at": "2026-10-19T13:27:47.723512Z" + }, + { + "arm": "main", + "task": "terminal-bench/model-extraction-relu-logits", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/model-extraction-relu-logits__wLmRiqE/result.json", + "result_sha256": "586176ca56a1e0f1458eea7174a0dc32870fa9f829166c5da19ca1f217aa4e9b", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:27:11.096025Z", + "finished_at": "2026-10-19T13:29:42.496123Z" + }, + { + "arm": "main", + "task": "terminal-bench/mteb-leaderboard", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/mteb-leaderboard__4fYZMaB/result.json", + "result_sha256": "0b881e62eec5a6ef77aef0961d09c72ffc39646ae886432f6b77e7064b9f1a1d", + "bootstrap_sha256": "a3d2b8905b957adcb3553196f9680346955b8f276e2915b0e3539aab45b5407e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:27:51.419391Z", + "finished_at": "2026-10-19T13:29:40.621920Z" + }, + { + "arm": "main", + "task": "terminal-bench/multi-source-data-merger", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/multi-source-data-merger__VEqER8d/result.json", + "result_sha256": "e213c89e2520ba176473e17b8e0990772af42f8657bf184375eca234a75c5f11", + "bootstrap_sha256": "5011ac5d6ab074bd66636cc77aae3a2d966293bcb051b929e7a9cb076f908948", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:29:41.906488Z", + "finished_at": "2026-10-19T13:30:36.452283Z" + }, + { + "arm": "main", + "task": "terminal-bench/nginx-request-logging", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/nginx-request-logging__TK6evE9/result.json", + "result_sha256": "dda3b153fabea4e03d3ceef5e747e15211043055449f5cf1bccb8979e4d9684a", + "bootstrap_sha256": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:29:42.751157Z", + "finished_at": "2026-10-19T13:30:41.480844Z" + }, + { + "arm": "main", + "task": "terminal-bench/openssl-selfsigned-cert", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/openssl-selfsigned-cert__NHLRsUU/result.json", + "result_sha256": "bff0dcb6001fad4c06aba7e225434b0bad9ea469cc25df7793ded1aa35ab9406", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:30:37.877689Z", + "finished_at": "2026-10-19T13:32:15.288500Z" + }, + { + "arm": "main", + "task": "terminal-bench/path-tracing", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/path-tracing__MDLwYoa/result.json", + "result_sha256": "441be3b43d7ad91c739f06f3aab716a44a69b9e04f52d16b1c359cf5780909bd", + "bootstrap_sha256": "2cdec543c50c5600582cfc3b2ebbe2f2c9d16988e065a02cf6e8e823f3a05542", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:30:41.936311Z", + "finished_at": "2026-10-19T13:32:04.606460Z" + }, + { + "arm": "main", + "task": "terminal-bench/pypi-server", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pypi-server__DrMvmHf/result.json", + "result_sha256": "b22a835c15ef8571934127f65034c95d750780d270019305556c42ea81e41fad", + "bootstrap_sha256": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:32:06.232779Z", + "finished_at": "2026-10-19T13:33:04.331816Z" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-cli", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pytorch-model-cli__479me7S/result.json", + "result_sha256": "ed1348c5f597546163447d4beed18a66989b165f8b4a6efab6c97ecbe74c37fe", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:32:16.867688Z", + "finished_at": "2026-10-19T13:33:17.110191Z" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-recovery", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pytorch-model-recovery__QgtL4QH/result.json", + "result_sha256": "d0246b8672e5863312892ec3da55944e67139f4fd6fbdcf029512fb1cd300706", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:33:06.594693Z", + "finished_at": "2026-10-19T13:34:06.142083Z" + }, + { + "arm": "main", + "task": "terminal-bench/qemu-alpine-ssh", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/qemu-alpine-ssh__fRwRKWY/result.json", + "result_sha256": "db70bcc900bcb84947f014b5cb9d80ecfcd26e81aac181b168a41efb03e72afd", + "bootstrap_sha256": "aaaa4fafbe6b8a8077e28cffbce3abddae4c4daa3e8fc813010552542cacc1a2", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": "qemu-alpine-ssh-v1", + "status": "passed" + }, + "started_at": "2026-10-19T13:33:18.703941Z", + "finished_at": "2026-10-19T13:34:30.414138Z" + }, + { + "arm": "main", + "task": "terminal-bench/query-optimize", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/query-optimize__PHokY2V/result.json", + "result_sha256": "86dda8afbd1df2bb80caa3bb154571396b9ce83664156badced3a3848650709b", + "bootstrap_sha256": "90bb56ff2a427a339bca3904e95ff8d7b2cfdca6af6af63ab01995c7c52cf240", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:34:08.137262Z", + "finished_at": "2026-10-19T13:36:18.370921Z" + }, + { + "arm": "main", + "task": "terminal-bench/regex-chess", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/regex-chess__mSWEp4p/result.json", + "result_sha256": "eb3d47204a40434cde47dc3e826107bb8b866c64207ec902af99c624713a14bc", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:34:32.403884Z", + "finished_at": "2026-10-19T13:36:53.843699Z" + }, + { + "arm": "main", + "task": "terminal-bench/regex-log", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/regex-log__m2ebsyq/result.json", + "result_sha256": "c12f672e7faff3bea9def494f81af8323083a7f9c4472b2b7d2adfa63d0f1014", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:36:21.322104Z", + "finished_at": "2026-10-19T13:38:04.837184Z" + }, + { + "arm": "main", + "task": "terminal-bench/schemelike-metacircular-eval", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/schemelike-metacircular-eval__nRi7rEH/result.json", + "result_sha256": "1597cae07207dac8b1fd030358f95e2f5c6368bdfabc265f926d69232df72782", + "bootstrap_sha256": "b96b882b4d82151ae41b76cba8fbc65cd511d27da8e50683be9180fb72d2e464", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_failed", + "exception_type": "NetworkConnectionError", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:36:55.358967Z", + "finished_at": "2026-10-19T13:37:45.290134Z" + }, + { + "arm": "main", + "task": "terminal-bench/torch-pipeline-parallelism", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/torch-pipeline-parallelism__vSaJRmN/result.json", + "result_sha256": "dee0aa434d2b82027fd8c4532b15deb9720d750654dfc27813deae804616d6f3", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:37:47.819341Z", + "finished_at": "2026-10-19T13:40:17.013593Z" + }, + { + "arm": "main", + "task": "terminal-bench/torch-tensor-parallelism", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/torch-tensor-parallelism__LaLFMf7/result.json", + "result_sha256": "0cdf7bfdc1d92b38bd0d42f344854a60e887c339bc3879f35ac0d3f118f89e53", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:38:09.015497Z", + "finished_at": "2026-10-19T13:39:53.948577Z" + }, + { + "arm": "main", + "task": "terminal-bench/write-compressor", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/write-compressor__TYgm2Z6/result.json", + "result_sha256": "320c9bb44fb28dc1acb45caf1889c66abd1d2973dea945d12ac289414d2cbae5", + "bootstrap_sha256": "031768ca83f80450befe1475e9f6bfc93052b2053d23aae46a188aa390a36aba", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:39:56.306661Z", + "finished_at": "2026-10-19T13:42:06.864705Z" + }, + { + "arm": "main", + "task": "terminal-bench/schemelike-metacircular-eval", + "result_path": "jobs/main-source/jobs/dev34-1017-schemelike-preflight-recovery-main-20261019/schemelike-metacircular-eval__agfE5zu/result.json", + "result_sha256": "425babe123990ef719e9d4525782b30b8279fdca68b2a829ade7f94d4ea5afff", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "8d355368404fbe99e2d8cc0baf9d36b0127f957d94d8c3aa318a087c67e019fd", + "input_config_sha256": "43327ff3a8de9865142362a0df3b8c4602a5713119c834843bc5a9acbab44b5c", + "job": "dev34-1017-schemelike-preflight-recovery-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:42:31.563242Z", + "finished_at": "2026-10-19T13:44:17.830597Z" + } + ] +} From 2caae1b39489865216ea751d39d3d96797850cc1 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 19 Oct 2026 07:17:27 -0700 Subject: [PATCH 03/12] chore(benchmarks): register recovery for pre-model launch errors --- .../dev34-flash1017-main-20261019/README.md | 26 +++++ .../launch-failure-evidence.json | 52 +++++++++ .../launch-recovery-git-multibranch.json | 106 ++++++++++++++++++ .../launch-recovery-kv-store-grpc.json | 105 +++++++++++++++++ ...unch-recovery-log-summary-date-ranges.json | 106 ++++++++++++++++++ .../launch-recovery-selection.json | 20 ++++ 6 files changed, 415 insertions(+) create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-git-multibranch.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-kv-store-grpc.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-log-summary-date-ranges.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-selection.json diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md index 6481340..9d37231 100644 --- a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md @@ -81,3 +81,29 @@ all 34 outputs before reviewing new trajectories or hidden-test content. Raw log remain local; publish derived findings and evidence hashes after review. All 34 tasks have prior development exposure. The remaining 55 dataset tasks stay unopened, and this run does not isolate a model-checkpoint effect. + +## Host launch failures and supplemental jobs + +Three primary jobs failed before any native trial or model output existed: +Git multibranch and gRPC KV store could not connect to the Harbor registry; +log-summary date ranges could not resolve wheel build requirements because the +PyPI TLS connection failed. The original empty summaries, logs, snapshots, and +job records remain intact, bound by [launch evidence](launch-failure-evidence.json). +These are host launch errors, separate from task setup, verifier errors, and +model correctness failures. + +Three single-task supplemental plans register one new output for each task: +[Git](launch-recovery-git-multibranch.json), +[KV](launch-recovery-kv-store-grpc.json), and +[logs](launch-recovery-log-summary-date-ranges.json). +They use the exact same frozen main checkout, configurations, successful native +installation evidence, images, prompt, resources, and budgets. Original jobs +remain failed; generated outputs and scored results are never replaced. + +Start these plans only when the original queue reaches its final pair and one +of those jobs has ended, leaving one free slot with no later primary jobs to +start. Execute the three single-task plans serially through the existing +orchestrator and standard entry point. Total task concurrency stays at two or +fewer. All primary and supplemental outputs must freeze before new trajectory +or official-test-content review. Report all original launch failures and the +supplemental scheduling separately from the original fixed queue. diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json new file mode 100644 index 0000000..01ea8a0 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json @@ -0,0 +1,52 @@ +{ + "collected_at": "2026-10-19T14:17:10.343381+00:00", + "original_plan": "experiment-v2.json", + "policy": "Preserve every failed launch. Supplemental jobs may only cover tasks with no native trial or model output; generated outputs and scored results are never replaced.", + "failed_launches": [ + { + "task": "terminal-bench/git-multibranch", + "job": "dev34-1017-git-multibranch-main-20261019", + "classification": "launch_error_before_model", + "native_trial_results": 0, + "model_outputs": 0, + "stage": "registry_lookup", + "reason": "Host TLS/network connection failed before task execution.", + "evidence_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-git-multibranch-main-20261019.json": "3ccd84b057ba3dbc8bb517d9e8148d4b73a7f75709770952af7bef9f2103aa02", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-git-multibranch-main-20261019.log": "83a71596b69f66e017e55b86971e74b4bd867262d81e3b9844edb20e074e91a3", + "jobs/main-source/jobs/dev34-1017-git-multibranch-main-20261019-input/harbor.log": "1667d71a6aea9f63b268b07433bff55c86f5d519ace2bf110e33d58fb6900c37", + "jobs/main-source/jobs/dev34-1017-git-multibranch-main-20261019-input/manifest.json": "77bed126c6acc050f7c390441c0b69673afc10c66976acc39600f32fd1297459", + "jobs/main-source/jobs/dev34-1017-git-multibranch-main-20261019-input/config.json": "528820d8642370dd93f64a6a7bf7068f005ae6fbc0d855dbf39bdb8aa4ab8823" + } + }, + { + "task": "terminal-bench/kv-store-grpc", + "job": "dev34-1017-kv-store-grpc-main-20261019", + "classification": "launch_error_before_model", + "native_trial_results": 0, + "model_outputs": 0, + "stage": "registry_lookup", + "reason": "Host TLS/network connection failed before task execution.", + "evidence_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-kv-store-grpc-main-20261019.json": "d359bb106f3e23a3ef3e73156cc5f7594a0dc60cefffb5da4aa3ff3e5b4d9a12", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-kv-store-grpc-main-20261019.log": "deeaa8da35c7499fac9262268a93833b20b58adc22c20e70e79017c10399f12a", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-main-20261019-input/harbor.log": "1667d71a6aea9f63b268b07433bff55c86f5d519ace2bf110e33d58fb6900c37", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-main-20261019-input/manifest.json": "a7912e4feac60685a27c953d2618c5fdd09d93b20983577a4e12b89e0afa7b75", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-main-20261019-input/config.json": "bb9bc3da8927365bfe312c4a5b0454cf2a78b83bb02728e7bf56d0bb01aca484" + } + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "job": "dev34-1017-log-summary-date-ranges-main-20261019", + "classification": "launch_error_before_model", + "native_trial_results": 0, + "model_outputs": 0, + "stage": "wheel_build", + "reason": "Host TLS/network connection failed before task execution.", + "evidence_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-log-summary-date-ranges-main-20261019.json": "6bd96070a09869ff104dee9e3e3d9fd11f3de8d30dad45d9ecc42313be78102f", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-log-summary-date-ranges-main-20261019.log": "93f786a8169395ffdfd522c157244dd4a7948fcea967f723183538676d961135" + } + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-git-multibranch.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-git-multibranch.json new file mode 100644 index 0000000..45b5f90 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-git-multibranch.json @@ -0,0 +1,106 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-19T14:17:10.360306+00:00", + "experiment": "dev34-flash1017-main-20261019-launch-recovery-git-multibranch", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-19T13:13:43.549137+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "main": { + "source_root": "jobs/main-source", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "system-administration", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + } + } + ], + "total_model_runs": 1, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/git-multibranch.json", + "config_sha256": "09db3a5a292440a611a1da389c422dc404853d21d8be54ad994d135939662f11", + "job_name": "dev34-1017-git-multibranch-launch-recovery-main-20261019" + } + ], + "preflight": [], + "selection_file": "launch-recovery-selection.json", + "selection_sha256": "fd63dbb557618c3e548082d6836a6f72e4ee453539898194c40ce00725968fb2", + "image_digests_file": "image-digests.json", + "log_dir": "jobs/dev34-1017-launch-recovery-git-multibranch-20261019-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json": "3970b156e2f4d429d09aea068a3aab2707f21df17902235ca4423c11e881e417", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/run_comparison.py": "04f2106f0099a7b4d77e9206a36767f98a5e92c875ec78dd2de858c4936c12a8", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json": "d5381e807867a54a655c77f5a93c480d0be1d9f592386dc9b285726ea070dd3a", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json": "4a479285131edd49b71d9c31ba5546c1846d7a14b844ba70f37ea1815f207c13", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json": "2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json": "a1d18be8c4966188926a997735e12c7bb7a76a91be02dd81162caeb3cca947aa" + }, + "budget_policy": "Same as the October 12 baseline: native agent timeout minus 180 seconds, separate 1800-second setup, 100 replies, 65536 output tokens per reply. Native verifier per-attempt limits; only timeouts may retry, at most three total attempts on the same output. Temperature and reasoning effort unset.", + "preflight_policy": "Fresh install-only checks for all 34 tasks before any model task. QEMU has the reviewed qemu-alpine-ssh-v1 verifier profile; the other 33 are explicitly not_configured. Agent installation success does not certify unsupported verifier dependencies.", + "information_policy": "Review original host launch errors and operational metadata only. Freeze all primary and supplemental model outputs before any new trajectory or official-test-content review.", + "queue_policy": "A single-task supplemental plan. Serialize the three registered supplemental plans in git-multibranch, kv-store-grpc, log-summary-date-ranges order while using the free slot of the original final pair.", + "source_policy": "Agent and adapter are evaluated from a detached, unmodified checkout of the main commit observed at registration. The standard entry point independently snapshots sources and verifies each wheel.", + "prior_baseline": "dev34-main-baseline-20261012", + "scope": "One fresh model output for a dev34 task whose original job failed before any native trial existed; retain the original launch error separately. This does not retry or replace an agent output.", + "image_digests_sha256": "fd9abd24bcc397d172ac0ad4e3faa875a2da8c482b35651a2ab296e04c3adcc4", + "accepted_preflight_trials": [ + { + "arm": "main", + "task": "terminal-bench/git-multibranch", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/git-multibranch__d7USbyH/result.json", + "result_sha256": "a234d4227ae620016cd873293619782d1e9aab01bb772f52ce454d1e21cfd20b", + "bootstrap_sha256": "6dc412dac065243b61cef728ad0604dd165c861955efb831afaf2fe1f4aa2feb", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + } + ], + "launch_gate": "Start only when the primary queue is in its final pair, one last-pair job has finished, and no earlier job remains active. Run the three supplemental single-task plans serially, keeping total active task jobs at two or fewer." +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-kv-store-grpc.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-kv-store-grpc.json new file mode 100644 index 0000000..38f5158 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-kv-store-grpc.json @@ -0,0 +1,105 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-19T14:17:10.372710+00:00", + "experiment": "dev34-flash1017-main-20261019-launch-recovery-kv-store-grpc", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-19T13:13:43.549137+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "main": { + "source_root": "jobs/main-source", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/kv-store-grpc:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "software-engineering", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "51fdc2ad06f0adb1270164f247ea41799b2ee2ad15490e48ddbf5e79d3767aaa", + "environment/Dockerfile": "47c9982789092437f950736110b2bd18899c39b8fd20f3af3c95b848e564a197", + "instruction.md": "f4b3facc199d5b1c78db6e603ffa29ec1b93a77ff7d291e1a92b42e6bf582bed", + "solution/solve.sh": "6bee2938ac9731d1156b0ca02b28c3d5793bf2cbe5f995c863d397093af01cb7", + "task.toml": "6ff0f30a526bfff86a67b0b53439b9965229c0f368167dc1a5d05f34d82299a3", + "tests/test.sh": "c348467a076bf614d0a850e25b2263f85bbf7b4c7a8d344fbd5454a0611a0df8", + "tests/test_outputs.py": "115de2e964bbe689d07930527113422ef8a9e4be4eb958ad84594e3fcd2b1b0e" + } + } + ], + "total_model_runs": 1, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/kv-store-grpc", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/kv-store-grpc.json", + "config_sha256": "22c29825f0762fb05628119d5a4a47f96038c7d05058ac61de0e0b5300987e17", + "job_name": "dev34-1017-kv-store-grpc-launch-recovery-main-20261019" + } + ], + "preflight": [], + "selection_file": "launch-recovery-selection.json", + "selection_sha256": "fd63dbb557618c3e548082d6836a6f72e4ee453539898194c40ce00725968fb2", + "image_digests_file": "image-digests.json", + "log_dir": "jobs/dev34-1017-launch-recovery-kv-store-grpc-20261019-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json": "3970b156e2f4d429d09aea068a3aab2707f21df17902235ca4423c11e881e417", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/run_comparison.py": "04f2106f0099a7b4d77e9206a36767f98a5e92c875ec78dd2de858c4936c12a8", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json": "d5381e807867a54a655c77f5a93c480d0be1d9f592386dc9b285726ea070dd3a", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json": "4a479285131edd49b71d9c31ba5546c1846d7a14b844ba70f37ea1815f207c13", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json": "2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json": "a1d18be8c4966188926a997735e12c7bb7a76a91be02dd81162caeb3cca947aa" + }, + "budget_policy": "Same as the October 12 baseline: native agent timeout minus 180 seconds, separate 1800-second setup, 100 replies, 65536 output tokens per reply. Native verifier per-attempt limits; only timeouts may retry, at most three total attempts on the same output. Temperature and reasoning effort unset.", + "preflight_policy": "Fresh install-only checks for all 34 tasks before any model task. QEMU has the reviewed qemu-alpine-ssh-v1 verifier profile; the other 33 are explicitly not_configured. Agent installation success does not certify unsupported verifier dependencies.", + "information_policy": "Review original host launch errors and operational metadata only. Freeze all primary and supplemental model outputs before any new trajectory or official-test-content review.", + "queue_policy": "A single-task supplemental plan. Serialize the three registered supplemental plans in git-multibranch, kv-store-grpc, log-summary-date-ranges order while using the free slot of the original final pair.", + "source_policy": "Agent and adapter are evaluated from a detached, unmodified checkout of the main commit observed at registration. The standard entry point independently snapshots sources and verifies each wheel.", + "prior_baseline": "dev34-main-baseline-20261012", + "scope": "One fresh model output for a dev34 task whose original job failed before any native trial existed; retain the original launch error separately. This does not retry or replace an agent output.", + "image_digests_sha256": "fd9abd24bcc397d172ac0ad4e3faa875a2da8c482b35651a2ab296e04c3adcc4", + "accepted_preflight_trials": [ + { + "arm": "main", + "task": "terminal-bench/kv-store-grpc", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/kv-store-grpc__eYejAv7/result.json", + "result_sha256": "da3ba5cacef8f87048e20c8fd9446c20270caa5b663ab1dcc9fc71c9e19a4ce9", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + } + ], + "launch_gate": "Start only when the primary queue is in its final pair, one last-pair job has finished, and no earlier job remains active. Run the three supplemental single-task plans serially, keeping total active task jobs at two or fewer." +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-log-summary-date-ranges.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-log-summary-date-ranges.json new file mode 100644 index 0000000..fe76ac1 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-log-summary-date-ranges.json @@ -0,0 +1,106 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-19T14:17:10.388382+00:00", + "experiment": "dev34-flash1017-main-20261019-launch-recovery-log-summary-date-ranges", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-19T13:13:43.549137+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "main": { + "source_root": "jobs/main-source", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/log-summary-date-ranges:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "026000415cb577f3d3d5132902670262f7d5d8f69841903ec06a882292f756b4", + "environment/Dockerfile": "b479f6e987b63a6ec118fa1691b8bb85c437b2a6a0fe63feab5ed7154957fdfc", + "environment/log_generator_deterministic.py": "beb6eff59979e9aa0be63ce8a0f1b0df1178f92871f04ba9f76fa70cf8beb81c", + "instruction.md": "c1d4516ad4ec2238209740eaf01291df9ab48b0028a3fc5df383c6f831453484", + "solution/solve.sh": "9f6a8ee8947bdf1c0f31b530b6194ae998d66841a3b785fa2e246ee840350ab6", + "task.toml": "9a9593673d6805ef3a301ea734e9c4dbaf80da1b37c0fe9d1f2ceea2a8c57e18", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2fc06a6ce1944ad82f84fb437395fa75ed8c0eadd385b92e375c27e5bc9cb2b4" + } + } + ], + "total_model_runs": 1, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/log-summary-date-ranges", + "config": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/log-summary-date-ranges.json", + "config_sha256": "4edf8f4c1c500ab2ec690aecdb5481dcaa76d7327b972e07c01a0472dd70e3dc", + "job_name": "dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019" + } + ], + "preflight": [], + "selection_file": "launch-recovery-selection.json", + "selection_sha256": "fd63dbb557618c3e548082d6836a6f72e4ee453539898194c40ce00725968fb2", + "image_digests_file": "image-digests.json", + "log_dir": "jobs/dev34-1017-launch-recovery-log-summary-date-ranges-20261019-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/dependency-cache.compose.json": "3970b156e2f4d429d09aea068a3aab2707f21df17902235ca4423c11e881e417", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/run_comparison.py": "04f2106f0099a7b4d77e9206a36767f98a5e92c875ec78dd2de858c4936c12a8", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment.json": "d5381e807867a54a655c77f5a93c480d0be1d9f592386dc9b285726ea070dd3a", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/preflight-evidence.json": "4a479285131edd49b71d9c31ba5546c1846d7a14b844ba70f37ea1815f207c13", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json": "2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c", + "benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-failure-evidence.json": "a1d18be8c4966188926a997735e12c7bb7a76a91be02dd81162caeb3cca947aa" + }, + "budget_policy": "Same as the October 12 baseline: native agent timeout minus 180 seconds, separate 1800-second setup, 100 replies, 65536 output tokens per reply. Native verifier per-attempt limits; only timeouts may retry, at most three total attempts on the same output. Temperature and reasoning effort unset.", + "preflight_policy": "Fresh install-only checks for all 34 tasks before any model task. QEMU has the reviewed qemu-alpine-ssh-v1 verifier profile; the other 33 are explicitly not_configured. Agent installation success does not certify unsupported verifier dependencies.", + "information_policy": "Review original host launch errors and operational metadata only. Freeze all primary and supplemental model outputs before any new trajectory or official-test-content review.", + "queue_policy": "A single-task supplemental plan. Serialize the three registered supplemental plans in git-multibranch, kv-store-grpc, log-summary-date-ranges order while using the free slot of the original final pair.", + "source_policy": "Agent and adapter are evaluated from a detached, unmodified checkout of the main commit observed at registration. The standard entry point independently snapshots sources and verifies each wheel.", + "prior_baseline": "dev34-main-baseline-20261012", + "scope": "One fresh model output for a dev34 task whose original job failed before any native trial existed; retain the original launch error separately. This does not retry or replace an agent output.", + "image_digests_sha256": "fd9abd24bcc397d172ac0ad4e3faa875a2da8c482b35651a2ab296e04c3adcc4", + "accepted_preflight_trials": [ + { + "arm": "main", + "task": "terminal-bench/log-summary-date-ranges", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/log-summary-date-ranges__RSkJxvq/result.json", + "result_sha256": "0b0916e25de82f5009edf6818ab5d2ee93f594f9281b4e0960abe4d410b4bd18", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a" + } + ], + "launch_gate": "Start only when the primary queue is in its final pair, one last-pair job has finished, and no earlier job remains active. Run the three supplemental single-task plans serially, keeping total active task jobs at two or fewer." +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-selection.json b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-selection.json new file mode 100644 index 0000000..9a319c0 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-selection.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "selected_at": "2026-10-19T14:17:10.343499+00:00", + "selection_method": "Only the three original jobs whose host-network failures occurred before any native trial or model output existed. No outcome-based selection.", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "tasks": [ + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + }, + { + "name": "terminal-bench/kv-store-grpc", + "ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307" + }, + { + "name": "terminal-bench/log-summary-date-ranges", + "ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6" + } + ] +} From 12da0f9dc3b570ce3f66bd3747a328b576df7f02 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 19 Oct 2026 11:20:20 -0700 Subject: [PATCH 04/12] chore(benchmarks): record current-main dev34 flash 1017 results --- .../dev34-flash1017-main-20261019/README.md | 4 + .../reports/dev34-flash1017-main-20261019.md | 83 + .../dev34-flash1017-main-20261019.json | 4814 +++++++++++++++++ 3 files changed, 4901 insertions(+) create mode 100644 benchmarks/harbor/reports/dev34-flash1017-main-20261019.md create mode 100644 benchmarks/harbor/results/dev34-flash1017-main-20261019.json diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md index 9d37231..373d06f 100644 --- a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md @@ -1,5 +1,7 @@ # Current-main dev34 baseline with DeepSeek Flash 1017 +**Completed:** 26 passes, five scored failures, and three unscored verifier errors. All 34 tasks started a fresh agent trial. See the [report](../../reports/dev34-flash1017-main-20261019.md) and [audited results](../../results/dev34-flash1017-main-20261019.json). Native zero rewards for pre-assertion verifier failures remain preserved; analytical scores are null. The CLI dependency failure followed agent-forced package installations. A passing Scheme evaluator uses a host depth-counter workaround, recorded separately. + This registration runs all 34 existing development tasks once with fresh model outputs from current main `ce28b1e826a292d796ba64f13e79199b66f65758`. The experiment branch and its independent runtime checkout are separate, so recording results cannot change the evaluated @@ -107,3 +109,5 @@ orchestrator and standard entry point. Total task concurrency stays at two or fewer. All primary and supplemental outputs must freeze before new trajectory or official-test-content review. Report all original launch failures and the supplemental scheduling separately from the original fixed queue. + +All primary and supplemental jobs completed. The six relocated OCI archives and 44 cached blobs were restored and hash-verified. Final inspection retained all 34 image identities and found no remaining task containers. diff --git a/benchmarks/harbor/reports/dev34-flash1017-main-20261019.md b/benchmarks/harbor/reports/dev34-flash1017-main-20261019.md new file mode 100644 index 0000000..029fd99 --- /dev/null +++ b/benchmarks/harbor/reports/dev34-flash1017-main-20261019.md @@ -0,0 +1,83 @@ +# Current-main dev34 baseline with Flash 1017 + +**26 tasks passed, five scored failures, and three unscored verifier errors.** All 34 tasks started one fresh agent trial. The observed pass count is **26/34 (76.47%)**; the three unscored tasks have unknown correctness and are not counted as scored failures. Two native reward zeros remain preserved alongside null analytical scores; the third unscored task raised `VerifierTimeoutError`. + +The run evaluates main `ce28b1e826a292d796ba64f13e79199b66f65758` using the official DeepSeek Anthropic-compatible endpoint and `deepseek-flash`. [Official documentation](https://api-docs.deepseek.com/quick_start/agent_integrations/) was checked before execution and maps that alias to **DeepSeek-V4.1-Flash-1017**. The alias is not an immutable checkpoint pin. Agent Python sources and the 385-word prompt are byte-identical to the October 12 baseline; the adapter is from current main, with optional dependency-error recovery explicitly disabled. + +The [structured results](../results/dev34-flash1017-main-20261019.json) retain native rewards, analytical scores, all trial IDs, source/wheel/task hashes, verifier attempts, test counts, accounting, audit notes, and original launch errors. The [active plan](../configs/dev34-flash1017-main-20261019/experiment-v2.json), [frozen prompt](../configs/dev34-flash1017-main-20261019/main-prompt.txt), and [configuration README](../configs/dev34-flash1017-main-20261019/README.md) record reproduction details. The primary plan SHA256 is `2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c`. + +## Per-task outcomes + +| Task | October 12 reward | New outcome | Official pytest passed / failed | Agent seconds | Phase | +| --- | ---: | --- | ---: | ---: | --- | +| adaptive-rejection-sampler | 0 | Passed | 9 / 0 | 614.4 | primary | +| build-cython-ext | 1 | Passed | 11 / 0 | 436.3 | primary | +| cancel-async-tasks | 1 | Passed | 6 / 0 | 179.8 | primary | +| cobol-modernization | 1 | Passed | 3 / 0 | 262.0 | primary | +| db-wal-recovery | 0 | Failed | 5 / 2 | 132.8 | primary | +| fix-code-vulnerability | 1 | Passed | 6 / 0 | 24.6 | primary | +| git-multibranch | 1 | Passed | 1 / 0 | 220.2 | supplemental | +| kv-store-grpc | 0 | Passed | 7 / 0 | 297.1 | supplemental | +| log-summary-date-ranges | 1 | Passed | 2 / 0 | 25.8 | supplemental | +| merge-diff-arc-agi-task | 1 | Passed | 5 / 0 | 58.6 | primary | +| model-extraction-relu-logits | 1 | Passed | 1 / 0 | 301.4 | primary | +| multi-source-data-merger | 1 | Unscored | Not started | 38.2 | primary | +| nginx-request-logging | 1 | Passed | 8 / 0 | 60.0 | primary | +| openssl-selfsigned-cert | 1 | Passed | 6 / 0 | 54.4 | primary | +| pypi-server | 1 | Passed | 1 / 0 | 184.6 | primary | +| pytorch-model-cli | 1 | Unscored | Collection error | 461.3 | primary | +| pytorch-model-recovery | 1 | Unscored | Not started | 109.8 | primary | +| qemu-alpine-ssh | 1 | Passed | 1 / 0 | 506.2 | primary | +| query-optimize | 0 | Passed | 6 / 0 | 490.9 | primary | +| regex-log | 1 | Passed | 1 / 0 | 82.5 | primary | +| torch-pipeline-parallelism | 0 | Failed | 2 / 2 | 584.3 | primary | +| torch-tensor-parallelism | 1 | Passed | 13 / 0 | 196.0 | primary | +| write-compressor | 1 | Passed | 3 / 0 | 274.2 | primary | +| constraints-scheduling | 1 | Passed | 3 / 0 | 21.0 | primary | +| custom-memory-heap-crash | 1 | Passed | 6 / 0 | 185.5 | primary | +| dna-assembly | 1 | Failed | 0 / 1 | 346.5 | primary | +| llm-inference-batching-scheduler | 1 | Passed | 6 / 0 | 529.6 | primary | +| path-tracing | 1 | Passed | 5 / 0 | 908.7 | primary | +| schemelike-metacircular-eval | 1 | Passed | 1 / 0 | 1098.5 | primary | +| caffe-cifar-10 | 1 | Passed | 6 / 0 | 2212.4 | primary | +| circuit-fibsqrt | 1 | Passed | 3 / 0 | 1101.1 | primary | +| mteb-leaderboard | 1 | Passed | 2 / 0 | 863.7 | primary | +| regex-chess | 0 | Failed | 2 / 2 | 1810.4 | primary | +| build-pov-ray | 1 | Failed | 2 / 1 | 654.7 | primary | + +Thirty-one tasks executed official assertions: **141 pytest cases, 133 passed and eight failed**, with no skips. The CLI task produced one additional collection error. Native CTRF summaries are retained separately because parameterized cases are deduplicated. Counts describe final verification summaries; archived attempts are also preserved. Neither the three recovery attempts nor task test-case counts are extra agent-task scores. + +## Failure and verifier-error evidence + +- `db-wal-recovery`: Opened the original SQLite database before preserving its obfuscated WAL. The agent reported that SQLite removed the WAL, then inferred missing rows. Official tests found missing WAL updates: five passed and two failed. +- `torch-pipeline-parallelism`: Both world-size cases failed with model.rotary_emb.fwd microbatch-count mismatches. Existence and no-hooks checks passed. The first verifier attempt timed out during dependency preparation; the second executed all four cases. +- `dna-assembly`: Submitted primers lacked the required nucleotide clamp before the BsaI recognition site. The official primer test failed. +- `regex-chess`: The submitted regex pipeline allowed an illegal king move onto a square attacked by a knight. Two official cases passed and two failed. A real re.json was submitted; this is a scored implementation failure, unlike the earlier truncation-only six-task rerun. +- `build-pov-ray`: The official source-distribution check failed because file_id.diz was absent. Executable and rendering checks passed; the recorded SSIM was 0.8731. The agent downloaded from the official 2.2 archive URLs. The marker failure alone does not establish that a different version was built. +- `multi-source-data-merger`: The verifier failed to download/extract pyarrow==22.0.0 because of a network timeout before pytest started. Native reward zero is retained, with a null analytical score. This completed attempt was not a verifier-process timeout, so it was not retried. +- `pytorch-model-cli`: Official pytest collection failed importing cv2 because libGL.so.1 was missing after APT reported unmet dependencies. The agent had force-installed compiler/development packages with dpkg --force-all, leaving dependency problems; agent environment modifications contributed to the verification failure. Zero assertions executed. Native reward zero is retained, with a null analytical score; CLI correctness remains unverified. +- `pytorch-model-recovery`: All three original 900-second verifier attempts timed out downloading PyTorch/CUDA dependencies before pytest started. The same output was reused throughout. The native VerifierTimeoutError has no reward and the analytical score is null. + +## Execution and integrity + +All task revisions, original image identities, resources, package sources, official verifier scripts, and per-attempt limits match the registered development pool. Every installed wheel matches its recorded source snapshot. Limits remained 100 model replies, 65,536 output tokens per reply, native task agent time minus 180 seconds for work, and a separate 1,800-second setup limit. Temperature and reasoning effort were unset. + +Fresh installation preflight produced **34 successes and one TLS setup failure** across 35 attempts. The failed Scheme installation was preserved and a new installation-only check succeeded. Only QEMU has the reviewed verifier dependency profile (`qemu-alpine-ssh-v1`), which passed; the other 33 remain explicitly `not_configured`. [Preflight evidence](../configs/dev34-flash1017-main-20261019/preflight-evidence.json) binds every native record. Installation success does not certify unsupported verifier dependencies. + +Three primary host jobs failed before any native task trial or model output existed: two Harbor registry connections and one wheel-build PyPI TLS request. [Original launch evidence](../configs/dev34-flash1017-main-20261019/launch-failure-evidence.json) remains unchanged. Three pre-registered single-task supplemental plans used the free slot in the original final pair and ran serially. Thus 37 host launches produced **34 distinct fresh agent trials**, with no replacement of any generated output. The primary pair barriers and a global maximum of two concurrent task jobs were verified from timestamps. + +Verification made **37 attempts, including three timeout retries**. Model recovery exhausted three 900-second attempts before pytest. Pipeline parallelism timed out at 900 seconds during dependency preparation, then completed its second attempt in 427.42 seconds, including the valid scored failure. Every other task used one attempt. No download-error retries, changed per-attempt limits, or retries after executed test results were enabled. + +During the PyTorch pair, duplicate host image-cache files were temporarily moved to the memory-backed temporary filesystem under disk pressure. All six OCI archives and 44 content-addressed blobs were restored as regular files, with every SHA256 rechecked. Original Docker images and task environments were not replaced. Final inspection confirmed all **34 original image identities unchanged** and **zero remaining task containers**. + +All primary and supplemental jobs finished before new trajectory or official-test-content review. The freeze was recorded at `2026-10-19T18:10:43.804209+00:00`. The model phase occupied **264.23 minutes**, excluding installation-only preflight. All 34 recorded usage and cost records were complete; the bundled estimator reports **$1.861**. These are estimated token costs, not a billing reconciliation. + +## Post-freeze audit and interpretation + +Automated scans covered all 34 trajectories and agent stdout logs. Flagged tool calls, observations, external URLs, failure evidence, and verifier setup errors were then reviewed. No exact configured-key matches, task-specific benchmark-answer retrieval, or hidden verifier-content access were observed. WAL inspected an empty verifier-log directory. Tensor parallelism searched for recently created tests, but its observations contained installed dependency tests. Public MTEB results were task-required source data. These findings do not prove absence of all information exposure or training contamination. + +The passing Scheme evaluator relies on host-interpreter introspection: it uses the multiplication primitive's Python evaluation to reset the host evaluation-depth counter. Its official pass is retained with this implementation caveat. + +The October 12 baseline had 28 passes and six scored failures. In this fresh run, the sampler, KV store, and SQL optimization changed from failure to pass; WAL, pipeline parallelism, and regex chess remained scored failures. DNA assembly and POV-Ray changed from pass to scored failure. Three formerly passing tasks became unscored verifier errors. Previous outputs and scores were not reused. + +All 34 tasks have prior development exposure. Single fresh trials, live dependency resolution, agent-modified environments, host-launch recovery, and changing provider conditions prevent a controlled checkpoint-effect conclusion. Keep the three unscored tasks separate, retain the passing Scheme implementation caveat, and use the per-task evidence for development rather than treating this as a held-out capability score. diff --git a/benchmarks/harbor/results/dev34-flash1017-main-20261019.json b/benchmarks/harbor/results/dev34-flash1017-main-20261019.json new file mode 100644 index 0000000..31b845b --- /dev/null +++ b/benchmarks/harbor/results/dev34-flash1017-main-20261019.json @@ -0,0 +1,4814 @@ +{ + "schema_version": 1, + "experiment": "dev34-flash1017-main-20261019", + "collected_at": "2026-10-19T18:10:44.027485+00:00", + "cohort": "Entire existing 34-task development pool, one fresh model output per task, with prior development exposure.", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-19T13:13:43.549137+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "runtime_source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "plan_path": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/experiment-v2.json", + "plan_sha256": "2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c", + "model_output_freeze": { + "recorded_at": "2026-10-19T18:10:43.804209+00:00", + "last_job_finished_at": "2026-10-19T18:09:39.640305+00:00", + "finished_jobs": 37, + "primary_jobs": 34, + "supplemental_jobs": 3, + "policy": "All primary and registered supplemental jobs finished before reading new trajectories or official test output.", + "job_record_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-adaptive-rejection-sampler-main-20261019.json": "c65cc8332c5cb6acb63bd50777e17a15928f75893cc0427486e90aedc6b96133", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-build-cython-ext-main-20261019.json": "e9b0d1920debba88a1d0759043fe6eaf036b7191bbca2a3ee75e11a65a81eb39", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-cancel-async-tasks-main-20261019.json": "f67e634f29019d581a04553b8bf6b4a0c3299f1148599a55d66a8509854effec", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-cobol-modernization-main-20261019.json": "0b63ad5e8d68ddae2a1fd50ed5ee6f6fb6ab2185f35770bebeeb69da84059d92", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-db-wal-recovery-main-20261019.json": "e57d7624d91538e03db29d4faac9cd7ade0934a34e891a7005258c52c01f1e88", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-fix-code-vulnerability-main-20261019.json": "0df3a5945b6c4235ff4ac9ee34eb2493cf9438f3719bec5b29524595380680f9", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-git-multibranch-main-20261019.json": "3ccd84b057ba3dbc8bb517d9e8148d4b73a7f75709770952af7bef9f2103aa02", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-kv-store-grpc-main-20261019.json": "d359bb106f3e23a3ef3e73156cc5f7594a0dc60cefffb5da4aa3ff3e5b4d9a12", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-log-summary-date-ranges-main-20261019.json": "6bd96070a09869ff104dee9e3e3d9fd11f3de8d30dad45d9ecc42313be78102f", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-merge-diff-arc-agi-task-main-20261019.json": "54a91b4a7a072c56c4e20b650897f5f9aa9813aaf961424c71dcb5d1be266aab", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-model-extraction-relu-logits-main-20261019.json": "a02f32f046bd22ba5c10960092e875f823fc61d730c8d20118160c6b80e11542", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-multi-source-data-merger-main-20261019.json": "1fe2a8510681dc681b0d2145503b4bfb72b60fe667e6fd4c0fcd098324b3aa7e", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-nginx-request-logging-main-20261019.json": "ddf01b7458253ebe208607579e0e08ece43cd47c28ec67ecc78bc86823715c7b", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-openssl-selfsigned-cert-main-20261019.json": "36c45c112ab61b7bd1d671b7fa0ec66beb3f865d5a229e5f560be68af9044111", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pypi-server-main-20261019.json": "3d875d5703a2f9ca5729a4d10162b5c45c9f4ab67b8d7fc84d53d2b3f89143a3", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-cli-main-20261019.json": "98e3548ee417240c814b98bb07b6dd53a772e83e30e6f555f32565c7dc18ebe1", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-recovery-main-20261019.json": "b71e4cca26a8670fcc6294e6744ac49a6d7cbb56661253122d15df8bbbde9da8", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-qemu-alpine-ssh-main-20261019.json": "ac728a1b36f9f6ea427fe44483042b0617527ee3cfb9ef097beb928f830dae45", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-query-optimize-main-20261019.json": "6707307d9c0bd6c32a577dbb1196ba1db00587887fcc9cb73e639db78c9aa460", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-regex-log-main-20261019.json": "74e08e996091a65a95c3c0371f5e23f331caf3ed31c90fbfe9e1d98d33cf868e", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-torch-pipeline-parallelism-main-20261019.json": "2b7c16b03b3722b5eec4fcba642574f2ca3626ab6a6229955cc7d6cb07734b23", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-torch-tensor-parallelism-main-20261019.json": "045a4325a02aabda7959a2a1b36ec9e846eea35bb470b558e49b59167bee4023", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-write-compressor-main-20261019.json": "14bd7f44dc69ecee099aa1c342a6a4553764ef5bb572c4b075005b0a379bf22b", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-constraints-scheduling-main-20261019.json": "9856526e760107eb0f3970c12e09bbc01658d2522664467c81610be35a1bbb5d", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-custom-memory-heap-crash-main-20261019.json": "ffab43dfae1d34f8c27fcf281799daecde22fad47c4cbfaa0ae1352a97bddb5d", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-dna-assembly-main-20261019.json": "6fae1f9e0b87bfe43b044035d8cbce0bb68ca6f8cca4f6af8f0c5d1475ff6301", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-llm-inference-batching-scheduler-main-20261019.json": "2237de012cb19d4556f328f70dd8391406fd00a7a39f47e52f0f17082d58f67a", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-path-tracing-main-20261019.json": "87355a3fa31c01142daeeb12d5f1c2ac96ef32875ba420ae44f655001a9eebe3", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-schemelike-metacircular-eval-main-20261019.json": "a65854ec5768dd3d3ec708a67aa61c44332ea43cf3d127863631c590348e247c", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-caffe-cifar-10-main-20261019.json": "0892465dd9f26ae8bf16d5d5ccf22fd3c1cd2d44ce21fe84724c5bcf84d93d80", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-circuit-fibsqrt-main-20261019.json": "a41bc43fee2a78760b298e399c5063195520b34a8784ba3fd96f8c99c17f2266", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-mteb-leaderboard-main-20261019.json": "7faf1011207025a01e341d0e5ffb6a6a115280aad5bca36f0d637df214fb2117", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-regex-chess-main-20261019.json": "072135ae9a8b36b01c65e13508ecd187698c4ddc991f6b297f2325caeb1075b0", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-build-pov-ray-main-20261019.json": "6d2240e63e519480597fdcdf78b5ba0c3f242a74b5794c5c86f4c7f15ada6518", + "jobs/dev34-1017-launch-recovery-git-multibranch-20261019-orchestration/dev34-1017-git-multibranch-launch-recovery-main-20261019.json": "000b0fd4263ccc5a4c4e1d8841f0a8c1c994fe9b82e998da71c26d2dc01cb7fd", + "jobs/dev34-1017-launch-recovery-kv-store-grpc-20261019-orchestration/dev34-1017-kv-store-grpc-launch-recovery-main-20261019.json": "e04771edea93444fa16ae3c55a055dc7330d39c7e0a5c455ee23aecc2c5f0e90", + "jobs/dev34-1017-launch-recovery-log-summary-date-ranges-20261019-orchestration/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019.json": "2a9301b40166a74b36047f3d4eba5e054ed96e877dc1249b45a0833bacd898c3" + } + }, + "integrity": { + "source_and_task_provenance_passed": 34, + "accepted_preflight_tasks": 34, + "primary_pair_barriers_passed": true, + "maximum_concurrent_task_jobs": 2, + "supplemental_jobs_serialized": true, + "automatic_whole_trial_retries": 0, + "repeated_agent_trials": 0, + "wheel_digests": [ + "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f" + ] + }, + "aggregate": { + "tasks": 34, + "classifications": { + "passed": 26, + "scored_failure": 5, + "verifier_error": 3 + }, + "passed_fraction_of_pool": 0.7647058823529411, + "official_pytest_counts": { + "passed": 133, + "failed": 8, + "skipped": 0, + "errors": 1, + "xfailed": 0, + "xpassed": 0 + }, + "verifier_attempts": 37, + "verifier_retries": 3, + "wall_seconds": 15853.918955, + "estimated_known_cost_usd": 1.860998832, + "usage_incomplete_tasks": [], + "cost_incomplete_tasks": [], + "recorded_tokens": { + "prompt_tokens": 47814612, + "completion_tokens": 1153308, + "cached_tokens": 47167872 + }, + "assertion_cases": 141, + "collection_errors": 1, + "agent_trials_started": 34, + "host_launch_errors_before_model": 3 + }, + "trials": [ + { + "order": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "task_ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "trial": "adaptive-rejection-sampler__iDY9wKc", + "job": "dev34-1017-adaptive-rejection-sampler-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 9, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 9, + "passed": 9, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418210.3668673, + "stop": 1792418212.3392768 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T13:45:28.424096Z", + "finished_at": "2026-10-19T13:57:13.521256Z", + "seconds": { + "environment_setup": 2.15897, + "agent_setup": 38.720621, + "agent_execution": 614.378442, + "verifier": 28.486088 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 28.286813046986936 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 16, + "tool_calls": 16, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 461604, + "completion_tokens": 56786, + "cached_tokens": 456064 + }, + "estimated_known_cost_usd": 0.072541584 + }, + "task_checksum": "a4726253ad23c3fc46b5cb9c3d21025874e367b18643a3920f636a1fa89e1207", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/result.json": "8233ceaeab91d8ce59dede46319b4b7742241cff3a94aa7b394ee21997b57783", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/config.json": "63ac0087fdf45eda94a222d75d7b95e105db921e70f520cdd73df2dd0cb87012", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/agent/bootstrap.json": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/agent/trajectory.json": "c2138e5b24120eedc5848d6b7d04f7143f06077772ca336bc462c00a4a8a277b", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/agent/nanopycodeagent.txt": "87c93b9815d7598af471ed7cb51a0fd299f34c3a161c8e7d59e60d695a8774ba", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/verifier/ctrf.json": "fd39e3bbab0f18d663991ec311aa897c93c50ef1fd7dc1430011a135921790b2", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/verifier/test-stdout.txt": "cef65d3ff3c08eb09fc3df5d3926d28ca5ae370fff62d3dcc3ac497661f40d77", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/verifier/retry-summary.json": "f432608bc84fcef83046475ec480371cd1376e51d6dd75d57921a131e95dff1e", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019-input/manifest.json": "ce1a78a80be9b624e2f7a10c2ed9403647e1ebe8861eabc738b94458ea8d7d0d", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019-input/config.json": "a6eff2fb5c7ddaa5c69d942c5103a0d50af79cd21963647353adfd359d88bdca", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-adaptive-rejection-sampler-main-20261019.json": "c65cc8332c5cb6acb63bd50777e17a15928f75893cc0427486e90aedc6b96133", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/verifier/attempts/1/test-stdout.txt": "cef65d3ff3c08eb09fc3df5d3926d28ca5ae370fff62d3dcc3ac497661f40d77", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/verifier/attempts/1/ctrf.json": "fd39e3bbab0f18d663991ec311aa897c93c50ef1fd7dc1430011a135921790b2", + "jobs/main-source/jobs/dev34-1017-adaptive-rejection-sampler-main-20261019/adaptive-rejection-sampler__iDY9wKc/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 0.0, + "audit_flags": [] + }, + { + "order": 2, + "task": "terminal-bench/build-cython-ext", + "task_ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12", + "trial": "build-cython-ext__Cuqzzfq", + "job": "dev34-1017-build-cython-ext-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 11, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 11, + "passed": 11, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418009.531498, + "stop": 1792418015.4700766 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T13:45:29.062582Z", + "finished_at": "2026-10-19T13:53:56.776236Z", + "seconds": { + "environment_setup": 2.12337, + "agent_setup": 33.636585, + "agent_execution": 436.324108, + "verifier": 14.245949 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 14.043371507999836 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 81, + "tool_calls": 88, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 2612581, + "completion_tokens": 25529, + "cached_tokens": 2568704 + }, + "estimated_known_cost_usd": 0.059210124 + }, + "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/result.json": "2bfdf7c9f5c3f4cf76ccb082ec521e77e621dfef53624383949ee266f4240ed4", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/config.json": "457b737e772acad9b92b8f58ddc5e18010c4934f26258bd4435cd51ab30b74c1", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/agent/bootstrap.json": "324a224408951769384fa327b2681ea1d72a8f50bee2f713befaeab399be1f8e", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/agent/trajectory.json": "30fc671e381332bb1182377dc64cf7b9742dd53500b22e2e2641934764f2a725", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/agent/nanopycodeagent.txt": "b53a65bd6b3ee3a8c6f6f0bdf820d7196c72a3316eb739cbedd6a3ee8c0148bf", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/verifier/ctrf.json": "ce134cc2965ea11a395677749db62a4e6a12f27c7867148fc79a86584248f0ca", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/verifier/test-stdout.txt": "00e997fd96471b4ea976f13a1a791a4c70967998a36fa45e50d43adba037cb88", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/verifier/retry-summary.json": "3ee30fdd10ce11129c722c5b11e9c45442f1d909d867271191624238de4d6d01", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019-input/manifest.json": "8d554660616432187374c3690f652ce6f77281e694aa5edffba0c81cabeb6c38", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019-input/config.json": "0341de047eff7c16c5fc6a397c20542dbe0d94c49f54dd4718e5745721e187db", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-build-cython-ext-main-20261019.json": "e9b0d1920debba88a1d0759043fe6eaf036b7191bbca2a3ee75e11a65a81eb39", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/verifier/attempts/1/test-stdout.txt": "00e997fd96471b4ea976f13a1a791a4c70967998a36fa45e50d43adba037cb88", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/verifier/attempts/1/ctrf.json": "ce134cc2965ea11a395677749db62a4e6a12f27c7867148fc79a86584248f0ca", + "jobs/main-source/jobs/dev34-1017-build-cython-ext-main-20261019/build-cython-ext__Cuqzzfq/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 3, + "task": "terminal-bench/cancel-async-tasks", + "task_ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1", + "trial": "cancel-async-tasks__RS9TwnB", + "job": "dev34-1017-cancel-async-tasks-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418475.3665807, + "stop": 1792418489.0334237 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T13:57:21.327463Z", + "finished_at": "2026-10-19T14:01:50.061143Z", + "seconds": { + "environment_setup": 2.266634, + "agent_setup": 34.767878, + "agent_execution": 179.834086, + "verifier": 30.644991 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 30.439429168007337 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 16, + "tool_calls": 16, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 245992, + "completion_tokens": 17662, + "cached_tokens": 240896 + }, + "estimated_known_cost_usd": 0.024168576 + }, + "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/result.json": "08aca38c1250e5ad12a4a862bec63bdc539099e3c144a0f7e10666d9807ef355", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/config.json": "be7a2a2f469062d6b8a6715eb84c09608c5329690f595c6dcf818c970dd5b259", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/agent/trajectory.json": "679703e63b26a3d732fccafd53524d768344a67f6dfd281d7ea81847c8eb3fe7", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/agent/nanopycodeagent.txt": "288317e1c4d6f32b10aa8ccf942b66a04d77b16214cfdf1a3d541a77d6611efe", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/verifier/ctrf.json": "accbf455a4a65faa6df06faea9df157879bf762da86bdb71480d26b749b9f384", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/verifier/test-stdout.txt": "4aa94adfde2392cde7e0b7febd175f85c210969b93b2a5940094ec8bed83b648", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/verifier/retry-summary.json": "f9eb394d8c156f964ad1df059b4d508c9cb2d21cb5ea906d76246d901be2f463", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019-input/manifest.json": "dbd5f24ecea1c6dda7f65436831058e106a4414a3f7a7c48c6ba72f4d1fc416c", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019-input/config.json": "109f7211165cdf9cb3ba1311d61ebaef1e86155ea3961a241c8622672b976df3", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-cancel-async-tasks-main-20261019.json": "f67e634f29019d581a04553b8bf6b4a0c3299f1148599a55d66a8509854effec", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/verifier/attempts/1/test-stdout.txt": "4aa94adfde2392cde7e0b7febd175f85c210969b93b2a5940094ec8bed83b648", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/verifier/attempts/1/ctrf.json": "accbf455a4a65faa6df06faea9df157879bf762da86bdb71480d26b749b9f384", + "jobs/main-source/jobs/dev34-1017-cancel-async-tasks-main-20261019/cancel-async-tasks__RS9TwnB/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 4, + "task": "terminal-bench/cobol-modernization", + "task_ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "trial": "cobol-modernization__xm3Uepa", + "job": "dev34-1017-cobol-modernization-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418555.9710445, + "stop": 1792418556.0224457 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T13:57:22.506247Z", + "finished_at": "2026-10-19T14:02:57.046128Z", + "seconds": { + "environment_setup": 2.12275, + "agent_setup": 34.344748, + "agent_execution": 261.983233, + "verifier": 14.850577 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 14.651533730007941 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 37, + "tool_calls": 37, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 1257727, + "completion_tokens": 47192, + "cached_tokens": 1238784 + }, + "estimated_known_cost_usd": 0.069746004 + }, + "task_checksum": "5e3806427eaaaa9f0f756a2a87d4f7f50ec1c8a0d329b26bab7dc09657b937cf", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/result.json": "acecd9b897ddece20abff88f9a3417f3ad8a542e7ff9c848d242a427a3c77f16", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/config.json": "5fed0b61b4ca1ff8b11f91004af289843dc77b8079701838dcb333d9bf4e78d4", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/agent/bootstrap.json": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/agent/trajectory.json": "4517ae2062f064a8201a244512411fa1f9ce0f71e4034f0ca3a03f9bf469f70e", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/agent/nanopycodeagent.txt": "9d95990d3702c67fadbecf6fef313cb3ea079259eb8d9a971d886bbf9f4010b0", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/verifier/ctrf.json": "c0b48c6829ec41e466090ddd2ebfa32bd8de65f9300b9fb395672670e8608cd8", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/verifier/test-stdout.txt": "936ba063dc2ffecb2a880c2e0555e78828bbb8eb184262fbfc800c6690c42e55", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/verifier/retry-summary.json": "ab1edf03242b69f7e708f6f05a4720dd691518603b3e6f65322d7867d208cab1", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019-input/manifest.json": "05364aadfc2ea3e26c50bd292efb935082fbc5e935b1f3f949a1ca46d4ead48e", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019-input/config.json": "dfa967f751eaebd9ef40dafb13efe6501abae2fb0ac6c475a5bf43de8415f981", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-cobol-modernization-main-20261019.json": "0b63ad5e8d68ddae2a1fd50ed5ee6f6fb6ab2185f35770bebeeb69da84059d92", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/verifier/attempts/1/test-stdout.txt": "936ba063dc2ffecb2a880c2e0555e78828bbb8eb184262fbfc800c6690c42e55", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/verifier/attempts/1/ctrf.json": "c0b48c6829ec41e466090ddd2ebfa32bd8de65f9300b9fb395672670e8608cd8", + "jobs/main-source/jobs/dev34-1017-cobol-modernization-main-20261019/cobol-modernization__xm3Uepa/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 5, + "task": "terminal-bench/db-wal-recovery", + "task_ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "trial": "db-wal-recovery__34E39iw", + "job": "dev34-1017-db-wal-recovery-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": 0.0, + "classification": "scored_failure", + "exception_type": null, + "official_pytest_counts": { + "passed": 5, + "failed": 2, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 7, + "passed": 5, + "failed": 2, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418801.7521174, + "stop": 1792418801.7792633 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:03:04.869197Z", + "finished_at": "2026-10-19T14:07:02.920077Z", + "seconds": { + "environment_setup": 2.143365, + "agent_setup": 35.959139, + "agent_execution": 132.80224, + "verifier": 45.824029 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 45.63358940600301 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 22, + "tool_calls": 31, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 437241, + "completion_tokens": 23349, + "cached_tokens": 421376 + }, + "estimated_known_cost_usd": 0.035306556 + }, + "task_checksum": "c18abdc4fdc3a01bf374c55a9700708fe6a9662077d29db81abb692f0a3c5f6f", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/result.json": "aa77bf501aacc0bb8bf6c2510804736478589c053d6850830fd5eb3e607117bf", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/config.json": "d4ce4cb035bab0046f43842be5c78158553436ed39863cb1902fe5849d76888e", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/agent/bootstrap.json": "b5e3b1241fffd9cb9c124014d0da1221f3dd38771c762332e33db42dd948a9b8", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/agent/trajectory.json": "8bbd1b37eecfd3452f56d3d964d55489e6ff560b56db241c25b57c3a36f0b87d", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/agent/nanopycodeagent.txt": "e963fc1cc8b03fdbc9f22e7c6e2c6842f41ce99a1a8f05c39b9979ce1fd984bd", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/verifier/ctrf.json": "421a06f1024c7865ce58a72aa50ab0e8bf85f990f8e573d02a538a8201e088ce", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/verifier/test-stdout.txt": "17f5e7b5e4c6725702081bed4bbba126040bee42935438ef06710d62941f8dd8", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/verifier/retry-summary.json": "6dd23b164ef51a2183dac770ed4eac5b682322a27bc382c729c2fd5416b5cc7c", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019-input/manifest.json": "372855aff59efe2e1c098e80d7a257f1649b1a1cb394c0c1811da4aee0bd10c8", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019-input/config.json": "f53940ccad1984fbca4ba2523bf4472b6de747f72ca76f224a52fb6fbfba6f29", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-db-wal-recovery-main-20261019.json": "e57d7624d91538e03db29d4faac9cd7ade0934a34e891a7005258c52c01f1e88", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/verifier/attempts/1/test-stdout.txt": "17f5e7b5e4c6725702081bed4bbba126040bee42935438ef06710d62941f8dd8", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/verifier/attempts/1/ctrf.json": "421a06f1024c7865ce58a72aa50ab0e8bf85f990f8e573d02a538a8201e088ce", + "jobs/main-source/jobs/dev34-1017-db-wal-recovery-main-20261019/db-wal-recovery__34E39iw/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 0.0, + "analysis_note": "Opened the original SQLite database before preserving its obfuscated WAL. The agent reported that SQLite removed the WAL, then inferred missing rows. Official tests found missing WAL updates: five passed and two failed.", + "audit_flags": [ + "inspected_empty_verifier_log_directory; no hidden test content observed" + ] + }, + { + "order": 6, + "task": "terminal-bench/fix-code-vulnerability", + "task_ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "trial": "fix-code-vulnerability__UYGKsiC", + "job": "dev34-1017-fix-code-vulnerability-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418645.7374392, + "stop": 1792418645.7557945 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:03:03.835450Z", + "finished_at": "2026-10-19T14:04:26.775050Z", + "seconds": { + "environment_setup": 2.266803, + "agent_setup": 26.073925, + "agent_execution": 24.561388, + "verifier": 8.839953 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 8.637557433001348 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 9, + "tool_calls": 12, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 61455, + "completion_tokens": 2703, + "cached_tokens": 55168 + }, + "estimated_known_cost_usd": 0.005460708 + }, + "task_checksum": "80d707a96acfd0e19af96f0f0c8f44deefdbf167538fefbc0d1dc6391392090a", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/result.json": "12a89d3c9e1b5c71fac089366aab8aabc86834798842002603811d534c307d51", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/config.json": "082023b7bf5a1b2681cc3b9afa2657cc49b9156a82584403bcefc5814bf370da", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/agent/bootstrap.json": "2ae9644664cb84e658bef87ecae8383314d569226c5ce17f2866b8663958ef92", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/agent/trajectory.json": "92982c39b055a82c8878d216486debbdeaffc1cb06c68708374e55ecab560304", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/agent/nanopycodeagent.txt": "792acb2879210bea82d27b04b61b2b29c61e2e951d54e90fc049927286bacce5", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/ctrf.json": "9d79f99255b09af7982b7a47d23b15c9ff03d059711783c2c44fa99aeb832e45", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/test-stdout.txt": "5b1c1b620157e4e7ee7e5368a1cd9f8de20235ae042eaeeb7f266332d7e38f69", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/retry-summary.json": "7432d82901ba2c0357fab9d312b5e23ff0f38a588ac0ceff82187800444e685f", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019-input/manifest.json": "296e256d9b9e20be5ed14240c91157aad004c99e0e00de735a024184e2725a0a", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019-input/config.json": "ba42145a47a0a7cbdf403e3a2cbcbf83316f193d8c90b08ac96a87c9f389512c", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-fix-code-vulnerability-main-20261019.json": "0df3a5945b6c4235ff4ac9ee34eb2493cf9438f3719bec5b29524595380680f9", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/attempts/1/test-stdout.txt": "5b1c1b620157e4e7ee7e5368a1cd9f8de20235ae042eaeeb7f266332d7e38f69", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/attempts/1/original-repo-ctrf.json": "c5c7d6933fafd641ac4b33964a75d3e44f970bd938053816344cbfd7a5eb9d04", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/attempts/1/ctrf.json": "9d79f99255b09af7982b7a47d23b15c9ff03d059711783c2c44fa99aeb832e45", + "jobs/main-source/jobs/dev34-1017-fix-code-vulnerability-main-20261019/fix-code-vulnerability__UYGKsiC/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 7, + "task": "terminal-bench/git-multibranch", + "task_ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "trial": "git-multibranch__B6QEuv6", + "job": "dev34-1017-git-multibranch-launch-recovery-main-20261019", + "raw_reward": 1.0, + "execution_phase": "supplemental", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 1, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792432520.9839647, + "stop": 1792432617.0065095 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T17:50:50.094875Z", + "finished_at": "2026-10-19T17:57:18.008673Z", + "seconds": { + "environment_setup": 2.240131, + "agent_setup": 27.094056, + "agent_execution": 220.201341, + "verifier": 117.172303 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 116.93149315798655 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 26, + "tool_calls": 31, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 303457, + "completion_tokens": 13739, + "cached_tokens": 295680 + }, + "estimated_known_cost_usd": 0.02059398 + }, + "task_checksum": "abe8a7f8f9dcb0e2170740e4735559b368cecd925ddee09c9ed81a7c67dce90a", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/result.json": "50d2828f760c670e6ad2807ee46bfa9e61c1f580745b119d577d70ba2d7d6d81", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/config.json": "b16d897f8fcf612456aec596f649641e82f64506d57496dd4111b5101735c98d", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/agent/bootstrap.json": "5d34c6089fc9b0f47d3b11e44320220f7075451e71e035a05e56df2238baf1aa", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/agent/trajectory.json": "fdc689fa89a6de1d951a68db64394c6add99afc62feaebd4853c71faa0bdceb9", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/agent/nanopycodeagent.txt": "b40fee2719aa1086d06c3c9dd7ca280b376e685155b7a7deb8ebf8d412850b79", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/verifier/ctrf.json": "09796e2a8834f1d8c8f73b6ca3eaff7915f83b9a6255c64953a9c48ba058cbae", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/verifier/test-stdout.txt": "df69e90edafea6c06323641c458a669a90f016d0cd6983a7c2844d0a201832f2", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/verifier/retry-summary.json": "27c231f8c225f4b24a6fe1b653689dcd61a21eeab33ec820cedbb04adf0e6c0d", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019-input/manifest.json": "a546906407d4c6d8753d5197e09df6b7faf0a46e221f5e5cf24340662580fe51", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019-input/config.json": "92dbe01fc2ec10e1c59925329299630430e56c05611f50f43214b7623a38e382", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-launch-recovery-git-multibranch-20261019-orchestration/dev34-1017-git-multibranch-launch-recovery-main-20261019.json": "000b0fd4263ccc5a4c4e1d8841f0a8c1c994fe9b82e998da71c26d2dc01cb7fd", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/verifier/attempts/1/test-stdout.txt": "df69e90edafea6c06323641c458a669a90f016d0cd6983a7c2844d0a201832f2", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/verifier/attempts/1/ctrf.json": "09796e2a8834f1d8c8f73b6ca3eaff7915f83b9a6255c64953a9c48ba058cbae", + "jobs/main-source/jobs/dev34-1017-git-multibranch-launch-recovery-main-20261019/git-multibranch__B6QEuv6/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 8, + "task": "terminal-bench/kv-store-grpc", + "task_ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307", + "trial": "kv-store-grpc__RZYUtbH", + "job": "dev34-1017-kv-store-grpc-launch-recovery-main-20261019", + "raw_reward": 1.0, + "execution_phase": "supplemental", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 7, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 7, + "passed": 7, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792432990.8987527, + "stop": 1792432990.9605036 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T17:57:24.207934Z", + "finished_at": "2026-10-19T18:03:32.025810Z", + "seconds": { + "environment_setup": 2.191329, + "agent_setup": 33.744934, + "agent_execution": 297.099239, + "verifier": 13.500335 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 13.297425041004317 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 17, + "tool_calls": 17, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 100143, + "completion_tokens": 5431, + "cached_tokens": 94848 + }, + "estimated_known_cost_usd": 0.008674788 + }, + "task_checksum": "901a4dd5c3078d5155043875a0d07f3583f462c73f919b52855c025e680b1edd", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/result.json": "9edbceee80d70ba06a48e299c8673353477f047119732ad23703227dbb183465", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/config.json": "334628b3014b8c1e56edb7e6dc37c7c69c7c6a991746277811cfae7b6f901319", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/agent/trajectory.json": "8d76980d966223e4c852352a8c7bfd2d80f1a4e75e3c9652641b0a84be8759e3", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/agent/nanopycodeagent.txt": "302046a85a6cb531b17f286d2d91b3e8c16f575fafbe3d58bfe6d0dfea221c30", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/verifier/ctrf.json": "6d9fa0436040a1f0cccada55339ff6859e3a8dbe9146b57c0f37b18be99f63e3", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/verifier/test-stdout.txt": "9468fbd0e3181a831203cb19761cefaac4599bcc93c9b4654511dbcdf895d074", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/verifier/retry-summary.json": "9929f26587ff6c3517ba609152972ce4d0ab2056edc085ff90a921eba86f7f37", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019-input/manifest.json": "65f8fc8e9c4509d9f3a6fda36759e62f96f59829accd7f396aa3b00a231e719b", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019-input/config.json": "bf02783b43629129355e914082600fa1a70777e6a4236a347cddd93fb8425a5f", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-launch-recovery-kv-store-grpc-20261019-orchestration/dev34-1017-kv-store-grpc-launch-recovery-main-20261019.json": "e04771edea93444fa16ae3c55a055dc7330d39c7e0a5c455ee23aecc2c5f0e90", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/verifier/attempts/1/test-stdout.txt": "9468fbd0e3181a831203cb19761cefaac4599bcc93c9b4654511dbcdf895d074", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/verifier/attempts/1/ctrf.json": "6d9fa0436040a1f0cccada55339ff6859e3a8dbe9146b57c0f37b18be99f63e3", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-launch-recovery-main-20261019/kv-store-grpc__RZYUtbH/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 0.0, + "audit_flags": [] + }, + { + "order": 9, + "task": "terminal-bench/log-summary-date-ranges", + "task_ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6", + "trial": "log-summary-date-ranges__K7yxDjX", + "job": "dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019", + "raw_reward": 1.0, + "execution_phase": "supplemental", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 2, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 2, + "passed": 2, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792433090.9174802, + "stop": 1792433090.9318697 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T18:03:39.407332Z", + "finished_at": "2026-10-19T18:05:12.004509Z", + "seconds": { + "environment_setup": 2.226844, + "agent_setup": 30.80336, + "agent_execution": 25.804958, + "verifier": 12.499822 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 12.294224377023056 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 9, + "tool_calls": 12, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 76526, + "completion_tokens": 3326, + "cached_tokens": 68480 + }, + "estimated_known_cost_usd": 0.00681588 + }, + "task_checksum": "c833c594814ec7b8cb32eba3b9cb5ed648171efe5a074767aa64c25ea060f08f", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/result.json": "120b4e93476914212100ef638eab9d5669bc798ef51ccbf61913cee5a3e00c0d", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/config.json": "1964947facc5dfb5889903590ca33c962d02ff73a3f2baca96080506125c3226", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/agent/trajectory.json": "73186581c257b9ff90e2eec937178c71870722b22b11bd256c8b404a5e2eb199", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/agent/nanopycodeagent.txt": "79306cfc8f5e124b8e118011c90c48534ff2e2c9f9e846bd2ac9f1339c5c3668", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/verifier/ctrf.json": "9d643406d55aab731071f8ca606f2e91930b9b68419da7853162dbdb1bfc6ba2", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/verifier/test-stdout.txt": "c8676937febab83ad3c464c80551cce18b54ef4b7eb6da1690ad35db855f20c8", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/verifier/retry-summary.json": "847d9101039f292e09a7f44337b004292ebbf507549fd5f911acd37b511175ad", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019-input/manifest.json": "45f5b9df85ecf04c118a3d7a15d4fd77cf8744ae7b96ea180be8aa1d15c46be9", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019-input/config.json": "62f8229dc038021cc505839b084e29039e5bca40f1d873d88a96120c494f8bec", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-launch-recovery-log-summary-date-ranges-20261019-orchestration/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019.json": "2a9301b40166a74b36047f3d4eba5e054ed96e877dc1249b45a0833bacd898c3", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/verifier/attempts/1/test-stdout.txt": "c8676937febab83ad3c464c80551cce18b54ef4b7eb6da1690ad35db855f20c8", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/verifier/attempts/1/ctrf.json": "9d643406d55aab731071f8ca606f2e91930b9b68419da7853162dbdb1bfc6ba2", + "jobs/main-source/jobs/dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019/log-summary-date-ranges__K7yxDjX/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 10, + "task": "terminal-bench/merge-diff-arc-agi-task", + "task_ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d", + "trial": "merge-diff-arc-agi-task__G7pgsxV", + "job": "dev34-1017-merge-diff-arc-agi-task-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 5, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 5, + "passed": 5, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792418988.167082, + "stop": 1792418988.1868722 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:07:51.691088Z", + "finished_at": "2026-10-19T14:10:09.316123Z", + "seconds": { + "environment_setup": 2.10701, + "agent_setup": 39.683989, + "agent_execution": 58.618606, + "verifier": 15.907554 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 15.706166567993932 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 15, + "tool_calls": 18, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 142325, + "completion_tokens": 9115, + "cached_tokens": 135424 + }, + "estimated_known_cost_usd": 0.013820844 + }, + "task_checksum": "f6ac6f6b96613c5a110b0b0d51172b7cad781c0adc99b556425408d8d3278147", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/result.json": "bc4a421b865887525c7862cd5b98431ebfb8687ffa29e5287dc302dec381357d", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/config.json": "f20c9a0764ddc08729695368403b32dd3819f2adcaaa9a5143a889ab1a6886e9", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/agent/bootstrap.json": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/agent/trajectory.json": "857e4271887e4f1536a370908df913458ebb6a4c8512ee23d9209afaf0c16e66", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/agent/nanopycodeagent.txt": "304f7dda29ea6f620616344a75443c719230190cda74726cfdb3f25715d876f7", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/verifier/ctrf.json": "fad0d6b3d7971fc565ad0bc9d49c52fa8b53ede52d24fd2d4707f4efd5dcd8d0", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/verifier/test-stdout.txt": "7efdc4cf6ca0f5310f3c7eee850907e368382ce047ac1057c966bf9640491a2d", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/verifier/retry-summary.json": "1ca3b80b15e1f79ab62b4a3efcfe7782ea24c2c99d448753580f2137de70d064", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019-input/manifest.json": "662b2702cde4532adbbd8686d5f666ade2a6f38a8ee39949d2cbc954e9e2b6b0", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019-input/config.json": "5137a8fb277ee52738f2da75abad60cb2aa7e2de69f3c6cf81d122426720f968", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-merge-diff-arc-agi-task-main-20261019.json": "54a91b4a7a072c56c4e20b650897f5f9aa9813aaf961424c71dcb5d1be266aab", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/verifier/attempts/1/test-stdout.txt": "7efdc4cf6ca0f5310f3c7eee850907e368382ce047ac1057c966bf9640491a2d", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/verifier/attempts/1/ctrf.json": "fad0d6b3d7971fc565ad0bc9d49c52fa8b53ede52d24fd2d4707f4efd5dcd8d0", + "jobs/main-source/jobs/dev34-1017-merge-diff-arc-agi-task-main-20261019/merge-diff-arc-agi-task__G7pgsxV/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 11, + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "trial": "model-extraction-relu-logits__THcSthq", + "job": "dev34-1017-model-extraction-relu-logits-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 1, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792419692.831144, + "stop": 1792419694.932337 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:10:16.305450Z", + "finished_at": "2026-10-19T14:21:56.031401Z", + "seconds": { + "environment_setup": 2.254607, + "agent_setup": 77.191098, + "agent_execution": 301.393011, + "verifier": 297.628582 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 296.94961491601134 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 26, + "tool_calls": 27, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 883350, + "completion_tokens": 39054, + "cached_tokens": 856832 + }, + "estimated_known_cost_usd": 0.059961192 + }, + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/result.json": "ce268fec9bd4356134c7c864d79e53e29e52cade57ce9250231f7c7d45006bac", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/config.json": "0957c0b09a0bcd94093b176cbfe8013101333358ca15c66a897e51cb16d82ddb", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/agent/trajectory.json": "d90e45a4f497ed6fffe0cfda2bf7291647c8900365ed5b9774d3eb62ff9ed3c4", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/agent/nanopycodeagent.txt": "bf75572ca1562445c33e02cfd004597ab11f9e5fcb2928e306096b1a6f4f1845", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/verifier/ctrf.json": "92d255842c0521d0c341744cb4ccdc3699d50aed2f67c5ef3cccf3c4f5292561", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/verifier/test-stdout.txt": "e8ca20aeedd332d8b107407b9461fcbde03048e911119d706134ae4953305dc0", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/verifier/retry-summary.json": "bec0e59d859aace2fc2db10b2eb28574de699842bbdd51b24e193bea2ccc0cf8", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019-input/manifest.json": "17146b5745e2302377f5107853138a0ceec465fba2efdd6a2ee9d7b6f8cf445b", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019-input/config.json": "be48d6e33cc4e99d7cdd95cf703228fd602290487a8b8aed1f4469c6826ee1e8", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-model-extraction-relu-logits-main-20261019.json": "a02f32f046bd22ba5c10960092e875f823fc61d730c8d20118160c6b80e11542", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/verifier/attempts/1/test-stdout.txt": "e8ca20aeedd332d8b107407b9461fcbde03048e911119d706134ae4953305dc0", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/verifier/attempts/1/ctrf.json": "92d255842c0521d0c341744cb4ccdc3699d50aed2f67c5ef3cccf3c4f5292561", + "jobs/main-source/jobs/dev34-1017-model-extraction-relu-logits-main-20261019/model-extraction-relu-logits__THcSthq/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 12, + "task": "terminal-bench/multi-source-data-merger", + "task_ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "trial": "multi-source-data-merger__ovUnaUM", + "job": "dev34-1017-multi-source-data-merger-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": null, + "classification": "verifier_error", + "exception_type": null, + "official_pytest_counts": null, + "official_ctrf_summary": null, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:10:18.128137Z", + "finished_at": "2026-10-19T14:18:55.946686Z", + "seconds": { + "environment_setup": 2.135313, + "agent_setup": 128.37817, + "agent_execution": 38.242802, + "verifier": 327.34777 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 327.15425308000704 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 8, + "tool_calls": 9, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 47667, + "completion_tokens": 5514, + "cached_tokens": 44288 + }, + "estimated_known_cost_usd": 0.007896228 + }, + "task_checksum": "33fa3b988ff60ec62b6ce40ee455208cb083ac1ac46ddc5247e954c88b9d5e8e", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/result.json": "bd84053547e596ad0f3a8285a80da37e3a22f0eae72f9fe831fea2d526888583", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/config.json": "32879386a7e47a3832e34f78fc25b31375362c8a98f7e94c2f97a56288ab0fda", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/bootstrap.json": "5011ac5d6ab074bd66636cc77aae3a2d966293bcb051b929e7a9cb076f908948", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/trajectory.json": "652ff7db3dd1b8f75a080cb68a12e9a7bbcb73b9ae245b4d52485487e901a424", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/nanopycodeagent.txt": "cb5d1b6d5480a1e87a6236a04545f52a35296ede77de27c7fe9770edbfcc4acd", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/test-stdout.txt": "7690d05931b29fda94b3c858291f9a35e557ebbe826c715725fda790311dba46", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/retry-summary.json": "f637a457f9341eefc7e30edd19559bc31c0bca312791506f4320847b38cd5c53", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/manifest.json": "7fda413bc1c8c6a57d8ab89a72c7f81a9b16f7fdc0140b03cd9c9696584be5ae", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/config.json": "175d9eea47f437346fc37f27016c08a2fb591e4cd345044b6254f6e13ab7c701", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-multi-source-data-merger-main-20261019.json": "1fe2a8510681dc681b0d2145503b4bfb72b60fe667e6fd4c0fcd098324b3aa7e", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/attempts/1/test-stdout.txt": "7690d05931b29fda94b3c858291f9a35e557ebbe826c715725fda790311dba46", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 1.0, + "analysis_note": "The verifier failed to download/extract pyarrow==22.0.0 because of a network timeout before pytest started. Native reward zero is retained, with a null analytical score. This completed attempt was not a verifier-process timeout, so it was not retried.", + "audit_flags": [] + }, + { + "order": 13, + "task": "terminal-bench/nginx-request-logging", + "task_ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "trial": "nginx-request-logging__PLRZDQE", + "job": "dev34-1017-nginx-request-logging-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 8, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 8, + "passed": 8, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792419838.1510918, + "stop": 1792419840.3131654 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:22:05.863613Z", + "finished_at": "2026-10-19T14:24:21.377680Z", + "seconds": { + "environment_setup": 2.142433, + "agent_setup": 33.326882, + "agent_execution": 60.026987, + "verifier": 18.768008 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 18.559525546006626 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 10, + "tool_calls": 12, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 56695, + "completion_tokens": 3582, + "cached_tokens": 51072 + }, + "estimated_known_cost_usd": 0.006291732 + }, + "task_checksum": "913305d8f286ff121b30e4893217142190b27f12952343f22c0999dd2a4d725e", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/result.json": "40e8537c77664d32fc817ace8d72ab9e944cfefe021e39faaaceb00342ce257e", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/config.json": "2b6ad455db2fda2fa679dc6fa799e7100378c4bc412af70e81af74945d8fd0d5", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/agent/bootstrap.json": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/agent/trajectory.json": "37e3d5521eacfae7baa32b3cd9888e5f82e5464795c92973449bf5f311c80da6", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/agent/nanopycodeagent.txt": "af0544b43204ff69271d74e98293457051b2c28562180a98895045bbb94cb5ca", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/verifier/ctrf.json": "3982e2d7f02466ea943e5c125741aba081b32a3dd456385f252a6a098bfa3a9f", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/verifier/test-stdout.txt": "1c6ee83addb831557adb7050c48a8395ec71a3c68579acc6d6b083ff33b0e9c9", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/verifier/retry-summary.json": "d7c46d2b8ec62fb6be37546a8127bcd173e5dd307909217d27734eab3af81991", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019-input/manifest.json": "4e17119521fce2910ee2c41c08808eda7bec47061c49860ff0b44bcab16933fd", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019-input/config.json": "71bc98474c38af612902d619358b60d6faa3d72f5cf23d9e00dc29cb5ea0a52f", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-nginx-request-logging-main-20261019.json": "ddf01b7458253ebe208607579e0e08ece43cd47c28ec67ecc78bc86823715c7b", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/verifier/attempts/1/test-stdout.txt": "1c6ee83addb831557adb7050c48a8395ec71a3c68579acc6d6b083ff33b0e9c9", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/verifier/attempts/1/ctrf.json": "3982e2d7f02466ea943e5c125741aba081b32a3dd456385f252a6a098bfa3a9f", + "jobs/main-source/jobs/dev34-1017-nginx-request-logging-main-20261019/nginx-request-logging__PLRZDQE/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 14, + "task": "terminal-bench/openssl-selfsigned-cert", + "task_ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302", + "trial": "openssl-selfsigned-cert__ekBysEE", + "job": "dev34-1017-openssl-selfsigned-cert-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792419833.8155434, + "stop": 1792419834.0020134 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:22:05.048093Z", + "finished_at": "2026-10-19T14:24:15.009967Z", + "seconds": { + "environment_setup": 2.251139, + "agent_setup": 35.802716, + "agent_execution": 54.393098, + "verifier": 16.294374 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 16.10200829799578 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 16, + "tool_calls": 21, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 127821, + "completion_tokens": 8065, + "cached_tokens": 122112 + }, + "estimated_known_cost_usd": 0.012123372 + }, + "task_checksum": "2b70d5535b5873f644fad37b76dbef86a1e42162e018c7bc06316e5e2521929a", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/result.json": "35bb9971f803aec3678ba7e43a053b6d628bd01cf1415accb520530c9a05756e", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/config.json": "b21d480647d5a8ba790bc3b9a9bb03380ab57c479068558ab4c3a34feb068ae2", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/agent/trajectory.json": "03e25dac324c882ccc94aec1b99a3c18ac151a22831483e6688ebbdbfcfc19f4", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/agent/nanopycodeagent.txt": "27dfe310f7ab14af5f64306290445e79af5b3c8ed85db07c6d206bf0283f368a", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/verifier/ctrf.json": "d82073fb0cec79cc9ec8620b5b29774e85f7a3ad1c40d8566c7cb8b9cebe2333", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/verifier/test-stdout.txt": "950ce344d7f1c5e3c6a608da701468e04235815894a773f0b317639024c473b9", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/verifier/retry-summary.json": "37d83cc953a6dae573b8ba3ab4140c705c963efaac89c478f14a588333ab574f", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019-input/manifest.json": "0167a054aa6abe2fe03f7fcadb20453270923b5691ca666495ad45baca73e00c", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019-input/config.json": "aec6cf8e6a3ab1da62e35904d0e3bb62e225b5ca71f9ff9ee4b6c2bbbe9679d2", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-openssl-selfsigned-cert-main-20261019.json": "36c45c112ab61b7bd1d671b7fa0ec66beb3f865d5a229e5f560be68af9044111", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/verifier/attempts/1/test-stdout.txt": "950ce344d7f1c5e3c6a608da701468e04235815894a773f0b317639024c473b9", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/verifier/attempts/1/ctrf.json": "d82073fb0cec79cc9ec8620b5b29774e85f7a3ad1c40d8566c7cb8b9cebe2333", + "jobs/main-source/jobs/dev34-1017-openssl-selfsigned-cert-main-20261019/openssl-selfsigned-cert__ekBysEE/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 15, + "task": "terminal-bench/pypi-server", + "task_ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1", + "trial": "pypi-server__VxSjRZh", + "job": "dev34-1017-pypi-server-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 1, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792420108.6082883, + "stop": 1792420109.5639458 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:24:28.588097Z", + "finished_at": "2026-10-19T14:28:52.719581Z", + "seconds": { + "environment_setup": 2.242137, + "agent_setup": 32.99102, + "agent_execution": 184.644625, + "verifier": 20.913821 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 20.7130962650117 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 18, + "tool_calls": 27, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 140767, + "completion_tokens": 8107, + "cached_tokens": 134656 + }, + "estimated_known_cost_usd": 0.012369636 + }, + "task_checksum": "9eb21236249d5b42ee64c7229bb1e5a3b3c0be7ed5ee9801bdd0d16ca575941a", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/result.json": "404a9ca5ad19f943136fdaa776a222323417d342196e6c719163382b5180c0bd", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/config.json": "34b59751b4966217db23317011caa35224c122a8811bcddd67a7d7b84b820d33", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/agent/bootstrap.json": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/agent/trajectory.json": "8f2ff0271d12613674e9e7bc1de93a4027d27ae9bad4311fcad7a65759173809", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/agent/nanopycodeagent.txt": "a6f679987d7e94a52b6d45ee7da88affe0d8cd23ded3bf666e9ee3d366219d3b", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/verifier/ctrf.json": "ac48805c30ce9f57c2d41c237cb9fa1556ad166ebfdf599694fa932d99b4b0a8", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/verifier/test-stdout.txt": "df6d1efbdcf4e1f0aeb5eecb216102db8e1fb16057403a6535d1886fb46e6904", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/verifier/retry-summary.json": "adb7987ee1bf81a17b20359b6c28604ce44c9f4a697b6690fcdf5c78defd4f28", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019-input/manifest.json": "3bab5f1d854128866a9bc4e7e5979c9f80b27eeb6e08e5d327e51779c594fb52", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019-input/config.json": "3b4fc23ec3d551f8174356f0309151389dc9d9438fe3013bb3627184446cc08e", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pypi-server-main-20261019.json": "3d875d5703a2f9ca5729a4d10162b5c45c9f4ab67b8d7fc84d53d2b3f89143a3", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/verifier/attempts/1/test-stdout.txt": "df6d1efbdcf4e1f0aeb5eecb216102db8e1fb16057403a6535d1886fb46e6904", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/verifier/attempts/1/ctrf.json": "ac48805c30ce9f57c2d41c237cb9fa1556ad166ebfdf599694fa932d99b4b0a8", + "jobs/main-source/jobs/dev34-1017-pypi-server-main-20261019/pypi-server__VxSjRZh/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 16, + "task": "terminal-bench/pytorch-model-cli", + "task_ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "trial": "pytorch-model-cli__t73XoLw", + "job": "dev34-1017-pytorch-model-cli-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": null, + "classification": "verifier_error", + "exception_type": null, + "official_pytest_counts": { + "passed": 0, + "failed": 0, + "skipped": 0, + "errors": 1, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 0, + "passed": 0, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792420505.2483206, + "stop": 1792420505.5948339 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:24:29.184622Z", + "finished_at": "2026-10-19T14:35:27.272927Z", + "seconds": { + "environment_setup": 2.129023, + "agent_setup": 39.301351, + "agent_execution": 461.315098, + "verifier": 133.491307 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 133.28830967300746 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 29, + "tool_calls": 39, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 456966, + "completion_tokens": 16625, + "cached_tokens": 445824 + }, + "estimated_known_cost_usd": 0.025967544 + }, + "task_checksum": "e8438573f503c2d98ad71905862638bfa485470010d15d5b092821e0096b9daa", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/result.json": "4847cac28aefa9154528ed40833214b4401c0065e2eee64aeeb8bdec6ec0fbb1", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/config.json": "b80389d25c6b68223471bb82b4ca1c5f47bedd16e6de6c7ad08887957e8ab398", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/trajectory.json": "9baf56afbccfcb6f39ab2df5c5c7f21b7684783d5f32fee2693897f337bfe446", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/nanopycodeagent.txt": "c689bf519d6c856396aa49c03dfb0a1bb65ad9dd46eb905372df6f595ca00872", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/ctrf.json": "0383370176312a002520c110ff53a75664836ed3d627225fd93b76ccd89f9166", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/test-stdout.txt": "dad670fa7c0265c95408dc5f33e01b4dc9d164574ab5e6625b75ecb289bc306f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/retry-summary.json": "9d5e42ed1f72ed1b2bbe1de2724041d8f06eec6bc2e79cad5b52b49131d0ed5b", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/manifest.json": "84502d6134216bcf9b867ed4a206b237fd14dc7a204b226888e75a64fc11ce97", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/config.json": "1b9e5409dc8e2c45449188a96cd3b0399cd99a60900dda4569273848f19ee407", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-cli-main-20261019.json": "98e3548ee417240c814b98bb07b6dd53a772e83e30e6f555f32565c7dc18ebe1", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/test-stdout.txt": "dad670fa7c0265c95408dc5f33e01b4dc9d164574ab5e6625b75ecb289bc306f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/ctrf.json": "0383370176312a002520c110ff53a75664836ed3d627225fd93b76ccd89f9166", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 1.0, + "analysis_note": "Official pytest collection failed importing cv2 because libGL.so.1 was missing after APT reported unmet dependencies. The agent had force-installed compiler/development packages with dpkg --force-all, leaving dependency problems; agent environment modifications contributed to the verification failure. Zero assertions executed. Native reward zero is retained, with a null analytical score; CLI correctness remains unverified.", + "audit_flags": [ + "forced_dpkg_installations_contributed_to_broken_verifier_dependencies" + ] + }, + { + "order": 17, + "task": "terminal-bench/pytorch-model-recovery", + "task_ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "trial": "pytorch-model-recovery__hogG9gA", + "job": "dev34-1017-pytorch-model-recovery-main-20261019", + "raw_reward": null, + "execution_phase": "primary", + "analytical_score": null, + "classification": "verifier_error", + "exception_type": "VerifierTimeoutError", + "official_pytest_counts": null, + "official_ctrf_summary": null, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:35:40.309857Z", + "finished_at": "2026-10-19T15:24:07.773335Z", + "seconds": { + "environment_setup": 2.139091, + "agent_setup": 72.884477, + "agent_execution": 109.773311, + "verifier": 2700.680207 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0023410760041 + }, + { + "attempt": 2, + "status": "timeout", + "duration_seconds": 900.0023236230045 + }, + { + "attempt": 3, + "status": "timeout", + "duration_seconds": 900.00202244299 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 14, + "tool_calls": 14, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 220725, + "completion_tokens": 13235, + "cached_tokens": 210560 + }, + "estimated_known_cost_usd": 0.02019486 + }, + "task_checksum": "e321c0e7501b3f021cfca26520574b343dff3dee6919f2e9dd5ff9e28ffc3725", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/result.json": "8cd798fd024cae52b15271b61516416a3697d9bb4e834a4f03e46878ea9df759", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/config.json": "fedb449dfc06e5ee34620f44b69539990d5753834b8c0de95ac9a355a8146fb3", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/trajectory.json": "7e4230a06c24c1751916117887130adcfc70429a5f4995669df5fc4afc479b4f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/nanopycodeagent.txt": "5dee8328da08df739c2285b303ffc9349c305d1b803232d82d68e56d3a6a6925", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/test-stdout.txt": "d20478d9b2f2a8527862c484b10fd0140c8f86bfa1c587967131dd9c7e998c8d", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/retry-summary.json": "386028e2e0fa4b477bd4342345e31bd3cdf2377d60cad33e5218d09777e0f43a", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/manifest.json": "992ab17d7c24c8e82ee153e98c5aee1805a191faa57bfab0a5693481ee44cb6f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/config.json": "4f6cd8791bdb04ff12ed8d1ae1786a1f947298490305e509c0badc0fb02feaaf", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-recovery-main-20261019.json": "b71e4cca26a8670fcc6294e6744ac49a6d7cbb56661253122d15df8bbbde9da8", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/1/test-stdout.txt": "90a456f12a085a8bf4fc5c23c00a01426f771f06a0d62623cf805cf5e57e0c64", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/2/test-stdout.txt": "9cff11e64d1ca45fcd585def9798cb2aa91d8908fc88350f24af3a770b909927", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/3/test-stdout.txt": "d20478d9b2f2a8527862c484b10fd0140c8f86bfa1c587967131dd9c7e998c8d" + }, + "historical_reward": 1.0, + "analysis_note": "All three original 900-second verifier attempts timed out downloading PyTorch/CUDA dependencies before pytest started. The same output was reused throughout. The native VerifierTimeoutError has no reward and the analytical score is null.", + "audit_flags": [] + }, + { + "order": 18, + "task": "terminal-bench/qemu-alpine-ssh", + "task_ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28", + "trial": "qemu-alpine-ssh__PaghLZ4", + "job": "dev34-1017-qemu-alpine-ssh-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 1, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792421135.5097606, + "stop": 1792421135.8298006 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T14:35:39.150786Z", + "finished_at": "2026-10-19T14:45:57.016961Z", + "seconds": { + "environment_setup": 2.253192, + "agent_setup": 76.998249, + "agent_execution": 506.166282, + "verifier": 11.021919 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 10.792920553998556 + } + ], + "verifier_dependency_profile": { + "profile": "qemu-alpine-ssh-v1", + "status": "passed" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 31, + "tool_calls": 38, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 454018, + "completion_tokens": 18389, + "cached_tokens": 444160 + }, + "estimated_known_cost_usd": 0.02768916 + }, + "task_checksum": "48bb495efc325f96892d45c341d8997c4a13b71bbe7051119f28dcaa7970df4f", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/result.json": "4040607da9067be2549b1a48737c6ac985ca2971dca011f25e4c3fbbd8d584e6", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/config.json": "ee95dbf25995a5f93af3312c924b0a790a4a0a808202ae18b0e91a30a5d70d9a", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/agent/bootstrap.json": "aaaa4fafbe6b8a8077e28cffbce3abddae4c4daa3e8fc813010552542cacc1a2", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/agent/trajectory.json": "800ac36f3cdbc20447359edfa4ea4ba541af3c9f62e80a0e55eebfc484b28cb7", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/agent/nanopycodeagent.txt": "0c7136e00b31a2c865886f0c52ff520cfe26f30219dda737fc5a5bc69fe996c8", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/verifier/ctrf.json": "c930b2cf607dc3cc5bda8c1ce5194033cd5a816a307a4dd88a0f2b11a57aba98", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/verifier/test-stdout.txt": "19e5f73b15b6ab27a6ea54f7694b8f3459264394e0a6f7ca1bb9f33e7c0f288c", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/verifier/retry-summary.json": "b500b5df8201e1eead855664febaa76551c7f025e956022b53c1fe1f572aaa68", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019-input/manifest.json": "21a1796d6095960b3a93d260d90cb323dc2926816de38dc6ca81cd2e66e922ba", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019-input/config.json": "9230ec9c6fd8cf0bfcd2b2e640e0b0c103fbdc8d6d92a929591ee2a3970d7866", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-qemu-alpine-ssh-main-20261019.json": "ac728a1b36f9f6ea427fe44483042b0617527ee3cfb9ef097beb928f830dae45", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/verifier/attempts/1/test-stdout.txt": "19e5f73b15b6ab27a6ea54f7694b8f3459264394e0a6f7ca1bb9f33e7c0f288c", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/verifier/attempts/1/ctrf.json": "c930b2cf607dc3cc5bda8c1ce5194033cd5a816a307a4dd88a0f2b11a57aba98", + "jobs/main-source/jobs/dev34-1017-qemu-alpine-ssh-main-20261019/qemu-alpine-ssh__PaghLZ4/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 19, + "task": "terminal-bench/query-optimize", + "task_ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "trial": "query-optimize__jDSk8AC", + "job": "dev34-1017-query-optimize-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792424012.8493927, + "stop": 1792424143.616417 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T15:24:19.165945Z", + "finished_at": "2026-10-19T15:36:04.741227Z", + "seconds": { + "environment_setup": 2.119758, + "agent_setup": 33.956721, + "agent_execution": 490.925768, + "verifier": 157.285622 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 1800.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 157.07986383800744 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 19, + "tool_calls": 25, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 217909, + "completion_tokens": 13092, + "cached_tokens": 211072 + }, + "estimated_known_cost_usd": 0.019027932 + }, + "task_checksum": "593405b6a2f6970f0f4c2ca2aff455a3b32bce3eecf8b506cfeeb0fe93b14838", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/result.json": "c06ec60ecef9df54f50383747e42bab1948ff85c7447736b9f513debdbd46d92", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/config.json": "b5f00e049fb2cd2ede06c63376e2a2622292a00e3229394b2826a2f7b26178ee", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/agent/bootstrap.json": "90bb56ff2a427a339bca3904e95ff8d7b2cfdca6af6af63ab01995c7c52cf240", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/agent/trajectory.json": "e578f67ce30b200e46d8c9a1a8f02a15f06eab2d0e8ef226a14fa12ca6041d05", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/agent/nanopycodeagent.txt": "5b5ca322b1e385e1c5d57bf829c19af364a73e4cfaaa5c5be3e12445e5457b0c", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/verifier/ctrf.json": "66a3ad9fb728ed0eb57e234f5fc323451e05f74d57a5ab259463d07c1e25fb79", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/verifier/test-stdout.txt": "c0eafc11bd7f6b35811941cd61caf3c8b38a1d79a3cd8766621c57e4f5c76caa", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/verifier/retry-summary.json": "9867129af80cbd32029df49cca76a14b94fbb6d0b12caa5d110f1ae37220313d", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019-input/manifest.json": "a23d2606ef728d6663779c1590f84e2bbd2ce3a98b310960d72e8474e26a8ffe", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019-input/config.json": "1f89710637af7cf3a5fe6c59d78fa68cab1400e790baca318211cf90a1436420", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-query-optimize-main-20261019.json": "6707307d9c0bd6c32a577dbb1196ba1db00587887fcc9cb73e639db78c9aa460", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/verifier/attempts/1/test-stdout.txt": "c0eafc11bd7f6b35811941cd61caf3c8b38a1d79a3cd8766621c57e4f5c76caa", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/verifier/attempts/1/ctrf.json": "66a3ad9fb728ed0eb57e234f5fc323451e05f74d57a5ab259463d07c1e25fb79", + "jobs/main-source/jobs/dev34-1017-query-optimize-main-20261019/query-optimize__jDSk8AC/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 0.0, + "audit_flags": [] + }, + { + "order": 20, + "task": "terminal-bench/regex-log", + "task_ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a", + "trial": "regex-log__4HnbzHi", + "job": "dev34-1017-regex-log-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 1, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792423599.9753232, + "stop": 1792423599.9883058 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T15:24:17.448808Z", + "finished_at": "2026-10-19T15:27:01.121228Z", + "seconds": { + "environment_setup": 1.994722, + "agent_setup": 37.819978, + "agent_execution": 82.451785, + "verifier": 20.062472 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 19.857408579002367 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 7, + "tool_calls": 6, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 73686, + "completion_tokens": 16157, + "cached_tokens": 71168 + }, + "estimated_known_cost_usd": 0.020570808 + }, + "task_checksum": "31dc6115c061b96539a5287090ce41a7a89d3201c291b9b843bd70e416f35c39", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/result.json": "ed6117ca719f5be69f378b2a17bdde2929f17bdb417ac8a13ca1c1d8e745dfe7", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/config.json": "4ae45df31b88f6fa33da37bade999c8bd5c5cf58b4886ead7b972fbe9124cc67", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/agent/bootstrap.json": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/agent/trajectory.json": "4dfc6581415e2b39973f085839c23f68c50092eda89cb5ab6ae24d01cbc45a64", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/agent/nanopycodeagent.txt": "032f52e15f9c5744d220c8f83154400aab11b48f7fc7c290fdf3a183f3db0f55", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/verifier/ctrf.json": "cbec85b41818876cb4ba261ad6b105a68184cd8efb7383c997853c94710d8b69", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/verifier/test-stdout.txt": "06ae21b3f117b0befc8d19e3baac6876aa9864cf7de5dd4f0183e75e599e09b8", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/verifier/retry-summary.json": "2132a0139b448ca3744aec01aeef81bd125f165c492f0a0a751736a23a2c3c32", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019-input/manifest.json": "5950af8694b8964c8ac9da81b44df9285993f709f6c84fb61e3401b75dd24e59", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019-input/config.json": "860df4e9a631d6b4411dc148dd9f05fae19db30914b8652aac398a87b38701f1", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-regex-log-main-20261019.json": "74e08e996091a65a95c3c0371f5e23f331caf3ed31c90fbfe9e1d98d33cf868e", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/verifier/attempts/1/test-stdout.txt": "06ae21b3f117b0befc8d19e3baac6876aa9864cf7de5dd4f0183e75e599e09b8", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/verifier/attempts/1/ctrf.json": "cbec85b41818876cb4ba261ad6b105a68184cd8efb7383c997853c94710d8b69", + "jobs/main-source/jobs/dev34-1017-regex-log-main-20261019/regex-log__4HnbzHi/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 21, + "task": "terminal-bench/torch-pipeline-parallelism", + "task_ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb", + "trial": "torch-pipeline-parallelism__PkyVbMZ", + "job": "dev34-1017-torch-pipeline-parallelism-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": 0.0, + "classification": "scored_failure", + "exception_type": null, + "official_pytest_counts": { + "passed": 2, + "failed": 2, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 2, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792426065.542477, + "stop": 1792426155.440486 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T15:36:17.339040Z", + "finished_at": "2026-10-19T16:09:39.357570Z", + "seconds": { + "environment_setup": 2.23697, + "agent_setup": 65.944927, + "agent_execution": 584.332499, + "verifier": 1326.533347 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0022778100101 + }, + { + "attempt": 2, + "status": "completed", + "return_code": 0, + "duration_seconds": 427.4184885600116 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 38, + "tool_calls": 41, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 1601358, + "completion_tokens": 48844, + "cached_tokens": 1588224 + }, + "estimated_known_cost_usd": 0.072082344 + }, + "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/result.json": "3b9f6f03b49d5e3d7d74fa541f1b9a7f43610aef980cdec70ea03df6a773e16f", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/config.json": "6798312a9dc0363b5ed562ccee4b145ee94e5f822af27f42d7ce09ab323e02ac", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/agent/bootstrap.json": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/agent/trajectory.json": "50bee32baba87b5cfafe7df7f0513dab36fdc06c7b8e166519d287cc5b6f3e39", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/agent/nanopycodeagent.txt": "aa9a6c1aa99ea75db702ddcd9b19ba7bfb8305f5db9ec6e581d5d8b8e94fbb90", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/ctrf.json": "0223c6750eae76580199bc51605cc18dd99d7fb895f119d3038bd2602c6e797f", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/test-stdout.txt": "b6bef2c091eb20534bbdf079af1778b240605364991331ad13bcb2355876648d", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/retry-summary.json": "8a0e94d792256506c2546d72ef435028fcb7aba43d5da2e4bfab90b103a1af33", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019-input/manifest.json": "0563ea62b3748bef26e28f281f02dbee281c9898d3ac4f79f6fe00ff2c9becef", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019-input/config.json": "33d367f52e778a276396e473e63120465fb020010cd91d850e8962052e45f5c1", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-torch-pipeline-parallelism-main-20261019.json": "2b7c16b03b3722b5eec4fcba642574f2ca3626ab6a6229955cc7d6cb07734b23", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/attempts/1/test-stdout.txt": "2bafb03cdbace4672b2fc0f0b609f93023a9c327ab7fff3e1d20110372233e3d", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/attempts/2/test-stdout.txt": "b6bef2c091eb20534bbdf079af1778b240605364991331ad13bcb2355876648d", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/attempts/2/ctrf.json": "0223c6750eae76580199bc51605cc18dd99d7fb895f119d3038bd2602c6e797f", + "jobs/main-source/jobs/dev34-1017-torch-pipeline-parallelism-main-20261019/torch-pipeline-parallelism__PkyVbMZ/verifier/attempts/2/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 0.0, + "analysis_note": "Both world-size cases failed with model.rotary_emb.fwd microbatch-count mismatches. Existence and no-hooks checks passed. The first verifier attempt timed out during dependency preparation; the second executed all four cases.", + "audit_flags": [] + }, + { + "order": 22, + "task": "terminal-bench/torch-tensor-parallelism", + "task_ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684", + "trial": "torch-tensor-parallelism__Z8AWvuk", + "job": "dev34-1017-torch-tensor-parallelism-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 13, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792424830.5007792, + "stop": 1792424865.0085137 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T15:36:18.023375Z", + "finished_at": "2026-10-19T15:48:07.674421Z", + "seconds": { + "environment_setup": 2.329545, + "agent_setup": 50.590553, + "agent_execution": 196.01488, + "verifier": 438.322726 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 438.2212119779724 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 17, + "tool_calls": 21, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 236446, + "completion_tokens": 13794, + "cached_tokens": 226560 + }, + "estimated_known_cost_usd": 0.02087796 + }, + "task_checksum": "d0160c1fe523063419b445f7b2974b64ff72a842ad5dc2b970d256e7629f0426", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/result.json": "54a8aaa873d131744fc975eaa877e4c852cc84ed6c7757df5c74636a6158359d", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/config.json": "bfc8006db0c64a55ff15e61141614acdcd40f92d746f24fa2e6e59616bc56b71", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/agent/bootstrap.json": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/agent/trajectory.json": "a0412925809e643403e72cbb258b7b13a830acfd10eddb6c70a09cfb76f0a13a", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/agent/nanopycodeagent.txt": "75527e1489f06b85755c89510e773c987af0bf2d69c27bae555a8a4cd3b8e7fa", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/verifier/ctrf.json": "de9700de6eb5495aa626edcd501cd1acbd988d1a67657012dd39f2fee3893682", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/verifier/test-stdout.txt": "b856a74cae8764f1b5cfd7530d9bfda537034f55106399b44a326726802894e2", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/verifier/retry-summary.json": "040ab5919a8c29abb81a3e3eb99ff4ac1ecba8c208efd4d9e9ebb23de74ecd8f", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019-input/manifest.json": "21a4fa76ab6b35d7a629006fb26bad14fb984e735fd52665103cfab28dfc6fc9", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019-input/config.json": "c5ee7c11188b02f121e0c71e135ccea6968daa21247473e164ffd912dbfe13f2", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-torch-tensor-parallelism-main-20261019.json": "045a4325a02aabda7959a2a1b36ec9e846eea35bb470b558e49b59167bee4023", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/verifier/attempts/1/test-stdout.txt": "b856a74cae8764f1b5cfd7530d9bfda537034f55106399b44a326726802894e2", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/verifier/attempts/1/ctrf.json": "de9700de6eb5495aa626edcd501cd1acbd988d1a67657012dd39f2fee3893682", + "jobs/main-source/jobs/dev34-1017-torch-tensor-parallelism-main-20261019/torch-tensor-parallelism__Z8AWvuk/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [ + "searched_recent_test_files; observations contained installed dependency tests, not hidden task tests" + ] + }, + { + "order": 23, + "task": "terminal-bench/write-compressor", + "task_ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e", + "trial": "write-compressor__6wdBiJG", + "job": "dev34-1017-write-compressor-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792426514.7465954, + "stop": 1792426514.8600838 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:09:49.308187Z", + "finished_at": "2026-10-19T16:15:35.918135Z", + "seconds": { + "environment_setup": 2.086382, + "agent_setup": 33.222548, + "agent_execution": 274.181461, + "verifier": 15.860256 + }, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 15.659770547004882 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 17, + "tool_calls": 18, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 823778, + "completion_tokens": 57372, + "cached_tokens": 809088 + }, + "estimated_known_cost_usd": 0.078107928 + }, + "task_checksum": "17ff45a3beaa04e5f08eb4f3c6136f4b56602bdcca52ade0896cfd341f374e99", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/result.json": "177ae2f816046d50111e1ee24754a03c0404a9236749ddfb2dc4b2ab0bc48026", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/config.json": "8c0fe884709fe914407637ae377181650dfb857718b48c606bd6748c0a964570", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/agent/bootstrap.json": "031768ca83f80450befe1475e9f6bfc93052b2053d23aae46a188aa390a36aba", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/agent/trajectory.json": "74d612c01a414b557b76d3460b44aa77fc9d615915f4806499a47f4432319aae", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/agent/nanopycodeagent.txt": "223aa1f1a5b14d855d78034edae1709af7137305234c51a5eed226bcf086ea02", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/verifier/ctrf.json": "f3afde1c0300f95143d4b14205817cee72ca98f25cc8767d60247633d908509e", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/verifier/test-stdout.txt": "4d856b4ec1bffc5c0e783b4fdbeb79cec5969197e11ac54f395c72741915c04e", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/verifier/retry-summary.json": "7cb070925c9a8879bc0f02a17322087d06d6331475b2a720a715a3c2374cc846", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019-input/manifest.json": "736637261485635989345ab341e5a3f6953533c2663e5e468d10285b31f8fc11", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019-input/config.json": "ea0171cc8884142a34aed19014562c82adc6eb9bc861993844cf8dd37bf78fbe", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-write-compressor-main-20261019.json": "14bd7f44dc69ecee099aa1c342a6a4553764ef5bb572c4b075005b0a379bf22b", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/verifier/attempts/1/test-stdout.txt": "4d856b4ec1bffc5c0e783b4fdbeb79cec5969197e11ac54f395c72741915c04e", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/verifier/attempts/1/ctrf.json": "f3afde1c0300f95143d4b14205817cee72ca98f25cc8767d60247633d908509e", + "jobs/main-source/jobs/dev34-1017-write-compressor-main-20261019/write-compressor__6wdBiJG/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 24, + "task": "terminal-bench/constraints-scheduling", + "task_ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "trial": "constraints-scheduling__yGXyem4", + "job": "dev34-1017-constraints-scheduling-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792426261.0829928, + "stop": 1792426261.4257693 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:09:48.460130Z", + "finished_at": "2026-10-19T16:11:22.521017Z", + "seconds": { + "environment_setup": 2.199526, + "agent_setup": 33.492246, + "agent_execution": 20.972981, + "verifier": 16.09658 + }, + "work_budget_seconds": 1020, + "verifier_timeout_seconds": 1200.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 15.808915652014548 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 5, + "tool_calls": 4, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 27603, + "completion_tokens": 3722, + "cached_tokens": 23808 + }, + "estimated_known_cost_usd": 0.005747748 + }, + "task_checksum": "5b311f974587adf67804532222c59395f41a6c984b54a5e96a56dd071ee801dc", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/result.json": "4cbb3f673dc726660ca5712dbe1023978be521eb4360a1971382411cb4137cc1", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/config.json": "0a9597b856996442e566b9cb0652fd4e4bcb18baee74178d3e6d0f38f252cbff", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/agent/bootstrap.json": "9705c14c20a581a03dadb6e7334925f68eb71992573c8b228c8c79ef0655e9e9", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/agent/trajectory.json": "c3e23f02b837ad1ba61f4d77785a895ac68d0a1a53e6cfa4e67870b917deb06c", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/agent/nanopycodeagent.txt": "b549b6bc95b6265f0e69daa0de1b84fa59f5da06d276c3e0ae2998a2b01f737d", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/verifier/ctrf.json": "02b2a226097200d9a411b26c0825729ed6cc8255e9fdac8b3f984ef75ab8d977", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/verifier/test-stdout.txt": "394a376be71130becba13d965a613f23e20c57952b0a20a56b015c03e8fb94fe", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/verifier/retry-summary.json": "61999d2cb613c6eb3394b7572075fcf6b7aa7336175021234c59e602a6416e9d", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019-input/manifest.json": "1a4caf259c4c645921077568ec984fea89335d91864240111a86fe89308f4720", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019-input/config.json": "e101b6c89bab95dab13f66fbd64bfdcd42451e9b32d219d5c97157a21539ae9d", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-constraints-scheduling-main-20261019.json": "9856526e760107eb0f3970c12e09bbc01658d2522664467c81610be35a1bbb5d", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/verifier/attempts/1/test-stdout.txt": "394a376be71130becba13d965a613f23e20c57952b0a20a56b015c03e8fb94fe", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/verifier/attempts/1/ctrf.json": "02b2a226097200d9a411b26c0825729ed6cc8255e9fdac8b3f984ef75ab8d977", + "jobs/main-source/jobs/dev34-1017-constraints-scheduling-main-20261019/constraints-scheduling__yGXyem4/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 25, + "task": "terminal-bench/custom-memory-heap-crash", + "task_ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "trial": "custom-memory-heap-crash__vJAhK9p", + "job": "dev34-1017-custom-memory-heap-crash-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792426775.5388458, + "stop": 1792426779.0979018 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:15:41.889982Z", + "finished_at": "2026-10-19T16:20:00.144589Z", + "seconds": { + "environment_setup": 2.289592, + "agent_setup": 29.666298, + "agent_execution": 185.507732, + "verifier": 19.553382 + }, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 19.31472117899102 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 29, + "tool_calls": 43, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 803470, + "completion_tokens": 27915, + "cached_tokens": 782208 + }, + "estimated_known_cost_usd": 0.044569848 + }, + "task_checksum": "a0db695f342c1b72f1b3d0b6c234eb802e970c4758931355e4ceeea4c9e73c37", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/result.json": "7680af5e058c8c44edc8694bc168376c51ce2e5850f2659bca6045a9b6c8d129", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/config.json": "b4f4b6765caf498cbbbd0b044eb4ea7b0fc11ce9fac80cefc87412b2cda5ff60", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/agent/bootstrap.json": "b1cca4c2159275ee7676539abaf5854f9c053ddaa2a5477b3762d3e244f2a04f", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/agent/trajectory.json": "2254e0d0ef8b1f850900596f46b5d8e94f64b7425448b466b27412e8675b2aa9", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/agent/nanopycodeagent.txt": "32fe8022d4805f70f1f4b7348be26633e7368e8a55fbb9f29f1c752dd78c99a5", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/verifier/ctrf.json": "56474a468f8fa673a9b4f1cf90a5b7efa38bdea86fb1ca04c8031096db0da394", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/verifier/test-stdout.txt": "9cdf965be5fdd96e62b02571e4e9f41ede20002ca93b292a27c4e04f35092452", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/verifier/retry-summary.json": "e7b2939933147be7063c4e6a75927525bfc19952c7e774ff9f9790bf5d8db6d7", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019-input/manifest.json": "0ab141aa5fddfa8cfc18c072cb809a672b3c699bf49e310972af4893b7ae558c", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019-input/config.json": "4bd08f65565a707b7c29b6a0f94c94da0ed435a880255d455f7212f4ab0954c7", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-custom-memory-heap-crash-main-20261019.json": "ffab43dfae1d34f8c27fcf281799daecde22fad47c4cbfaa0ae1352a97bddb5d", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/verifier/attempts/1/test-stdout.txt": "9cdf965be5fdd96e62b02571e4e9f41ede20002ca93b292a27c4e04f35092452", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/verifier/attempts/1/ctrf.json": "56474a468f8fa673a9b4f1cf90a5b7efa38bdea86fb1ca04c8031096db0da394", + "jobs/main-source/jobs/dev34-1017-custom-memory-heap-crash-main-20261019/custom-memory-heap-crash__vJAhK9p/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 26, + "task": "terminal-bench/dna-assembly", + "task_ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "trial": "dna-assembly__voS27vm", + "job": "dev34-1017-dna-assembly-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": 0.0, + "classification": "scored_failure", + "exception_type": null, + "official_pytest_counts": { + "passed": 0, + "failed": 1, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792426943.2275877, + "stop": 1792426943.2525215 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:15:42.094004Z", + "finished_at": "2026-10-19T16:22:44.422912Z", + "seconds": { + "environment_setup": 2.204863, + "agent_setup": 32.562455, + "agent_execution": 346.549849, + "verifier": 19.664709 + }, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 19.465466562018264 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 40, + "tool_calls": 45, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 1786738, + "completion_tokens": 61573, + "cached_tokens": 1760000 + }, + "estimated_known_cost_usd": 0.092469 + }, + "task_checksum": "e2acbc34aa5f20bbe3b10d2d297cdec9e6f2086c94a70068f2325e7bb7ab1212", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/result.json": "89927638ed418482a0da0491762b8f56898e12c87c1df28a3d8c449e998a9ee4", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/config.json": "26ac5f2fd14f28844e2ae5defc09d7f8bc439fd008f139677729702bb61da4a1", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/agent/bootstrap.json": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/agent/trajectory.json": "fd7ac084992f5e7f60b38747dd30656a88aca215e76d71a00a1bb09fbf8a3edf", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/agent/nanopycodeagent.txt": "8eb9b7aaf204d4ccc5649a910fbc896961d452d9e85908ab5ea008446ee739f6", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/verifier/ctrf.json": "daafb77b81e8212a2cd1e995c80a4eed87f119c934f72716d4554d1ae1dbf040", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/verifier/test-stdout.txt": "ba686eedad726c33bc8bc77014583283d7fe97279687ea207ed8f46462d03d07", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/verifier/retry-summary.json": "7881dfac62d836c4e4a4af8fdddb10899f17234bf4b80d9b13b8d1be9700732d", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019-input/manifest.json": "14da8f385749cec9c2fcea408deda9044e36afd20dff2a4c590d20c7a46f5bb4", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019-input/config.json": "23a8c9c2c02fe1d1c71e4f2272ff6bad79f2524855d58f6393e4bd0d6994a100", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-dna-assembly-main-20261019.json": "6fae1f9e0b87bfe43b044035d8cbce0bb68ca6f8cca4f6af8f0c5d1475ff6301", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/verifier/attempts/1/test-stdout.txt": "ba686eedad726c33bc8bc77014583283d7fe97279687ea207ed8f46462d03d07", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/verifier/attempts/1/ctrf.json": "daafb77b81e8212a2cd1e995c80a4eed87f119c934f72716d4554d1ae1dbf040", + "jobs/main-source/jobs/dev34-1017-dna-assembly-main-20261019/dna-assembly__voS27vm/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 1.0, + "analysis_note": "Submitted primers lacked the required nucleotide clamp before the BsaI recognition site. The official primer test failed.", + "audit_flags": [] + }, + { + "order": 27, + "task": "terminal-bench/llm-inference-batching-scheduler", + "task_ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67", + "trial": "llm-inference-batching-scheduler__h3Hk76H", + "job": "dev34-1017-llm-inference-batching-scheduler-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792427550.6042082, + "stop": 1792427550.6611388 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:22:52.131051Z", + "finished_at": "2026-10-19T16:32:51.666920Z", + "seconds": { + "environment_setup": 2.181852, + "agent_setup": 32.319456, + "agent_execution": 529.60559, + "verifier": 14.199232 + }, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 13.999289879982825 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 35, + "tool_calls": 36, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 1883594, + "completion_tokens": 70575, + "cached_tokens": 1854976 + }, + "estimated_known_cost_usd": 0.104405256 + }, + "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/result.json": "912d05172977e4e6a04e9afcfb79888c05cc630e903d56ea2b4e0a318209fdd9", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/config.json": "35fdabea46b854a65a1732fae7a102ff92951a8b089c511a3e70dacdd4811d28", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/agent/trajectory.json": "6ddafe7835da31246916c9f91cabc0c638ee96c987996a0ba6e0135772b2f03b", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/agent/nanopycodeagent.txt": "763425dd2346f6baca37079413ecd1f704ced26ec28dc90973dfea16b8ddbc5b", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/verifier/ctrf.json": "28a4d6728a7da4c7a6a3a6b3d03ba8cf8db482e28b841eeee5192125f64df633", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/verifier/test-stdout.txt": "78d9249accb27cf0a3c39ee60dd30529d954fc8a180b79b8617bd93a2342fe1d", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/verifier/retry-summary.json": "5e173bc99860b42d62bd70d9339d877a49a74adcacf64713a1aef418e7b84b32", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019-input/manifest.json": "1f7a6b959f9fb42bd665c21c980e63bd56361c9bff94f0576dea3eb3c2b45acc", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019-input/config.json": "a701cbc1289238c756ef031d989376de4f6b9f17145b813fb5c512612ad0da88", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-llm-inference-batching-scheduler-main-20261019.json": "2237de012cb19d4556f328f70dd8391406fd00a7a39f47e52f0f17082d58f67a", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/verifier/attempts/1/test-stdout.txt": "78d9249accb27cf0a3c39ee60dd30529d954fc8a180b79b8617bd93a2342fe1d", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/verifier/attempts/1/ctrf.json": "28a4d6728a7da4c7a6a3a6b3d03ba8cf8db482e28b841eeee5192125f64df633", + "jobs/main-source/jobs/dev34-1017-llm-inference-batching-scheduler-main-20261019/llm-inference-batching-scheduler__h3Hk76H/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 28, + "task": "terminal-bench/path-tracing", + "task_ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c", + "trial": "path-tracing__SqNiG3f", + "job": "dev34-1017-path-tracing-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 5, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 5, + "passed": 5, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792427937.865896, + "stop": 1792427944.8194184 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:22:51.990715Z", + "finished_at": "2026-10-19T16:39:25.994047Z", + "seconds": { + "environment_setup": 2.282904, + "agent_setup": 31.147842, + "agent_execution": 908.654927, + "verifier": 30.56129 + }, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 30.318850896990625 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 85, + "tool_calls": 84, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 10401556, + "completion_tokens": 155756, + "cached_tokens": 10325376 + }, + "estimated_known_cost_usd": 0.271713456 + }, + "task_checksum": "549aa348dde50ec3824205f402d9686da5d1dedf65ae2bd2ae8806513839fe93", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/result.json": "d6e88b4a28d8bed7a7a87f46457a08f60e86691c1c214f5b1c87803e05999f7e", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/config.json": "08ec2c41f3d835062e541223659d2cc1ff501f12bfe8794716558ecd190406a4", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/agent/bootstrap.json": "2cdec543c50c5600582cfc3b2ebbe2f2c9d16988e065a02cf6e8e823f3a05542", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/agent/trajectory.json": "0ca215cfeed0f8d7eafc6efeb8dd41496d2c0f618ff7e2844c55c5e01be78251", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/agent/nanopycodeagent.txt": "42937f72d3bfb0d16f1663b2d3fcf116b94e55aefb588d263dca8cd38b682d6f", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/verifier/ctrf.json": "88fd3bb6268545a1cb100ba0a0b58c52c80041643a4e6bf93927860e87e18536", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/verifier/test-stdout.txt": "ceef333c9ca50b05ef2c9ce2154e255e7e566ab2afaae5c6d8645f2e3fd0be4f", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/verifier/retry-summary.json": "7291f184b4cb475c4e416768e6bee7572900418a56edf11af163697ef9d25e58", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019-input/manifest.json": "14be3027b8c1e48f45a3626b09885de01d6f6d654471106a8f770833fd35df3f", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019-input/config.json": "c53b1c0cb49ec8ac56215502f2abb4938fe9f9c2f94b74a7c9b740cb37d17e64", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-path-tracing-main-20261019.json": "87355a3fa31c01142daeeb12d5f1c2ac96ef32875ba420ae44f655001a9eebe3", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/verifier/attempts/1/test-stdout.txt": "ceef333c9ca50b05ef2c9ce2154e255e7e566ab2afaae5c6d8645f2e3fd0be4f", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/verifier/attempts/1/ctrf.json": "88fd3bb6268545a1cb100ba0a0b58c52c80041643a4e6bf93927860e87e18536", + "jobs/main-source/jobs/dev34-1017-path-tracing-main-20261019/path-tracing__SqNiG3f/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 29, + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "trial": "schemelike-metacircular-eval__9dPioaX", + "job": "dev34-1017-schemelike-metacircular-eval-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 1, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792429123.5684304, + "stop": 1792429144.1516893 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:39:33.452464Z", + "finished_at": "2026-10-19T16:59:25.179955Z", + "seconds": { + "environment_setup": 2.231402, + "agent_setup": 33.708862, + "agent_execution": 1098.470333, + "verifier": 36.068665 + }, + "work_budget_seconds": 2220, + "verifier_timeout_seconds": 2400.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 35.84452914100257 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 35, + "tool_calls": 42, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 1952299, + "completion_tokens": 53426, + "cached_tokens": 1912320 + }, + "estimated_known_cost_usd": 0.08757882 + }, + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/result.json": "f3b40e766059953fd9073821bd0f6b04ef247e2ba98741e259d4488521b30e23", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/config.json": "225f04c117633696c1aa6140b4eddc1e08841a4cd39155659f54622b7526598c", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/agent/trajectory.json": "3e55a239094080bb9a5cfb6848059a7381f3ffcc84fa82f9df4be9ab56f5d646", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/agent/nanopycodeagent.txt": "6d180649a73896eb04d940464e9368bd8ab667a7733964a28c58d08ee2028ce4", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/verifier/ctrf.json": "9b9f6c10edf8db981017706669b32828aec35f82aab7e6b3af89fdda966ee8ac", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/verifier/test-stdout.txt": "708a9ce46956e60839e0cf66441b48c09fd257850ff76ea2105838d4da06e061", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/verifier/retry-summary.json": "95e84d0146c7de14f95d2768f3293f95519ec8ee3fbb9e9508c3fdf309702e76", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019-input/manifest.json": "250b96a75f95fea4b5786cc235f429eeeba9f1840974b246b60320a2cb8ad356", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019-input/config.json": "704d9413c835fdf67886ad1b38fa2fed44255846f534b732a3f4d21f4977a637", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-schemelike-metacircular-eval-main-20261019.json": "a65854ec5768dd3d3ec708a67aa61c44332ea43cf3d127863631c590348e247c", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/verifier/attempts/1/test-stdout.txt": "708a9ce46956e60839e0cf66441b48c09fd257850ff76ea2105838d4da06e061", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/verifier/attempts/1/ctrf.json": "9b9f6c10edf8db981017706669b32828aec35f82aab7e6b3af89fdda966ee8ac", + "jobs/main-source/jobs/dev34-1017-schemelike-metacircular-eval-main-20261019/schemelike-metacircular-eval__9dPioaX/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [ + "host_interpreter_depth_counter_workaround_via_multiplication_primitive_python_eval; native pass retained" + ] + }, + { + "order": 30, + "task": "terminal-bench/caffe-cifar-10", + "task_ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf", + "trial": "caffe-cifar-10__MSA6sHv", + "job": "dev34-1017-caffe-cifar-10-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 6, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792430234.8993564, + "stop": 1792430272.8941817 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T16:39:33.763239Z", + "finished_at": "2026-10-19T17:18:14.878637Z", + "seconds": { + "environment_setup": 2.125386, + "agent_setup": 32.515352, + "agent_execution": 2212.424996, + "verifier": 51.835188 + }, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 1200.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 51.642851095995866 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 76, + "tool_calls": 75, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 2197625, + "completion_tokens": 34384, + "cached_tokens": 2166016 + }, + "estimated_known_cost_usd": 0.063739596 + }, + "task_checksum": "38da84d32a37ce45b201f202f0deb320dd80b5482afbcf66ba8456c8ea814c69", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/result.json": "ee90c77ca3ffe06ad97a1e2689c253e4020366f693729b8db76dec6e47aa4b4c", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/config.json": "995d67d3a830bd5804350005b1ce1a10f9ae0af6886d2ea9c133b74b85c7124c", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/agent/bootstrap.json": "208b54a2f6972c29fb49fbb3a4e826fa28c729d1cf556f7a3835b1cf3ba1496d", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/agent/trajectory.json": "891ece928a76e467618bf04332cb3701684c9bf951f2e332df33da4904289282", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/agent/nanopycodeagent.txt": "bd72abfe3bfed1981263ef205be5b9abe899a3058512268b02268b7adf6d4bf2", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/verifier/ctrf.json": "d46bc3785bd2b2ae4347f21e383e4fd2e6f82648b0922a53d8e6c8bd7c027998", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/verifier/test-stdout.txt": "66b9a085b4c508e58a11d5b3436521e461eb2da321c7db2d3f296ea4b7a58571", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/verifier/retry-summary.json": "10039f12ff028cd0a42ba6fa1fddade56ebdaf7740a317c8ff691f1c10512e02", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019-input/manifest.json": "84faddcf27ca3fe184ab81350299a8b37f9f12a0386316ddeb7db34f0f389ffc", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019-input/config.json": "c49e0ea45e4cb9056364eb9ab9cf2f176814b101a00d4dd94b0ca8f097ad7e16", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-caffe-cifar-10-main-20261019.json": "0892465dd9f26ae8bf16d5d5ccf22fd3c1cd2d44ce21fe84724c5bcf84d93d80", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/verifier/attempts/1/test-stdout.txt": "66b9a085b4c508e58a11d5b3436521e461eb2da321c7db2d3f296ea4b7a58571", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/verifier/attempts/1/ctrf.json": "d46bc3785bd2b2ae4347f21e383e4fd2e6f82648b0922a53d8e6c8bd7c027998", + "jobs/main-source/jobs/dev34-1017-caffe-cifar-10-main-20261019/caffe-cifar-10__MSA6sHv/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 31, + "task": "terminal-bench/circuit-fibsqrt", + "task_ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3", + "trial": "circuit-fibsqrt__f5qXCdZ", + "job": "dev34-1017-circuit-fibsqrt-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792431451.7212543, + "stop": 1792431452.4641151 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T17:18:22.436190Z", + "finished_at": "2026-10-19T17:37:53.497811Z", + "seconds": { + "environment_setup": 2.248553, + "agent_setup": 31.787996, + "agent_execution": 1101.139758, + "verifier": 14.652172 + }, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 14.451029582007322 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 31, + "tool_calls": 34, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 1602242, + "completion_tokens": 60518, + "cached_tokens": 1574016 + }, + "estimated_known_cost_usd": 0.090533496 + }, + "task_checksum": "dc4c18df98c940d9ec11336861e07f7f5ca2a13b9e8e0f95e886c20f3d61bc0d", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/result.json": "aa41bdc9827b2529aa3b0c589986c0e9087f325068c33568ebcf73e25d7d3054", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/config.json": "a64efe2f2ee4c14b1150239e1d61080e5e61fc5971b5b124a77a267def9cee28", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/agent/bootstrap.json": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/agent/trajectory.json": "9905fd3cfe516b2404ce3fa8a396bf0de1633b84a4fd054757b1f37e5db72a21", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/agent/nanopycodeagent.txt": "ff9118b4980a80addf199601d6fbd4d16f8779e18569c1507cdbbe3471a03cf7", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/verifier/ctrf.json": "6f652e8586322f8f028cb715840b73665c89f392d8f23455bbc080e30675405e", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/verifier/test-stdout.txt": "bcfa7301afc8eb8ef011914a5fa820422f572f7db323b201c9ebbfe4f39aa2e6", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/verifier/retry-summary.json": "7e6c9515263011c3211d13db47461780d1fe815cefb222ea87788a6ea37434bd", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019-input/manifest.json": "59a12683db71bf34b4a1cb7cf9d502c7370f015a1b63f817eba7e2c4974dc70a", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019-input/config.json": "70c689d5048b43c075a5c3a8ea02ce38bf82d69ff8deaaf87d036fbd41638909", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-circuit-fibsqrt-main-20261019.json": "a41bc43fee2a78760b298e399c5063195520b34a8784ba3fd96f8c99c17f2266", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/verifier/attempts/1/test-stdout.txt": "bcfa7301afc8eb8ef011914a5fa820422f572f7db323b201c9ebbfe4f39aa2e6", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/verifier/attempts/1/ctrf.json": "6f652e8586322f8f028cb715840b73665c89f392d8f23455bbc080e30675405e", + "jobs/main-source/jobs/dev34-1017-circuit-fibsqrt-main-20261019/circuit-fibsqrt__f5qXCdZ/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 32, + "task": "terminal-bench/mteb-leaderboard", + "task_ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c", + "trial": "mteb-leaderboard__NKwLRpd", + "job": "dev34-1017-mteb-leaderboard-main-20261019", + "raw_reward": 1.0, + "execution_phase": "primary", + "analytical_score": 1.0, + "classification": "passed", + "exception_type": null, + "official_pytest_counts": { + "passed": 2, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 2, + "passed": 2, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792431207.3439736, + "stop": 1792431207.3627293 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T17:18:22.335373Z", + "finished_at": "2026-10-19T17:33:51.314431Z", + "seconds": { + "environment_setup": 2.399506, + "agent_setup": 25.445527, + "agent_execution": 863.742688, + "verifier": 13.171726 + }, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 12.898303352994844 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 60, + "tool_calls": 67, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 2271317, + "completion_tokens": 34983, + "cached_tokens": 2228352 + }, + "estimated_known_cost_usd": 0.068239212 + }, + "task_checksum": "f4c2fa1f189121ba4a0f654a270b92bfa2dad319c3315d89184f71c5c346a497", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/result.json": "6d1a3b11017779708a7dd5fdb0ccd400cbfa85f4a20e757c162c3aa2988516b7", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/config.json": "d24c8177d64b8144dddfef627e432fc29fe1ee43eb45beb474f12eadf4dfedb9", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/agent/bootstrap.json": "a3d2b8905b957adcb3553196f9680346955b8f276e2915b0e3539aab45b5407e", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/agent/trajectory.json": "1017e2acc583740670ec8fccc1f13684e2416366f593187707408f95af8265da", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/agent/nanopycodeagent.txt": "9b8bc93195a09b365589183a716c88f45f64338356b44d9d51d86fc9ef0d011c", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/verifier/ctrf.json": "50f436f5a85300fe8cbaff9aafde40a9aa8abfbce702372f263cd80da4f6a81b", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/verifier/test-stdout.txt": "e3d5dc5efabc077e278a52e614ddf91e0a199ae5d79a73f1f440a37bdeda98a1", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/verifier/retry-summary.json": "40e65891b7071e623a952a66befd1800bfe3176205ed061d8f84b8cdbba3e2b3", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019-input/manifest.json": "5cea90fa0ecfc3bf7dfcc549e2bdeb16bdd611725f8bf394428dbf4e1a2887cf", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019-input/config.json": "8f2e89b600719866c777d0ef46f63ae24ee49251b849b27ea5137149424a5545", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-mteb-leaderboard-main-20261019.json": "7faf1011207025a01e341d0e5ffb6a6a115280aad5bca36f0d637df214fb2117", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/verifier/attempts/1/test-stdout.txt": "e3d5dc5efabc077e278a52e614ddf91e0a199ae5d79a73f1f440a37bdeda98a1", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/verifier/attempts/1/ctrf.json": "50f436f5a85300fe8cbaff9aafde40a9aa8abfbce702372f263cd80da4f6a81b", + "jobs/main-source/jobs/dev34-1017-mteb-leaderboard-main-20261019/mteb-leaderboard__NKwLRpd/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "historical_reward": 1.0, + "audit_flags": [] + }, + { + "order": 33, + "task": "terminal-bench/regex-chess", + "task_ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78", + "trial": "regex-chess__7UgkTBc", + "job": "dev34-1017-regex-chess-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": 0.0, + "classification": "scored_failure", + "exception_type": null, + "official_pytest_counts": { + "passed": 2, + "failed": 2, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 4, + "passed": 2, + "failed": 2, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792433343.5913875, + "stop": 1792433355.1533968 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T17:38:00.872579Z", + "finished_at": "2026-10-19T18:09:36.246835Z", + "seconds": { + "environment_setup": 2.222967, + "agent_setup": 36.082544, + "agent_execution": 1810.4492, + "verifier": 25.322897 + }, + "work_budget_seconds": 3420, + "verifier_timeout_seconds": 3600.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 25.123640032019466 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 92, + "tool_calls": 92, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 11076098, + "completion_tokens": 151207, + "cached_tokens": 11026176 + }, + "estimated_known_cost_usd": 0.262582056 + }, + "task_checksum": "64993ab339d437cea9397a033e825663a77d5527a42ef3c2fb2ad25133dcbd4e", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/result.json": "466ea17a29f8141ab4c74eea69d3260a756d7f39627e44a99f7e1aa9b07c58d4", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/config.json": "abb77e0c03a2421a9d5ae18a4f92c6276184ca3c324c1e45345623592191c79e", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/agent/trajectory.json": "c5e5504a7ab5a0a4b8ccff713006b66a98afed709206952e8e7ba8ef6133d0ca", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/agent/nanopycodeagent.txt": "8824051cbe679310ec942d8606845d88997b1ef51bb47736d272588c9f833649", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/verifier/ctrf.json": "de880d0bd4e8f014235c498dc1ce4b72e3fb8aa02a01f66e500928caa560f4f7", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/verifier/test-stdout.txt": "26735de300199b894f49b916dbca4538f2f9ee109d9004b929d9ba365424c828", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/verifier/retry-summary.json": "0f073841045bbb0570689b278d408b04d51e11ced05d0c534b2f8a8e7d8eb226", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019-input/manifest.json": "2571e80412626a4be2981bf3da28f10dfa1443454d0c1b9600f36f7c7a9c3218", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019-input/config.json": "bea86261b8548f7bd45b02379f7bdb907249c763b8aeda9ba6702b4a7b8e3d96", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-regex-chess-main-20261019.json": "072135ae9a8b36b01c65e13508ecd187698c4ddc991f6b297f2325caeb1075b0", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/verifier/attempts/1/test-stdout.txt": "26735de300199b894f49b916dbca4538f2f9ee109d9004b929d9ba365424c828", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/verifier/attempts/1/ctrf.json": "de880d0bd4e8f014235c498dc1ce4b72e3fb8aa02a01f66e500928caa560f4f7", + "jobs/main-source/jobs/dev34-1017-regex-chess-main-20261019/regex-chess__7UgkTBc/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 0.0, + "analysis_note": "The submitted regex pipeline allowed an illegal king move onto a square attacked by a knight. Two official cases passed and two failed. A real re.json was submitted; this is a scored implementation failure, unlike the earlier truncation-only six-task rerun.", + "audit_flags": [] + }, + { + "order": 34, + "task": "terminal-bench/build-pov-ray", + "task_ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "trial": "build-pov-ray__VwRGNZk", + "job": "dev34-1017-build-pov-ray-main-20261019", + "raw_reward": 0.0, + "execution_phase": "primary", + "analytical_score": 0.0, + "classification": "scored_failure", + "exception_type": null, + "official_pytest_counts": { + "passed": 2, + "failed": 1, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 2, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792432193.3050778, + "stop": 1792432198.8822556 + }, + "official_counts_scope": "Completed final verification summary; archived attempts are retained separately.", + "started_at": "2026-10-19T17:38:01.150484Z", + "finished_at": "2026-10-19T17:50:20.241320Z", + "seconds": { + "environment_setup": 2.072151, + "agent_setup": 29.387598, + "agent_execution": 654.694413, + "verifier": 31.459416 + }, + "work_budget_seconds": 11820, + "verifier_timeout_seconds": 12000.0, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 31.24800779801444 + } + ], + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "bootstrap_status": "ready", + "model_started": true, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 72, + "tool_calls": 89, + "incomplete_model_steps": [], + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 2771833, + "completion_tokens": 32587, + "cached_tokens": 2723584 + }, + "estimated_known_cost_usd": 0.069920604 + }, + "task_checksum": "3426fb4714b5bb0463cc30bd872712a2400b23934b135c40ef7f88b27da4cd7a", + "source_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "source_sha256": { + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "d718f7c5528c156df549d68ab2deff247a289d08a453c6d083b72d0e9c7013dc", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "44bf247282437a0092bd7e649fd7fa9c776b6d34a0c14ffedcc10b37a47608ef", + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206" + }, + "wheel_sha256": "cd81f9ee91dd10287a08932ff0b62cdba45b9041e6c6cb7a3a76442bc9d7453f", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/result.json": "65b2ef7c21a8cd42c11f08cd3b09b01ad5290e72518f50d91e189da9fd06a0ea", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/config.json": "676547b7bcff1f95b590ab002e3837e62fc34b1d8c24a9e3cb6cb4deafc3eaf3", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/agent/bootstrap.json": "99082fd67697df1eb8a8b4174efe6fb9f2cd70f4d201f6b276a185c3df7f9477", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/agent/trajectory.json": "c4478cd5f28b321c62112a659e141e0b3284146bbaf8389afaa2bf0348f0bbe7", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/agent/nanopycodeagent.txt": "40b5da5a183d18d5944551f431fac07e7cf829721dcde383afaa38a0f88897ea", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/verifier/ctrf.json": "606f21037cfe4c7039fe9ef8234021c58eef807b5cb0a10d7913a91a717d91bc", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/verifier/test-stdout.txt": "fea09c64e27cc46d69684b610941328dd8676b79e96e22098f5246e7a7ecc9a1", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/verifier/retry-summary.json": "b3e77f53cec3cb2a55b1d41ad62c4e90f834e0c626cffeb35a5150ee6e812dbc", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019-input/manifest.json": "e04c31322526347298421837c10d4f713698f6d6158214a2ffd36914413a3435", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019-input/config.json": "6c3a7466a202402d6202adecb9328cdf0dc489c4f46ddcbffafc86b97f1e641f", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-build-pov-ray-main-20261019.json": "6d2240e63e519480597fdcdf78b5ba0c3f242a74b5794c5c86f4c7f15ada6518", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/verifier/attempts/1/test-stdout.txt": "fea09c64e27cc46d69684b610941328dd8676b79e96e22098f5246e7a7ecc9a1", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/verifier/attempts/1/ctrf.json": "606f21037cfe4c7039fe9ef8234021c58eef807b5cb0a10d7913a91a717d91bc", + "jobs/main-source/jobs/dev34-1017-build-pov-ray-main-20261019/build-pov-ray__VwRGNZk/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "historical_reward": 1.0, + "analysis_note": "The official source-distribution check failed because file_id.diz was absent. Executable and rendering checks passed; the recorded SSIM was 0.8731. The agent downloaded from the official 2.2 archive URLs. The marker failure alone does not establish that a different version was built.", + "audit_flags": [] + } + ], + "preflight_history": { + "collected_at": "2026-10-19T13:44:55.714589+00:00", + "model_started": false, + "attempts": 35, + "successful_installations": 34, + "setup_failures": 1, + "administrative_interruptions": 0, + "recovery_reason": "Fresh installation-only trial after the original uv installer download failed with an astral.sh TLS connection error. Same source, task, environment, budgets, and official sources. No model or verifier execution occurred.", + "trials": [ + { + "arm": "main", + "task": "terminal-bench/adaptive-rejection-sampler", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/adaptive-rejection-sampler__q2UYcaW/result.json", + "result_sha256": "47f301285fb59b6e09e01ced977a956b08927b5c362dec94ccd60c9de46461a9", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:15:12.636428Z", + "finished_at": "2026-10-19T13:16:25.741910Z" + }, + { + "arm": "main", + "task": "terminal-bench/build-cython-ext", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/build-cython-ext__AcKrSeD/result.json", + "result_sha256": "eb91a96d979392d78cbd190808933119057ac3fe3f880b7c16edb3b4a746ee18", + "bootstrap_sha256": "324a224408951769384fa327b2681ea1d72a8f50bee2f713befaeab399be1f8e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:15:12.540236Z", + "finished_at": "2026-10-19T13:16:08.891548Z" + }, + { + "arm": "main", + "task": "terminal-bench/build-pov-ray", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/build-pov-ray__YrawpJV/result.json", + "result_sha256": "777bbd00ac72baf20292d65c3c600158323de04380e5b33efb074a95e2c64400", + "bootstrap_sha256": "0ebfd791a33d4b0978a4942b1cf9effb2a4347febb1a4ac2efa9520b1f79096e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:16:11.491224Z", + "finished_at": "2026-10-19T13:17:04.626533Z" + }, + { + "arm": "main", + "task": "terminal-bench/caffe-cifar-10", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/caffe-cifar-10__JtiTSgA/result.json", + "result_sha256": "f9b7522ee51557ee07bea2de648d6ece5ecfc9a5aca863cc29114ce6f6047776", + "bootstrap_sha256": "0ebfd791a33d4b0978a4942b1cf9effb2a4347febb1a4ac2efa9520b1f79096e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:16:27.100557Z", + "finished_at": "2026-10-19T13:17:22.025668Z" + }, + { + "arm": "main", + "task": "terminal-bench/cancel-async-tasks", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/cancel-async-tasks__abay9aV/result.json", + "result_sha256": "38ad7f1144c033d424a9512f1f75e449b7c11c5237ff51fbdb0fa8322be6b067", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:17:06.074938Z", + "finished_at": "2026-10-19T13:18:04.698437Z" + }, + { + "arm": "main", + "task": "terminal-bench/circuit-fibsqrt", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/circuit-fibsqrt__SVkt6UK/result.json", + "result_sha256": "58451281e419ba7f796b53ac9577c54bb93d8bc4496992400dd706b6b8d367fe", + "bootstrap_sha256": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:17:23.836052Z", + "finished_at": "2026-10-19T13:18:20.535668Z" + }, + { + "arm": "main", + "task": "terminal-bench/cobol-modernization", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/cobol-modernization__tUc2bp6/result.json", + "result_sha256": "b15ce7b9d9d3d6d396b7136d53abe936f37af405a1f9575272861089dacf7d7b", + "bootstrap_sha256": "92f37a4e542194f29e3ed9bbb49ac8e85ed44b50d7302bed6ba53dee30b37d61", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:18:06.038349Z", + "finished_at": "2026-10-19T13:19:16.468335Z" + }, + { + "arm": "main", + "task": "terminal-bench/constraints-scheduling", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/constraints-scheduling__wGKDyi9/result.json", + "result_sha256": "4b6b33f3c5b88cf535767a36a89178ee2cba1a9ec83fde223c3f5a7678c4a45e", + "bootstrap_sha256": "9705c14c20a581a03dadb6e7334925f68eb71992573c8b228c8c79ef0655e9e9", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:18:22.135295Z", + "finished_at": "2026-10-19T13:19:22.955131Z" + }, + { + "arm": "main", + "task": "terminal-bench/custom-memory-heap-crash", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/custom-memory-heap-crash__YkCeR9M/result.json", + "result_sha256": "eb31be14382121d747f2c799be8916e6b317165f8882c0daa992415e5aa2d9e6", + "bootstrap_sha256": "b1cca4c2159275ee7676539abaf5854f9c053ddaa2a5477b3762d3e244f2a04f", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:19:17.799127Z", + "finished_at": "2026-10-19T13:21:04.686171Z" + }, + { + "arm": "main", + "task": "terminal-bench/db-wal-recovery", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/db-wal-recovery__72Euzgt/result.json", + "result_sha256": "11609b4cc04662eb603584f485ab6193fa0ecd981c06f691e476d9cd0c3f785b", + "bootstrap_sha256": "b5e3b1241fffd9cb9c124014d0da1221f3dd38771c762332e33db42dd948a9b8", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:19:25.570501Z", + "finished_at": "2026-10-19T13:21:06.938778Z" + }, + { + "arm": "main", + "task": "terminal-bench/dna-assembly", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/dna-assembly__oLidH7H/result.json", + "result_sha256": "ce78ce2f2c7327921b9cc91d327e5937f8af5efdbbe8953b23d78a356be4bc66", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:21:07.358307Z", + "finished_at": "2026-10-19T13:22:52.593972Z" + }, + { + "arm": "main", + "task": "terminal-bench/fix-code-vulnerability", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/fix-code-vulnerability__zSiCJCL/result.json", + "result_sha256": "d54ad89483fc5d576816dab4c2d696a02cc8e5ca4f2cac908407b02a48756621", + "bootstrap_sha256": "b8f2c9df149283c7445145220c5cb37c36fc9acfe0b60dc3a92532845b1966cc", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:21:07.596608Z", + "finished_at": "2026-10-19T13:23:26.804966Z" + }, + { + "arm": "main", + "task": "terminal-bench/git-multibranch", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/git-multibranch__d7USbyH/result.json", + "result_sha256": "a234d4227ae620016cd873293619782d1e9aab01bb772f52ce454d1e21cfd20b", + "bootstrap_sha256": "6dc412dac065243b61cef728ad0604dd165c861955efb831afaf2fe1f4aa2feb", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:22:54.595191Z", + "finished_at": "2026-10-19T13:24:20.010352Z" + }, + { + "arm": "main", + "task": "terminal-bench/kv-store-grpc", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/kv-store-grpc__eYejAv7/result.json", + "result_sha256": "da3ba5cacef8f87048e20c8fd9446c20270caa5b663ab1dcc9fc71c9e19a4ce9", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:23:29.233529Z", + "finished_at": "2026-10-19T13:25:35.910272Z" + }, + { + "arm": "main", + "task": "terminal-bench/llm-inference-batching-scheduler", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/llm-inference-batching-scheduler__H8UUtuL/result.json", + "result_sha256": "8c57cbcb6700400662fb3d9c7c6178985a6f93fc3409642893fb69060fb470f3", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:24:23.090832Z", + "finished_at": "2026-10-19T13:26:14.362653Z" + }, + { + "arm": "main", + "task": "terminal-bench/log-summary-date-ranges", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/log-summary-date-ranges__RSkJxvq/result.json", + "result_sha256": "0b0916e25de82f5009edf6818ab5d2ee93f594f9281b4e0960abe4d410b4bd18", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:25:37.965528Z", + "finished_at": "2026-10-19T13:27:08.774851Z" + }, + { + "arm": "main", + "task": "terminal-bench/merge-diff-arc-agi-task", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/merge-diff-arc-agi-task__pebzZBA/result.json", + "result_sha256": "1d11b6daabd37a2690bbdd77e31b435b3783c4e9ba91673b06872211b47dca91", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:26:16.038482Z", + "finished_at": "2026-10-19T13:27:47.723512Z" + }, + { + "arm": "main", + "task": "terminal-bench/model-extraction-relu-logits", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/model-extraction-relu-logits__wLmRiqE/result.json", + "result_sha256": "586176ca56a1e0f1458eea7174a0dc32870fa9f829166c5da19ca1f217aa4e9b", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:27:11.096025Z", + "finished_at": "2026-10-19T13:29:42.496123Z" + }, + { + "arm": "main", + "task": "terminal-bench/mteb-leaderboard", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/mteb-leaderboard__4fYZMaB/result.json", + "result_sha256": "0b881e62eec5a6ef77aef0961d09c72ffc39646ae886432f6b77e7064b9f1a1d", + "bootstrap_sha256": "a3d2b8905b957adcb3553196f9680346955b8f276e2915b0e3539aab45b5407e", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:27:51.419391Z", + "finished_at": "2026-10-19T13:29:40.621920Z" + }, + { + "arm": "main", + "task": "terminal-bench/multi-source-data-merger", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/multi-source-data-merger__VEqER8d/result.json", + "result_sha256": "e213c89e2520ba176473e17b8e0990772af42f8657bf184375eca234a75c5f11", + "bootstrap_sha256": "5011ac5d6ab074bd66636cc77aae3a2d966293bcb051b929e7a9cb076f908948", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:29:41.906488Z", + "finished_at": "2026-10-19T13:30:36.452283Z" + }, + { + "arm": "main", + "task": "terminal-bench/nginx-request-logging", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/nginx-request-logging__TK6evE9/result.json", + "result_sha256": "dda3b153fabea4e03d3ceef5e747e15211043055449f5cf1bccb8979e4d9684a", + "bootstrap_sha256": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:29:42.751157Z", + "finished_at": "2026-10-19T13:30:41.480844Z" + }, + { + "arm": "main", + "task": "terminal-bench/openssl-selfsigned-cert", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/openssl-selfsigned-cert__NHLRsUU/result.json", + "result_sha256": "bff0dcb6001fad4c06aba7e225434b0bad9ea469cc25df7793ded1aa35ab9406", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:30:37.877689Z", + "finished_at": "2026-10-19T13:32:15.288500Z" + }, + { + "arm": "main", + "task": "terminal-bench/path-tracing", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/path-tracing__MDLwYoa/result.json", + "result_sha256": "441be3b43d7ad91c739f06f3aab716a44a69b9e04f52d16b1c359cf5780909bd", + "bootstrap_sha256": "2cdec543c50c5600582cfc3b2ebbe2f2c9d16988e065a02cf6e8e823f3a05542", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:30:41.936311Z", + "finished_at": "2026-10-19T13:32:04.606460Z" + }, + { + "arm": "main", + "task": "terminal-bench/pypi-server", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pypi-server__DrMvmHf/result.json", + "result_sha256": "b22a835c15ef8571934127f65034c95d750780d270019305556c42ea81e41fad", + "bootstrap_sha256": "a3e136f6dc450bc5944d3e77cf83fe313eaf3e432eed5e039bdea46131dc318b", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:32:06.232779Z", + "finished_at": "2026-10-19T13:33:04.331816Z" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-cli", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pytorch-model-cli__479me7S/result.json", + "result_sha256": "ed1348c5f597546163447d4beed18a66989b165f8b4a6efab6c97ecbe74c37fe", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:32:16.867688Z", + "finished_at": "2026-10-19T13:33:17.110191Z" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-recovery", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/pytorch-model-recovery__QgtL4QH/result.json", + "result_sha256": "d0246b8672e5863312892ec3da55944e67139f4fd6fbdcf029512fb1cd300706", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:33:06.594693Z", + "finished_at": "2026-10-19T13:34:06.142083Z" + }, + { + "arm": "main", + "task": "terminal-bench/qemu-alpine-ssh", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/qemu-alpine-ssh__fRwRKWY/result.json", + "result_sha256": "db70bcc900bcb84947f014b5cb9d80ecfcd26e81aac181b168a41efb03e72afd", + "bootstrap_sha256": "aaaa4fafbe6b8a8077e28cffbce3abddae4c4daa3e8fc813010552542cacc1a2", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": "qemu-alpine-ssh-v1", + "status": "passed" + }, + "started_at": "2026-10-19T13:33:18.703941Z", + "finished_at": "2026-10-19T13:34:30.414138Z" + }, + { + "arm": "main", + "task": "terminal-bench/query-optimize", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/query-optimize__PHokY2V/result.json", + "result_sha256": "86dda8afbd1df2bb80caa3bb154571396b9ce83664156badced3a3848650709b", + "bootstrap_sha256": "90bb56ff2a427a339bca3904e95ff8d7b2cfdca6af6af63ab01995c7c52cf240", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:34:08.137262Z", + "finished_at": "2026-10-19T13:36:18.370921Z" + }, + { + "arm": "main", + "task": "terminal-bench/regex-chess", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/regex-chess__mSWEp4p/result.json", + "result_sha256": "eb3d47204a40434cde47dc3e826107bb8b866c64207ec902af99c624713a14bc", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:34:32.403884Z", + "finished_at": "2026-10-19T13:36:53.843699Z" + }, + { + "arm": "main", + "task": "terminal-bench/regex-log", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/regex-log__m2ebsyq/result.json", + "result_sha256": "c12f672e7faff3bea9def494f81af8323083a7f9c4472b2b7d2adfa63d0f1014", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:36:21.322104Z", + "finished_at": "2026-10-19T13:38:04.837184Z" + }, + { + "arm": "main", + "task": "terminal-bench/schemelike-metacircular-eval", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/schemelike-metacircular-eval__nRi7rEH/result.json", + "result_sha256": "1597cae07207dac8b1fd030358f95e2f5c6368bdfabc265f926d69232df72782", + "bootstrap_sha256": "b96b882b4d82151ae41b76cba8fbc65cd511d27da8e50683be9180fb72d2e464", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_failed", + "exception_type": "NetworkConnectionError", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:36:55.358967Z", + "finished_at": "2026-10-19T13:37:45.290134Z" + }, + { + "arm": "main", + "task": "terminal-bench/torch-pipeline-parallelism", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/torch-pipeline-parallelism__vSaJRmN/result.json", + "result_sha256": "dee0aa434d2b82027fd8c4532b15deb9720d750654dfc27813deae804616d6f3", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:37:47.819341Z", + "finished_at": "2026-10-19T13:40:17.013593Z" + }, + { + "arm": "main", + "task": "terminal-bench/torch-tensor-parallelism", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/torch-tensor-parallelism__LaLFMf7/result.json", + "result_sha256": "0cdf7bfdc1d92b38bd0d42f344854a60e887c339bc3879f35ac0d3f118f89e53", + "bootstrap_sha256": "4c37ba0ed09649898da5700b59007b8d2e16adc7b59113adf139e8a5c66a2cec", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:38:09.015497Z", + "finished_at": "2026-10-19T13:39:53.948577Z" + }, + { + "arm": "main", + "task": "terminal-bench/write-compressor", + "result_path": "jobs/main-source/jobs/dev34-1017-install-only-main-20261019/write-compressor__TYgm2Z6/result.json", + "result_sha256": "320c9bb44fb28dc1acb45caf1889c66abd1d2973dea945d12ac289414d2cbae5", + "bootstrap_sha256": "031768ca83f80450befe1475e9f6bfc93052b2053d23aae46a188aa390a36aba", + "manifest_sha256": "5c9b34c5eed748a6f7ac47de5840a09941a81ba2f05579b27b5dfa52c55d5601", + "input_config_sha256": "92a7aa5de7a92c36cad840c53227a849c7e26e8348454b489f34c8f6887e3c7a", + "job": "dev34-1017-install-only-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:39:56.306661Z", + "finished_at": "2026-10-19T13:42:06.864705Z" + }, + { + "arm": "main", + "task": "terminal-bench/schemelike-metacircular-eval", + "result_path": "jobs/main-source/jobs/dev34-1017-schemelike-preflight-recovery-main-20261019/schemelike-metacircular-eval__agfE5zu/result.json", + "result_sha256": "425babe123990ef719e9d4525782b30b8279fdca68b2a829ade7f94d4ea5afff", + "bootstrap_sha256": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "manifest_sha256": "8d355368404fbe99e2d8cc0baf9d36b0127f957d94d8c3aa318a087c67e019fd", + "input_config_sha256": "43327ff3a8de9865142362a0df3b8c4602a5713119c834843bc5a9acbab44b5c", + "job": "dev34-1017-schemelike-preflight-recovery-main-20261019", + "classification": "setup_passed", + "exception_type": null, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "started_at": "2026-10-19T13:42:31.563242Z", + "finished_at": "2026-10-19T13:44:17.830597Z" + } + ] + }, + "host_launch_failures": { + "collected_at": "2026-10-19T14:17:10.343381+00:00", + "original_plan": "experiment-v2.json", + "policy": "Preserve every failed launch. Supplemental jobs may only cover tasks with no native trial or model output; generated outputs and scored results are never replaced.", + "failed_launches": [ + { + "task": "terminal-bench/git-multibranch", + "job": "dev34-1017-git-multibranch-main-20261019", + "classification": "launch_error_before_model", + "native_trial_results": 0, + "model_outputs": 0, + "stage": "registry_lookup", + "reason": "Host TLS/network connection failed before task execution.", + "evidence_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-git-multibranch-main-20261019.json": "3ccd84b057ba3dbc8bb517d9e8148d4b73a7f75709770952af7bef9f2103aa02", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-git-multibranch-main-20261019.log": "83a71596b69f66e017e55b86971e74b4bd867262d81e3b9844edb20e074e91a3", + "jobs/main-source/jobs/dev34-1017-git-multibranch-main-20261019-input/harbor.log": "1667d71a6aea9f63b268b07433bff55c86f5d519ace2bf110e33d58fb6900c37", + "jobs/main-source/jobs/dev34-1017-git-multibranch-main-20261019-input/manifest.json": "77bed126c6acc050f7c390441c0b69673afc10c66976acc39600f32fd1297459", + "jobs/main-source/jobs/dev34-1017-git-multibranch-main-20261019-input/config.json": "528820d8642370dd93f64a6a7bf7068f005ae6fbc0d855dbf39bdb8aa4ab8823" + } + }, + { + "task": "terminal-bench/kv-store-grpc", + "job": "dev34-1017-kv-store-grpc-main-20261019", + "classification": "launch_error_before_model", + "native_trial_results": 0, + "model_outputs": 0, + "stage": "registry_lookup", + "reason": "Host TLS/network connection failed before task execution.", + "evidence_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-kv-store-grpc-main-20261019.json": "d359bb106f3e23a3ef3e73156cc5f7594a0dc60cefffb5da4aa3ff3e5b4d9a12", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-kv-store-grpc-main-20261019.log": "deeaa8da35c7499fac9262268a93833b20b58adc22c20e70e79017c10399f12a", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-main-20261019-input/harbor.log": "1667d71a6aea9f63b268b07433bff55c86f5d519ace2bf110e33d58fb6900c37", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-main-20261019-input/manifest.json": "a7912e4feac60685a27c953d2618c5fdd09d93b20983577a4e12b89e0afa7b75", + "jobs/main-source/jobs/dev34-1017-kv-store-grpc-main-20261019-input/config.json": "bb9bc3da8927365bfe312c4a5b0454cf2a78b83bb02728e7bf56d0bb01aca484" + } + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "job": "dev34-1017-log-summary-date-ranges-main-20261019", + "classification": "launch_error_before_model", + "native_trial_results": 0, + "model_outputs": 0, + "stage": "wheel_build", + "reason": "Host TLS/network connection failed before task execution.", + "evidence_sha256": { + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-log-summary-date-ranges-main-20261019.json": "6bd96070a09869ff104dee9e3e3d9fd11f3de8d30dad45d9ecc42313be78102f", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-log-summary-date-ranges-main-20261019.log": "93f786a8169395ffdfd522c157244dd4a7948fcea967f723183538676d961135" + } + } + ] + }, + "supplemental_plans": [ + { + "path": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-git-multibranch.json", + "sha256": "8dbb29c09d65ffe0a109d909257404352922790f389e658dcf3426b4d340c20a", + "job": "dev34-1017-git-multibranch-launch-recovery-main-20261019" + }, + { + "path": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-kv-store-grpc.json", + "sha256": "3238039ee64e1b00771bd2e40512b14fbc16d82710e8b62c743939853ef632b3", + "job": "dev34-1017-kv-store-grpc-launch-recovery-main-20261019" + }, + { + "path": "benchmarks/harbor/configs/dev34-flash1017-main-20261019/launch-recovery-log-summary-date-ranges.json", + "sha256": "386ae22644903dcdd5b859524daac99dd16f3c6dcba130283804600ded46c60f", + "job": "dev34-1017-log-summary-date-ranges-launch-recovery-main-20261019" + } + ], + "historical_comparison": { + "baseline": "dev34-main-baseline-20261012", + "passed": 28, + "tasks": 34, + "note": "Single fresh trials on previously exposed tasks under changing service and dependency conditions do not isolate checkpoint effects.", + "transition_counts": { + "previous_failure -> passed": 3, + "previous_pass -> passed": 23, + "previous_failure -> scored_failure": 3, + "previous_pass -> verifier_error": 3, + "previous_pass -> scored_failure": 2 + }, + "agent_python_sources_byte_identical": true, + "prompt_byte_identical": true + }, + "publication_policy": "Raw trajectories, native logs, and credentials remain local. Publish derived findings and evidence hashes only.", + "post_freeze_audit": { + "completed_at": "2026-10-19T18:18:42.430670+00:00", + "reviewed_agent_trajectories": 34, + "reviewed_agent_stdout_logs": 34, + "exact_configured_provider_key_matches": 0, + "observed_task_specific_benchmark_answer_retrieval": false, + "observed_hidden_verifier_content_access": false, + "observations": [ + "WAL listed an empty verifier-log directory and filesystem metadata.", + "Tensor parallelism searched recently created test files; returned paths were installed dependency tests.", + "Network references concerned general dependencies, original build sources, public datasets, and task-requested MTEB leaderboard data.", + "The passing Scheme evaluator resets the supplied interpreter evaluation-depth counter through the host multiplication primitive Python-eval behavior. Preserve its native pass and record this implementation caveat.", + "The CLI verifier environment failure followed agent dpkg --force-all installations; do not describe all three unscored tasks as external network outages." + ], + "scope_limit": "Tool-call, observation, URL, and exact-key review supports these observed findings; it cannot prove absence of pretraining contamination or all possible information exposure.", + "method": "Automated scans of all 34 trajectories and stdout logs, followed by manual review of flagged tool calls, observations, external URLs, scored failures, and verifier setup errors." + }, + "final_image_integrity": { + "checked_at": "2026-10-19T18:11:25.008070+00:00", + "unchanged_images": 34, + "all_original_image_identities_match": true, + "remaining_containers": [], + "evidence_sha256": "e290cb41566b44ac8a2ca43905a79b7f57b6c005e9162704f528c84edb53d11e" + }, + "host_storage_maintenance": { + "files_temporarily_relocated": 50, + "bytes": 1576934879, + "restored_at": "2026-10-19T16:09:58.282218+00:00", + "restoration_hashes_verified": true, + "scope": "Duplicate host OCI archives and image blobs only; original Docker images, task inputs, package sources, and verifier scripts unchanged.", + "evidence_sha256": "2ae38631745ea71f3f08756eb1f04a2af6592b2f934f7d290fa3b04c47320e7b" + } +} From 1ae6e05bd0e00b1a0f152604476362836812d576 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 19 Oct 2026 11:25:43 -0700 Subject: [PATCH 05/12] docs(benchmarks): publish bilingual flash 1017 dev34 reports --- .../dev34-flash1017-main-20261019/README.md | 2 +- .../{ => en}/dev34-flash1017-main-20261019.md | 14 +-- .../zh-CN/dev34-flash1017-main-20261019.md | 87 +++++++++++++++++++ .../dev34-flash1017-main-20261019.json | 9 ++ 4 files changed, 106 insertions(+), 6 deletions(-) rename benchmarks/harbor/reports/{ => en}/dev34-flash1017-main-20261019.md (81%) create mode 100644 benchmarks/harbor/reports/zh-CN/dev34-flash1017-main-20261019.md diff --git a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md index 373d06f..eef2dd2 100644 --- a/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md +++ b/benchmarks/harbor/configs/dev34-flash1017-main-20261019/README.md @@ -1,6 +1,6 @@ # Current-main dev34 baseline with DeepSeek Flash 1017 -**Completed:** 26 passes, five scored failures, and three unscored verifier errors. All 34 tasks started a fresh agent trial. See the [report](../../reports/dev34-flash1017-main-20261019.md) and [audited results](../../results/dev34-flash1017-main-20261019.json). Native zero rewards for pre-assertion verifier failures remain preserved; analytical scores are null. The CLI dependency failure followed agent-forced package installations. A passing Scheme evaluator uses a host depth-counter workaround, recorded separately. +**Completed:** 26 passes, five scored failures, and three unscored verifier errors. All 34 tasks started a fresh agent trial. See the [English report](../../reports/en/dev34-flash1017-main-20261019.md), [complete Simplified Chinese report](../../reports/zh-CN/dev34-flash1017-main-20261019.md) and [audited results](../../results/dev34-flash1017-main-20261019.json). Native zero rewards for pre-assertion verifier failures remain preserved; analytical scores are null. The CLI dependency failure followed agent-forced package installations. A passing Scheme evaluator uses a host depth-counter workaround, recorded separately. This registration runs all 34 existing development tasks once with fresh model outputs from current main `ce28b1e826a292d796ba64f13e79199b66f65758`. The experiment branch and its independent diff --git a/benchmarks/harbor/reports/dev34-flash1017-main-20261019.md b/benchmarks/harbor/reports/en/dev34-flash1017-main-20261019.md similarity index 81% rename from benchmarks/harbor/reports/dev34-flash1017-main-20261019.md rename to benchmarks/harbor/reports/en/dev34-flash1017-main-20261019.md index 029fd99..f80409d 100644 --- a/benchmarks/harbor/reports/dev34-flash1017-main-20261019.md +++ b/benchmarks/harbor/reports/en/dev34-flash1017-main-20261019.md @@ -1,10 +1,14 @@ -# Current-main dev34 baseline with Flash 1017 +# Dev34 baseline on main at registration with Flash 1017 + +[English](../en/dev34-flash1017-main-20261019.md) | [简体中文](../zh-CN/dev34-flash1017-main-20261019.md) **26 tasks passed, five scored failures, and three unscored verifier errors.** All 34 tasks started one fresh agent trial. The observed pass count is **26/34 (76.47%)**; the three unscored tasks have unknown correctness and are not counted as scored failures. Two native reward zeros remain preserved alongside null analytical scores; the third unscored task raised `VerifierTimeoutError`. The run evaluates main `ce28b1e826a292d796ba64f13e79199b66f65758` using the official DeepSeek Anthropic-compatible endpoint and `deepseek-flash`. [Official documentation](https://api-docs.deepseek.com/quick_start/agent_integrations/) was checked before execution and maps that alias to **DeepSeek-V4.1-Flash-1017**. The alias is not an immutable checkpoint pin. Agent Python sources and the 385-word prompt are byte-identical to the October 12 baseline; the adapter is from current main, with optional dependency-error recovery explicitly disabled. -The [structured results](../results/dev34-flash1017-main-20261019.json) retain native rewards, analytical scores, all trial IDs, source/wheel/task hashes, verifier attempts, test counts, accounting, audit notes, and original launch errors. The [active plan](../configs/dev34-flash1017-main-20261019/experiment-v2.json), [frozen prompt](../configs/dev34-flash1017-main-20261019/main-prompt.txt), and [configuration README](../configs/dev34-flash1017-main-20261019/README.md) record reproduction details. The primary plan SHA256 is `2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c`. +The [structured results](../../results/dev34-flash1017-main-20261019.json) retain native rewards, analytical scores, all trial IDs, source/wheel/task hashes, verifier attempts, test counts, accounting, audit notes, and original launch errors. The [active plan](../../configs/dev34-flash1017-main-20261019/experiment-v2.json), [frozen prompt](../../configs/dev34-flash1017-main-20261019/main-prompt.txt), and [configuration README](../../configs/dev34-flash1017-main-20261019/README.md) record reproduction details. The primary plan SHA256 is `2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c`. + +During the run, main advanced to `8f85809e7e0270bfe646f0b175707adc835daca4` through a documentation-only merge. Agent and adapter Python sources remain identical to the evaluated checkout. The report pair follows those updated bilingual conventions; measured inputs and outputs remain pinned to the original registration. ## Per-task outcomes @@ -62,9 +66,9 @@ Thirty-one tasks executed official assertions: **141 pytest cases, 133 passed an All task revisions, original image identities, resources, package sources, official verifier scripts, and per-attempt limits match the registered development pool. Every installed wheel matches its recorded source snapshot. Limits remained 100 model replies, 65,536 output tokens per reply, native task agent time minus 180 seconds for work, and a separate 1,800-second setup limit. Temperature and reasoning effort were unset. -Fresh installation preflight produced **34 successes and one TLS setup failure** across 35 attempts. The failed Scheme installation was preserved and a new installation-only check succeeded. Only QEMU has the reviewed verifier dependency profile (`qemu-alpine-ssh-v1`), which passed; the other 33 remain explicitly `not_configured`. [Preflight evidence](../configs/dev34-flash1017-main-20261019/preflight-evidence.json) binds every native record. Installation success does not certify unsupported verifier dependencies. +Fresh installation preflight produced **34 successes and one TLS setup failure** across 35 attempts. The failed Scheme installation was preserved and a new installation-only check succeeded. Only QEMU has the reviewed verifier dependency profile (`qemu-alpine-ssh-v1`), which passed; the other 33 remain explicitly `not_configured`. [Preflight evidence](../../configs/dev34-flash1017-main-20261019/preflight-evidence.json) binds every native record. Installation success does not certify unsupported verifier dependencies. -Three primary host jobs failed before any native task trial or model output existed: two Harbor registry connections and one wheel-build PyPI TLS request. [Original launch evidence](../configs/dev34-flash1017-main-20261019/launch-failure-evidence.json) remains unchanged. Three pre-registered single-task supplemental plans used the free slot in the original final pair and ran serially. Thus 37 host launches produced **34 distinct fresh agent trials**, with no replacement of any generated output. The primary pair barriers and a global maximum of two concurrent task jobs were verified from timestamps. +Three primary host jobs failed before any native task trial or model output existed: two Harbor registry connections and one wheel-build PyPI TLS request. [Original launch evidence](../../configs/dev34-flash1017-main-20261019/launch-failure-evidence.json) remains unchanged. Three pre-registered single-task supplemental plans used the free slot in the original final pair and ran serially. Thus 37 host launches produced **34 distinct fresh agent trials**, with no replacement of any generated output. The primary pair barriers and a global maximum of two concurrent task jobs were verified from timestamps. Verification made **37 attempts, including three timeout retries**. Model recovery exhausted three 900-second attempts before pytest. Pipeline parallelism timed out at 900 seconds during dependency preparation, then completed its second attempt in 427.42 seconds, including the valid scored failure. Every other task used one attempt. No download-error retries, changed per-attempt limits, or retries after executed test results were enabled. @@ -78,6 +82,6 @@ Automated scans covered all 34 trajectories and agent stdout logs. Flagged tool The passing Scheme evaluator relies on host-interpreter introspection: it uses the multiplication primitive's Python evaluation to reset the host evaluation-depth counter. Its official pass is retained with this implementation caveat. -The October 12 baseline had 28 passes and six scored failures. In this fresh run, the sampler, KV store, and SQL optimization changed from failure to pass; WAL, pipeline parallelism, and regex chess remained scored failures. DNA assembly and POV-Ray changed from pass to scored failure. Three formerly passing tasks became unscored verifier errors. Previous outputs and scores were not reused. +[The October 12 baseline](dev34-main-baseline-20261012.md) had 28 passes and six scored failures. In this fresh run, the sampler, KV store, and SQL optimization changed from failure to pass; WAL, pipeline parallelism, and regex chess remained scored failures. DNA assembly and POV-Ray changed from pass to scored failure. Three formerly passing tasks became unscored verifier errors. Previous outputs and scores were not reused. All 34 tasks have prior development exposure. Single fresh trials, live dependency resolution, agent-modified environments, host-launch recovery, and changing provider conditions prevent a controlled checkpoint-effect conclusion. Keep the three unscored tasks separate, retain the passing Scheme implementation caveat, and use the per-task evidence for development rather than treating this as a held-out capability score. diff --git a/benchmarks/harbor/reports/zh-CN/dev34-flash1017-main-20261019.md b/benchmarks/harbor/reports/zh-CN/dev34-flash1017-main-20261019.md new file mode 100644 index 0000000..d9f4702 --- /dev/null +++ b/benchmarks/harbor/reports/zh-CN/dev34-flash1017-main-20261019.md @@ -0,0 +1,87 @@ +# 登记时 main 的 Flash 1017 dev34 baseline + +[English](../en/dev34-flash1017-main-20261019.md) | [简体中文](../zh-CN/dev34-flash1017-main-20261019.md) + +**26 题通过、5 题计分失败、3 题 verifier 错误未计分。** 34 题均启动了一次全新的 agent 试次。观察到的通过数为 **26/34（76.47%）**；三道未计分题的正确性仍未知，不将它们计为评分失败。两项原生零分奖励与空的分析分数分别保留；第三项未计分任务抛出 `VerifierTimeoutError`。 + +本轮评估 main `ce28b1e826a292d796ba64f13e79199b66f65758`，通过 DeepSeek 官方 Anthropic 兼容端点使用 `deepseek-flash`。运行前核对的[官方文档](https://api-docs.deepseek.com/quick_start/agent_integrations/)将该别名映射到 **DeepSeek-V4.1-Flash-1017**。该别名并非不可变 checkpoint 固定标识。Agent 的 Python 源码及 385 词提示词与 10 月 12 日 baseline 逐字节一致；adapter 来自当前 main，显式关闭可选的依赖错误恢复。 + +[结构化结果](../../results/dev34-flash1017-main-20261019.json)保留原生奖励、分析分数、全部试次 ID、源码/wheel/任务哈希、verifier 尝试、测试计数、用量与费用记录、审计备注和原始启动错误。[生效计划](../../configs/dev34-flash1017-main-20261019/experiment-v2.json)、[冻结提示词](../../configs/dev34-flash1017-main-20261019/main-prompt.txt)和[配置 README](../../configs/dev34-flash1017-main-20261019/README.md)记录复现细节。主计划 SHA256 为 `2a565629eab164eee1f4738bd65bac0f86e8d44017370e6ce9ca31803656ec4c`。 + +运行期间，main 通过一次仅修改文档的合并推进至 `8f85809e7e0270bfe646f0b175707adc835daca4`。Agent 与 adapter 的 Python 源码仍与被评估 checkout 完全一致。报告遵循更新后的双语规范；实际输入与输出仍固定于原登记计划。 + +## 逐题结果 + +| 任务 | 10 月 12 日奖励 | 本轮结果 | 官方 pytest 通过 / 失败 | Agent 秒数 | 阶段 | +| --- | ---: | --- | ---: | ---: | --- | +| adaptive-rejection-sampler | 0 | 通过 | 9 / 0 | 614.4 | 原队列 | +| build-cython-ext | 1 | 通过 | 11 / 0 | 436.3 | 原队列 | +| cancel-async-tasks | 1 | 通过 | 6 / 0 | 179.8 | 原队列 | +| cobol-modernization | 1 | 通过 | 3 / 0 | 262.0 | 原队列 | +| db-wal-recovery | 0 | 失败 | 5 / 2 | 132.8 | 原队列 | +| fix-code-vulnerability | 1 | 通过 | 6 / 0 | 24.6 | 原队列 | +| git-multibranch | 1 | 通过 | 1 / 0 | 220.2 | 补启动 | +| kv-store-grpc | 0 | 通过 | 7 / 0 | 297.1 | 补启动 | +| log-summary-date-ranges | 1 | 通过 | 2 / 0 | 25.8 | 补启动 | +| merge-diff-arc-agi-task | 1 | 通过 | 5 / 0 | 58.6 | 原队列 | +| model-extraction-relu-logits | 1 | 通过 | 1 / 0 | 301.4 | 原队列 | +| multi-source-data-merger | 1 | 未计分 | 未启动 | 38.2 | 原队列 | +| nginx-request-logging | 1 | 通过 | 8 / 0 | 60.0 | 原队列 | +| openssl-selfsigned-cert | 1 | 通过 | 6 / 0 | 54.4 | 原队列 | +| pypi-server | 1 | 通过 | 1 / 0 | 184.6 | 原队列 | +| pytorch-model-cli | 1 | 未计分 | 收集错误 | 461.3 | 原队列 | +| pytorch-model-recovery | 1 | 未计分 | 未启动 | 109.8 | 原队列 | +| qemu-alpine-ssh | 1 | 通过 | 1 / 0 | 506.2 | 原队列 | +| query-optimize | 0 | 通过 | 6 / 0 | 490.9 | 原队列 | +| regex-log | 1 | 通过 | 1 / 0 | 82.5 | 原队列 | +| torch-pipeline-parallelism | 0 | 失败 | 2 / 2 | 584.3 | 原队列 | +| torch-tensor-parallelism | 1 | 通过 | 13 / 0 | 196.0 | 原队列 | +| write-compressor | 1 | 通过 | 3 / 0 | 274.2 | 原队列 | +| constraints-scheduling | 1 | 通过 | 3 / 0 | 21.0 | 原队列 | +| custom-memory-heap-crash | 1 | 通过 | 6 / 0 | 185.5 | 原队列 | +| dna-assembly | 1 | 失败 | 0 / 1 | 346.5 | 原队列 | +| llm-inference-batching-scheduler | 1 | 通过 | 6 / 0 | 529.6 | 原队列 | +| path-tracing | 1 | 通过 | 5 / 0 | 908.7 | 原队列 | +| schemelike-metacircular-eval | 1 | 通过 | 1 / 0 | 1098.5 | 原队列 | +| caffe-cifar-10 | 1 | 通过 | 6 / 0 | 2212.4 | 原队列 | +| circuit-fibsqrt | 1 | 通过 | 3 / 0 | 1101.1 | 原队列 | +| mteb-leaderboard | 1 | 通过 | 2 / 0 | 863.7 | 原队列 | +| regex-chess | 0 | 失败 | 2 / 2 | 1810.4 | 原队列 | +| build-pov-ray | 1 | 失败 | 2 / 1 | 654.7 | 原队列 | + +31 题执行了官方断言：**141 个 pytest 用例，133 通过、8 失败**，没有跳过。CLI 题另产生一次收集错误。原生 CTRF 摘要单独保留，因为参数化用例会按名称去重。此处计数来自最终验证摘要；先前尝试的归档也保留。三次恢复尝试和逐题测试用例数都不是额外的 agent 题目分数。 + +## 计分失败与 verifier 错误证据 + +- `db-wal-recovery`：保留混淆 WAL 前先打开原始 SQLite 数据库。Agent 报告 SQLite 删除了 WAL，随后推测缺失记录。官方测试发现 WAL 更新缺失：5 项通过、2 项失败。 +- `torch-pipeline-parallelism`：两个 world-size 用例均因 `model.rotary_emb.fwd` 的 microbatch 数量不匹配失败。存在性和无 hooks 检查通过。第一次 verifier 尝试在依赖准备阶段超时；第二次执行全部四个用例。 +- `dna-assembly`：提交的引物在 BsaI 识别位点前缺少要求的核苷酸 clamp，官方引物测试失败。 +- `regex-chess`：提交的正则流水线允许王移入被马攻击的格子。官方两项通过、两项失败。本轮实际提交了 `re.json`；这是实现的计分失败，与此前六题重跑中仅因截断而未提交文件的情况不同。 +- `build-pov-ray`：因缺少 `file_id.diz`，官方源码发行包检查失败。可执行文件和渲染检查通过；记录的 SSIM 为 0.8731。Agent 从官方 2.2 归档 URL 下载源码。仅凭标记文件缺失，不能认定实际构建的是其他版本。 +- `multi-source-data-merger`：verifier 在 pytest 启动前因网络超时无法下载或解压 `pyarrow==22.0.0`。原生零分保留，分析分数为空。此次尝试已结束，并非 verifier 进程达到时限，因此未重试。 +- `pytorch-model-cli`：APT 报告依赖不满足后，官方 pytest 在导入 cv2 时因缺少 `libGL.so.1` 而收集失败。Agent 此前使用 `dpkg --force-all` 强制安装编译器和开发包，留下依赖问题；agent 的环境修改促成了此次验证失败。未执行任何断言。原生零分保留，分析分数为空；CLI 正确性仍未验证。 +- `pytorch-model-recovery`：三次原始 900 秒 verifier 尝试均在 pytest 启动前下载 PyTorch/CUDA 依赖时超时，始终复用同一输出。原生 `VerifierTimeoutError` 没有奖励，分析分数为空。 + +## 执行与完整性 + +全部任务修订、原始镜像身份、资源、包源、官方 verifier 脚本和逐次时限均与登记的开发池一致。每个已安装 wheel 均与记录的源码快照匹配。限制仍为 100 次模型回复、每次最多 65,536 个输出 token、原始任务 agent 时限减 180 秒的工作预算，以及独立的 1,800 秒安装时限。温度和 reasoning effort 均未设置。 + +新的安装预检共保留 35 次尝试，得到 **34 次成功、1 次 TLS 安装失败**。失败的 Scheme 安装记录保留，另一次新的仅安装检查成功。仅 QEMU 配置了已审阅的 verifier 依赖 profile（`qemu-alpine-ssh-v1`），且已通过；其余 33 题明确保持 `not_configured`。[预检证据](../../configs/dev34-flash1017-main-20261019/preflight-evidence.json)按哈希绑定每份原生记录。Agent 安装成功不代表未配置的 verifier 依赖已通过认证。 + +原队列中的三项主机作业在任何原生任务试次或模型输出产生前失败：两次 Harbor registry 连接失败和一次构建 wheel 时的 PyPI TLS 请求失败。[原始启动证据](../../configs/dev34-flash1017-main-20261019/launch-failure-evidence.json)保留。三个预登记的单题补启动计划利用原队列最后一对任务空出的执行位串行运行。因此，37 次主机启动产生了 **34 个独立的新 agent 试次**，没有替换任何已生成输出。已根据时间戳核对原队列的成对屏障，以及全局最多两个并发任务作业的限制。 + +验证共进行 **37 次尝试，其中 3 次是超时重试**。模型恢复题在 pytest 启动前用尽三次 900 秒尝试。流水线并行题第一次在依赖准备阶段达到 900 秒时限，第二次在 427.42 秒内完成并得到有效的计分失败。其余每题均只尝试一次。未启用下载错误重试，未改变逐次时限，也未在执行得到测试结果后重试。 + +PyTorch 两题运行期间，因磁盘空间紧张，将重复的主机镜像缓存临时迁移到内存临时文件系统。六个 OCI 归档和 44 个内容寻址 blob 均恢复为普通文件，逐个重新核对 SHA256。原始 Docker 镜像及任务环境未被替换。最终检查确认 **34 个原始镜像身份均未改变**，且**没有残留任务容器**。 + +全部原队列及补启动作业结束后，才审阅新轨迹和官方测试内容。输出冻结记录时间为 `2026-10-19T18:10:43.804209+00:00`。模型阶段占用 **264.23 分钟**，不含仅安装预检。34 份记录的用量和费用信息均完整；随 agent 提供的估算器给出 **$1.861**。这是 token 费用估算，并非账单对账结果。 + +## 冻结后审计与解释 + +自动扫描覆盖全部 34 份轨迹和 agent 标准输出日志，随后审阅标记的工具调用、观察结果、外部 URL、失败证据和 verifier 准备错误。未观察到配置的提供商密钥完整匹配、检索任务专用 benchmark 答案，或读取隐藏 verifier 内容。WAL 题检查了空的 verifier 日志目录。张量并行题搜索最近创建的测试，但观察中返回的是已安装依赖的测试。公开 MTEB 结果属于任务要求的数据来源。这些发现不能证明完全没有信息接触或训练污染。 + +通过的 Scheme 求值器依赖宿主解释器内省：它通过乘法原语的 Python 求值行为，重置宿主求值深度计数器。保留其官方通过分数，同时记录这一实现上的限制。 + +[10 月 12 日 baseline](dev34-main-baseline-20261012.md)为 28 题通过、6 题计分失败。本轮采样器、KV 存储和 SQL 优化由失败变为通过；WAL、流水线并行和正则棋仍为计分失败。DNA 组装和 POV-Ray 由通过变为计分失败。此前通过的三题变为 verifier 错误未计分。未复用任何此前的输出或分数。 + +全部 34 题均已有开发暴露。每题单次新试次、在线依赖解析、agent 修改的环境、主机启动恢复，以及变化的提供商条件，都使本轮无法作为隔离 checkpoint 影响的受控比较。三道未计分题单独保留，记录通过的 Scheme 实现限制，逐题证据用于开发分析；不将本轮视为保留集能力分数。 diff --git a/benchmarks/harbor/results/dev34-flash1017-main-20261019.json b/benchmarks/harbor/results/dev34-flash1017-main-20261019.json index 31b845b..ec121b9 100644 --- a/benchmarks/harbor/results/dev34-flash1017-main-20261019.json +++ b/benchmarks/harbor/results/dev34-flash1017-main-20261019.json @@ -4810,5 +4810,14 @@ "restoration_hashes_verified": true, "scope": "Duplicate host OCI archives and image blobs only; original Docker images, task inputs, package sources, and verifier scripts unchanged.", "evidence_sha256": "2ae38631745ea71f3f08756eb1f04a2af6592b2f934f7d290fa3b04c47320e7b" + }, + "documentation_sync": { + "main_ref": "8f85809e7e0270bfe646f0b175707adc835daca4", + "agent_and_adapter_python_unchanged": true, + "reports": { + "en": "benchmarks/harbor/reports/en/dev34-flash1017-main-20261019.md", + "zh-CN": "benchmarks/harbor/reports/zh-CN/dev34-flash1017-main-20261019.md" + }, + "both_language_versions_in_sync": true } } From 40fdd041c370a96dc06f86b436eacc01ce1566d7 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Tue, 20 Oct 2026 06:41:30 -0700 Subject: [PATCH 06/12] fix(benchmarks): support authorized collection-error recovery --- benchmarks/harbor/README.md | 14 ++++ benchmarks/harbor/README.zh-CN.md | 10 +++ .../src/harbor_adapter/_verifier_runner.py | 52 ++++++++++++- .../harbor/src/harbor_adapter/verifier.py | 19 ++++- benchmarks/harbor/tests/test_verifier.py | 77 ++++++++++++++++++- 5 files changed, 164 insertions(+), 8 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 82412b7..4ae7791 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -295,6 +295,20 @@ valid scored failures stop immediately. Exhausted dependency failures raise canonical reward. The default remains timeout-only; record the authorization and selected policy in the experiment plan. +For separately authorized recovery of pytest collection failures, also set +`retry_collection_errors=true`. This retries only an explicit collection abort +before any test case runs. A successful reward, executed test cases, or an +ambiguous CTRF report prevents this recovery. Exhaustion raises +`VerifierCollectionError` and preserves archived rewards without publishing a +canonical score. The default is false, and the three-attempt limit is shared +with timeout and dependency recovery. + +When future verification may be needed, set `environment.delete=false` to retain +the stopped task containers and register solution paths such as `/app` in +`artifacts`. Harbor collects these files before verification. Keep both the +artifact manifest and file hashes with the run evidence; trajectories alone do +not preserve binaries or the original system state. + ## Benchmark reports Reports are available in [English](reports/en/) and diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 10823d0..8a9931d 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -238,6 +238,16 @@ Harbor 原生的 provider 凭证和已经配置的 base URL 也会转换为 nano 评分也会立即停止。依赖错误耗尽尝试后抛出 `VerifierDependencyError`，保留每次 证据并清除当前评分。默认仍只重试超时；实验计划应记录授权和实际选用的策略。 +对单独授权的 pytest 收集失败恢复，还可设置 +`retry_collection_errors=true`。它只重试尚未运行任何测试用例时明确发生的收集中断。 +成功评分、已执行的测试用例或含义不明确的 CTRF 报告都会阻止这种恢复。耗尽次数后 +抛出 `VerifierCollectionError`，保留归档评分，但不发布当前评分。默认值为 false， +与超时及依赖恢复共享最多三次的尝试上限。 + +如果之后可能需要继续验证，应设置 `environment.delete=false` 保留已停止的任务 +容器，并在 `artifacts` 中登记 `/app` 等解答路径。Harbor 会在验证前收集这些文件。 +将产物清单和文件哈希与运行证据一起保存；仅有轨迹无法保留二进制文件和原始系统状态。 + ## Benchmark 报告 报告提供[英文](reports/en/)和[简体中文](reports/zh-CN/)两个版本，文件名一一对应。 diff --git a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py index 3b2f830..ce3e766 100644 --- a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py +++ b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py @@ -79,14 +79,49 @@ def dependency_download_failed(logs): ) -def run(command, timeout_sec, max_attempts, logs, control, retry_dependency_errors=False): - """Retry deadlines and, when enabled, pre-test dependency download failures.""" +def collection_failed(logs): + """Recognize aborted pytest collection without retrying executed test cases.""" + for name in ("reward.txt", "reward.json"): + path = logs / name + if path.exists(): + try: + value = (float(path.read_text()) if name.endswith(".txt") + else json.loads(path.read_text()).get("reward")) + except (ValueError, TypeError, AttributeError): + return False + if value != 0: + return False + report = logs / "ctrf.json" + if report.exists(): + try: + results = json.loads(report.read_text())["results"] + summary = results["summary"] + if summary["tests"] != 0 or any( + summary.get(key, 0) != 0 for key in ("passed", "failed", "skipped") + ) or results.get("tests"): + return False + except (ValueError, KeyError, TypeError, AttributeError): + return False + path = logs / "test-stdout.txt" + text = path.read_text(errors="replace") if path.exists() else "" + if re.search(r"\b\d+ (?:passed|failed|skipped|xfailed|xpassed)\b", text): + return False + return bool( + re.search(r"\bERROR collecting\b", text) + and re.search(r"Interrupted: \d+ errors? during collection", text) + ) + + +def run(command, timeout_sec, max_attempts, logs, control, retry_dependency_errors=False, + retry_collection_errors=False): + """Retry deadlines and explicitly authorized pre-assertion failures.""" logs = Path(logs) control = Path(control) cancelled = control.with_suffix(".cancelled") attempts = [] summary = {"timeout_sec": timeout_sec, "max_attempts": max_attempts, - "retry_dependency_errors": retry_dependency_errors, "attempts": attempts} + "retry_dependency_errors": retry_dependency_errors, + "retry_collection_errors": retry_collection_errors, "attempts": attempts} def interrupted(signum, frame): raise InterruptedError("Verifier execution cancelled") @@ -122,6 +157,8 @@ def interrupted(signum, frame): record["status"] = "completed" if retry_dependency_errors and dependency_download_failed(logs): record["status"] = "dependency_error" + elif retry_collection_errors and collection_failed(logs): + record["status"] = "collection_error" except subprocess.TimeoutExpired: _stop_process(process) record["status"] = "timeout" @@ -173,9 +210,16 @@ def cancel(control): if len(arguments) == 5: command, timeout, count, logs, control = arguments retry_dependencies = False - else: + retry_collection = False + elif len(arguments) == 6: command, timeout, count, retry, logs, control = arguments retry_dependencies = retry == "1" + retry_collection = False + else: + command, timeout, count, retry, collection, logs, control = arguments + retry_dependencies = retry == "1" + retry_collection = collection == "1" print(json.dumps(run( command, float(timeout), int(count), logs, control, retry_dependencies, + retry_collection, ))) diff --git a/benchmarks/harbor/src/harbor_adapter/verifier.py b/benchmarks/harbor/src/harbor_adapter/verifier.py index cd5fc3b..155f02f 100644 --- a/benchmarks/harbor/src/harbor_adapter/verifier.py +++ b/benchmarks/harbor/src/harbor_adapter/verifier.py @@ -20,17 +20,22 @@ class VerifierDependencyError(RuntimeError): """Authorized dependency-download recovery exhausted its attempt budget.""" +class VerifierCollectionError(RuntimeError): + """Authorized pytest collection recovery exhausted its attempt budget.""" + + class _RetryEnvironment: """Delegate Harbor operations, wrapping only the test-script execution.""" def __init__(self, environment, command, timeout_sec, max_attempts, logger, - retry_dependency_errors=False): + retry_dependency_errors=False, retry_collection_errors=False): self._environment = environment self._command = command self._timeout_sec = timeout_sec self._max_attempts = max_attempts self._logger = logger self._retry_dependency_errors = retry_dependency_errors + self._retry_collection_errors = retry_collection_errors def __getattr__(self, name): return getattr(self._environment, name) @@ -42,6 +47,7 @@ async def exec(self, *, command, **kwargs): invocation = shlex.join([ "python3", "-c", _RUNNER, command, str(self._timeout_sec), str(self._max_attempts), str(int(self._retry_dependency_errors)), + str(int(self._retry_collection_errors)), "/logs/verifier", control, ]) try: @@ -66,6 +72,10 @@ async def exec(self, *, command, **kwargs): raise VerifierDependencyError( f"Verifier dependency downloads failed after {self._max_attempts} attempts" ) + if record["status"] == "collection_error": + raise VerifierCollectionError( + f"Verifier collection failed after {self._max_attempts} attempts" + ) if record["status"] == "timeout": raise TimeoutError( f"Verifier timed out after {self._max_attempts} attempts " @@ -82,16 +92,20 @@ class RetryingVerifier(Verifier): 4``. Each attempt retains the task's native verifier timeout. """ - def __init__(self, *, max_attempts=3, retry_dependency_errors=False, **kwargs): + def __init__(self, *, max_attempts=3, retry_dependency_errors=False, + retry_collection_errors=False, **kwargs): if type(max_attempts) is not int or not 1 <= max_attempts <= 3: raise ValueError("max_attempts must be an integer between 1 and 3") if type(retry_dependency_errors) is not bool: raise ValueError("retry_dependency_errors must be a boolean") + if type(retry_collection_errors) is not bool: + raise ValueError("retry_collection_errors must be a boolean") super().__init__(**kwargs) if self.environment.os != TaskOS.LINUX or self.step_name is not None: raise ValueError("RetryingVerifier supports single-step Linux tasks") self._max_attempts = max_attempts self._retry_dependency_errors = retry_dependency_errors + self._retry_collection_errors = retry_collection_errors self._attempt_timeout = self.task.config.verifier.timeout_sec if not math.isfinite(self._attempt_timeout) or self._attempt_timeout <= 0: raise ValueError("The task's verifier timeout must be finite and positive") @@ -125,6 +139,7 @@ async def verify(self): self.environment = _RetryEnvironment( environment, command, self._attempt_timeout, self._max_attempts, self.logger, self._retry_dependency_errors, + self._retry_collection_errors, ) try: return await super().verify() diff --git a/benchmarks/harbor/tests/test_verifier.py b/benchmarks/harbor/tests/test_verifier.py index db7620f..4b816b2 100644 --- a/benchmarks/harbor/tests/test_verifier.py +++ b/benchmarks/harbor/tests/test_verifier.py @@ -82,7 +82,8 @@ async def exec(self, *, command, env=None, **kwargs): def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3, include_logs=None, - exclude_logs=None, retry_dependency_errors=False): + exclude_logs=None, retry_dependency_errors=False, + retry_collection_errors=False): task_dir = tmp_path / "task" (task_dir / "tests").mkdir(parents=True) (task_dir / "instruction.md").write_text("Write the solution.") @@ -96,7 +97,8 @@ def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3, include_log verifier=VerifierConfig( import_path="harbor_adapter:RetryingVerifier", kwargs={"max_attempts": max_attempts, - "retry_dependency_errors": retry_dependency_errors}, + "retry_dependency_errors": retry_dependency_errors, + "retry_collection_errors": retry_collection_errors}, include_logs=include_logs or [], exclude_logs=exclude_logs or [], ), ) @@ -234,6 +236,77 @@ def test_invalid_dependency_retry_flags_are_rejected(value): RetryingVerifier(retry_dependency_errors=value) +@pytest.mark.parametrize("mounted", [True, False]) +def test_authorized_collection_retries_stop_on_executed_failure(tmp_path, mounted): + verifier, _, _ = make_verifier(tmp_path, ''' +count=$(cat "$STATE" 2>/dev/null || echo 0) +count=$((count + 1)) +echo "$count" > "$STATE" +if [ "$count" -lt 3 ]; then + echo 'ERROR collecting tests/test_output.py' + echo 'Interrupted: 1 error during collection' + echo '{"results":{"summary":{"tests":0,"passed":0,"failed":0}}}' > "$LOGS/ctrf.json" +else + echo '1 failed in 0.01s' + echo '{"results":{"summary":{"tests":1,"failed":1}}}' > "$LOGS/ctrf.json" +fi +echo 0 > "$LOGS/reward.txt" +''', mounted=mounted, retry_collection_errors=True) + assert asyncio.run(verifier.verify()).rewards == {"reward": 0} + assert [r["status"] for r in summary(verifier)["attempts"]] == [ + "collection_error", "collection_error", "completed", + ] + assert (verifier.trial_paths.verifier_dir / "attempts/1/reward.txt").exists() + + +def test_exhausted_collection_errors_are_unscored(tmp_path): + from harbor_adapter.verifier import VerifierCollectionError + + verifier, _, _ = make_verifier(tmp_path, ''' +echo 'ERROR collecting tests/test_output.py' +echo 'Interrupted: 1 error during collection' +echo 0 > "$LOGS/reward.txt" +''', retry_collection_errors=True) + with pytest.raises(VerifierCollectionError, match="3 attempts"): + asyncio.run(verifier.verify()) + assert len(summary(verifier)["attempts"]) == 3 + assert not verifier.trial_paths.reward_text_path.exists() + assert (verifier.trial_paths.verifier_dir / "attempts/3/reward.txt").exists() + + +@pytest.mark.parametrize("guard", [ + 'echo "1 failed in 0.1s"', + 'echo "1 passed in 0.1s"', + 'echo \'{"results":{"summary":{"tests":1}}}\' > "$LOGS/ctrf.json"', + 'echo "{}" > "$LOGS/ctrf.json"', + 'echo 1 > "$LOGS/reward.txt"', +]) +def test_collection_retries_never_replace_executed_scores(tmp_path, guard): + verifier, _, _ = make_verifier(tmp_path, ''' +echo 'ERROR collecting tests/test_output.py' +echo 'Interrupted: 1 error during collection' +echo 0 > "$LOGS/reward.txt" +''' + guard + "\n", retry_collection_errors=True) + asyncio.run(verifier.verify()) + assert len(summary(verifier)["attempts"]) == 1 + + +def test_collection_recovery_is_disabled_by_default(tmp_path): + verifier, _, _ = make_verifier(tmp_path, ''' +echo 'ERROR collecting tests/test_output.py' +echo 'Interrupted: 1 error during collection' +echo 0 > "$LOGS/reward.txt" +''') + assert asyncio.run(verifier.verify()).rewards == {"reward": 0} + assert len(summary(verifier)["attempts"]) == 1 + + +@pytest.mark.parametrize("value", [1, "true", None]) +def test_invalid_collection_retry_flags_are_rejected(value): + with pytest.raises(ValueError, match="retry_collection_errors"): + RetryingVerifier(retry_collection_errors=value) + + @pytest.mark.parametrize("mounted", [True, False]) def test_three_timeouts_leave_no_canonical_reward_and_keep_all_logs(tmp_path, mounted): verifier, environment, _ = make_verifier(tmp_path, ''' From 0d12394853207d6606468c5876e4af9777b91909 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Tue, 20 Oct 2026 06:48:33 -0700 Subject: [PATCH 07/12] chore(benchmarks): register three-task flash 1017 recovery --- benchmarks/harbor/README.md | 8 +- benchmarks/harbor/README.zh-CN.md | 6 +- .../README.md | 66 ++++++ .../apt-cache.conf | 8 + .../dependency-cache.compose.json | 100 +++++++++ .../dependency-release-manifest.json | 58 +++++ .../experiment-v2.json | 209 ++++++++++++++++++ .../experiment.json | 182 +++++++++++++++ .../image-digests.json | 30 +++ .../install-only.json | 46 ++++ .../main-prompt.txt | 9 + .../multi-source-data-merger-retained.json | 44 ++++ .../multi-source-data-merger.json | 41 ++++ .../original-baseline-evidence.json | 177 +++++++++++++++ .../pytorch-model-cli-retained.json | 43 ++++ .../pytorch-model-cli.json | 40 ++++ .../pytorch-model-recovery-retained.json | 43 ++++ .../pytorch-model-recovery.json | 40 ++++ 18 files changed, 1145 insertions(+), 5 deletions(-) create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/image-digests.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/install-only.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/main-prompt.txt create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger-retained.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/original-baseline-evidence.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli-retained.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery-retained.json create mode 100644 benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery.json diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 4ae7791..39cb137 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -303,9 +303,11 @@ ambiguous CTRF report prevents this recovery. Exhaustion raises canonical score. The default is false, and the three-attempt limit is shared with timeout and dependency recovery. -When future verification may be needed, set `environment.delete=false` to retain -the stopped task containers and register solution paths such as `/app` in -`artifacts`. Harbor collects these files before verification. Keep both the +When future verification may be needed, set Docker's +`environment.kwargs.keep_containers=true` to retain the stopped task containers +and register solution paths such as `/app` in `artifacts`. +`environment.delete=false` alone still removes Docker containers. Harbor collects +the registered files before verification. Keep both the artifact manifest and file hashes with the run evidence; trajectories alone do not preserve binaries or the original system state. diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 8a9931d..5ecd9b8 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -244,8 +244,10 @@ Harbor 原生的 provider 凭证和已经配置的 base URL 也会转换为 nano 抛出 `VerifierCollectionError`，保留归档评分，但不发布当前评分。默认值为 false， 与超时及依赖恢复共享最多三次的尝试上限。 -如果之后可能需要继续验证，应设置 `environment.delete=false` 保留已停止的任务 -容器，并在 `artifacts` 中登记 `/app` 等解答路径。Harbor 会在验证前收集这些文件。 +如果之后可能需要继续验证，应设置 Docker 的 +`environment.kwargs.keep_containers=true` 保留已停止的任务容器，并在 `artifacts` +中登记 `/app` 等解答路径。仅设置 `environment.delete=false` 仍会删除 Docker +容器。Harbor 会在验证前收集已登记的文件。 将产物清单和文件哈希与运行证据一起保存；仅有轨迹无法保留二进制文件和原始系统状态。 ## Benchmark 报告 diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md new file mode 100644 index 0000000..5fc651c --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md @@ -0,0 +1,66 @@ +# Flash 1017 verifier-recovery supplement + +This is a separately authorized three-task experiment, not a regrade or +replacement of the October 19 dev34 baseline. The original containers were +deleted and no `/app` solution artifacts were archived. The user explicitly +selected one new agent solution for each previously unscored task, with at most +three verifier attempts and retained workspaces. + +## Frozen inputs + +The [active plan](experiment-v2.json) fixes `multi-source-data-merger`, +`pytorch-model-cli`, and `pytorch-model-recovery` at the original task revisions. +The detached `jobs/runtime-source` checkout is +`40fdd041c370a96dc06f86b436eacc01ce1566d7`. Its core agent and 385-word prompt +are byte-identical to baseline main +`ce28b1e826a292d796ba64f13e79199b66f65758`; only the verifier harness gains an +explicit, tested collection-error recovery option. Each task uses the original +100-reply, 65,536-token-per-reply and 720-second work budget, 1,800-second setup +limit, original images/resources/package sources/tests, and 900-second verifier +attempt limit. + +The model remains `deepseek/deepseek-flash` at +`https://api.deepseek.com/anthropic`. [Official documentation](https://api-docs.deepseek.com/quick_start/agent_integrations/) +checked on October 20 maps this mutable API alias to DeepSeek-V4.1-Flash-1017. +Temperature and reasoning effort remain unset. + +## Execution and retention + +Run fresh installation-only checks before the model phase. All three tasks have +no registered verifier dependency profile and must report `not_configured`; +installation success does not certify their verifier dependencies. + +```bash +python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json \ + --phase preflight +python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json \ + --phase model +``` + +The existing orchestrator calls the standard Harbor entry point. It runs the +merger and CLI in the first pair, then model recovery, with maximum concurrency +two. Each model runs once. Both `retry_dependency_errors` and +`retry_collection_errors` are explicitly enabled. Timeout, recognized network +download failures before pytest, and aborted pytest collection share a maximum +of three attempts in the same container and with the same agent output. Any +valid scored pass or failure stops immediately. No solution repair, hidden-test +feedback, whole-trial retry, dependency substitution, or longer attempt deadline +is authorized by this registration. + +Model runs set `environment.kwargs.keep_containers=true` as well as +`environment.delete=false`, and collect `/app` (plus `/data` for +the merger) before verification. Retain stopped containers, artifact manifests +and file hashes, input snapshots and all attempts. Original baseline records +remain immutable. Freeze all three new outputs before reviewing trajectories, +artifacts or official test contents. Report new outcomes separately, including +any remaining setup/verifier errors. Prior development exposure and the changed +recovery policy prevent interpreting this as a held-out or checkpoint-only +comparison. + +The initial `experiment.json` ran installation-only checks. Before any model +call, the active v2 plan explicitly adds Docker `keep_containers=true`, because +Harbor 0.21 removes containers even with `delete=false` alone. Both registrations +and original configuration files remain intact; v2 accepts successful native +preflight records by hash and selects the corrected `*-retained.json` configs. diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf new file mode 100644 index 0000000..e191ddb --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf @@ -0,0 +1,8 @@ +// Route only public APT HTTP origins through the verified index cache. +Acquire::http::Proxy::deb.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::security.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::archive.debian.org "http://127.0.0.1:18742/"; +Acquire::http::Proxy::archive.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::security.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::ports.ubuntu.com "http://127.0.0.1:18742/"; +Acquire::http::Proxy::old-releases.ubuntu.com "http://127.0.0.1:18742/"; diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json new file mode 100644 index 0000000..4b46367 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json @@ -0,0 +1,100 @@ +{ + "services": { + "main": { + "volumes": [ + { + "type": "bind", + "source": "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf", + "target": "/etc/apt/apt.conf.d/99nanopy-index-cache", + "read_only": true + } + ], + "environment": { + "UV_INSTALLER_GITHUB_BASE_URL": "http://127.0.0.1:18743", + "UV_PYTHON_INSTALL_MIRROR": "http://127.0.0.1:18743/astral-sh/python-build-standalone/releases/download" + } + }, + "apt-index-cache": { + "image": "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "network_mode": "service:main", + "entrypoint": [ + "python3", + "/proxy/apt_index_proxy.py", + "--bind", + "127.0.0.1", + "--port", + "18742", + "--cache", + "/cache", + "--workers", + "16" + ], + "user": "1000:1000", + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "mem_limit": "256m", + "cpus": 0.5, + "volumes": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/scripts/apt_index_proxy.py:/proxy/apt_index_proxy.py:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/scripts/cache_image.py:/proxy/cache_image.py:ro", + "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/apt-indexes:/cache" + ], + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + "import socket; socket.create_connection((\"127.0.0.1\",18742), timeout=2).close()" + ], + "interval": "1s", + "timeout": "3s", + "retries": 30 + } + }, + "uv-release-cache": { + "image": "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "network_mode": "service:main", + "entrypoint": [ + "python3", + "/proxy/uv_release_cache.py", + "serve", + "--cache", + "/cache", + "--manifest", + "/manifest.json" + ], + "user": "1000:1000", + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cpus": 0.5, + "mem_limit": "256m", + "volumes": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/scripts/uv_release_cache.py:/proxy/uv_release_cache.py:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/scripts/cache_image.py:/proxy/cache_image.py:ro", + "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/uv-releases:/cache:ro", + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json:/manifest.json:ro" + ], + "healthcheck": { + "test": [ + "CMD", + "python3", + "-c", + "import socket; socket.create_connection((\"127.0.0.1\",18743), timeout=2).close()" + ], + "interval": "1s", + "timeout": "3s", + "retries": 30 + } + } + } +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json new file mode 100644 index 0000000..e762efb --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json @@ -0,0 +1,58 @@ +{ + "registered_at": "2026-10-18T13:25:43.131474+00:00", + "metadata_source": "Original release metadata preserved by the source manifests; no versions or payloads substituted.", + "source_manifests": { + "benchmarks/harbor/configs/dev34-main-baseline-20261012/dependency-release-manifest-v5.json": "bdf3d3ef8b7743700c5294c4a511c686a3928c178a77fd4abcba90ee4557ffcc", + "python-agent-release-manifest.json": "78aecd7c397c469196f9d246a160bf1fc99ad2d51cb71320b7d26ebf839647ce" + }, + "assets": [ + { + "version": "0.9.11", + "filename": "uv-x86_64-unknown-linux-gnu.tar.gz", + "url": "https://github.com/astral-sh/uv/releases/download/0.9.11/uv-x86_64-unknown-linux-gnu.tar.gz", + "digest": "sha256:817c0722b437b4b45b9a7e0231616a09db76bab1b8d178ba7a9680c690db19f0", + "size": 21699161, + "asset_id": 318993196, + "release_metadata_sha256": "c1a1378ece65ce1aa8343e57bdbd305289a8fc6d3d9180e8bb6b0f69acd91413" + }, + { + "version": "0.9.5", + "filename": "uv-x86_64-unknown-linux-gnu.tar.gz", + "url": "https://github.com/astral-sh/uv/releases/download/0.9.5/uv-x86_64-unknown-linux-gnu.tar.gz", + "digest": "sha256:2cf10babba653310606f8b49876cfb679928669e7ddaa1fb41fb00ce73e64f66", + "size": 21370871, + "asset_id": 306970260, + "release_metadata_sha256": "1921b17d3370aafb690b29d1d121ee8719ef3d3dbc0c7cf3b948c3059a927a5d" + }, + { + "version": "20251014", + "filename": "cpython-3.13.9+20251014-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251014/cpython-3.13.9%2B20251014-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:c0ccda275948c79996e46993c2c5476ff5cca606dee530f1dea712179131b348", + "size": 33517986, + "asset_id": 304305340, + "release_metadata_sha256": "8e5db21cfcc2bcafcb2127f269f00cbfdd1f74842a685acd2a14e3c1ad6f8394", + "repository": "astral-sh/python-build-standalone" + }, + { + "version": "20251120", + "filename": "cpython-3.14.0+20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251120/cpython-3.14.0%2B20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:66f385305ae0eefd6b65e2cf942bc91d943a61e26fb7581e029f760a2b04f393", + "size": 35376131, + "asset_id": 318881850, + "release_metadata_sha256": "e3a737862d2dddf11c73bfef6c2f2579e9d4ce5bd271cca53790c998590dee58", + "repository": "astral-sh/python-build-standalone" + }, + { + "version": "20251120", + "filename": "cpython-3.13.9+20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251120/cpython-3.13.9%2B20251120-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "digest": "sha256:1bc9a94ab2425fb6ae58e45f69cab61647759b57895649701390822d976fa923", + "size": 34257108, + "asset_id": 318881608, + "release_metadata_sha256": "705690ab2fd466e90b4c868635c529f3c7fd0cb597d399e350f399e3f3e35e6c", + "repository": "astral-sh/python-build-standalone" + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json new file mode 100644 index 0000000..c89d0fe --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json @@ -0,0 +1,209 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-20T13:46:55.453937+00:00", + "experiment": "dev34-flash1017-recovery-20261020", + "separate_from": "dev34-flash1017-main-20261019", + "original_baseline_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "authorization": "The user explicitly selected three fresh agent trials, one new solution per task, retained workspaces, at most three verifier attempts, and separate reporting from the original baseline.", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-20T13:42:17.189959+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "recovery": { + "source_root": "jobs/runtime-source", + "source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385, + "core_agent_matches_ref": "ce28b1e826a292d796ba64f13e79199b66f65758" + } + }, + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-recovery:20260430", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e080308a05703b40edd16f53aea41a0e6b9874750ecee35bb432029d0d44bf53", + "environment/Dockerfile": "8a6b7edbb9e7c241e447f15a71e13e5d043b25169be2fd9802b84f5be3d56eb6", + "environment/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "environment/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "instruction.md": "2bef07dc28be3ab2c35ae389584034c2e4dde92c2c31b3eb6bd829628c8c3f4a", + "solution/solve.sh": "175a7802777b43e0fa4f629d71f7c68fb40706813c342aaddc72108d89bd5b2c", + "task.toml": "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd", + "tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + } + } + ], + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "total_model_runs": 3, + "verifier_policy": { + "max_attempts": 3, + "timeout_seconds": 900, + "retry_dependency_errors": true, + "retry_collection_errors": true, + "stop_on_valid_score": true, + "same_container_and_output": true, + "test_scripts_unchanged": true + }, + "retention": { + "environment_delete": false, + "pre_verifier_artifacts": [ + "/app", + "/data for merger only" + ], + "raw_evidence": "Git-ignored jobs/; solution files, stopped containers, every verifier attempt, and input snapshots retained.", + "docker_keep_containers": true + }, + "preflight": [], + "queue": [ + { + "arm": "recovery", + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger-retained.json", + "config_sha256": "de9d8b9378950822acbc361cb8c8997f84ac12ae0d5a8de4d80d4e953849d32e", + "job_name": "dev3-1017-multi-source-data-merger-recovery-20261020" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli-retained.json", + "config_sha256": "d0f0fa76cb9575e338b2ce13c257f7e5f766218e41fba459b8903e4f69f8da93", + "job_name": "dev3-1017-pytorch-model-cli-recovery-20261020" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery-retained.json", + "config_sha256": "eafe5060a5b932f4523b0897353afebfcc82baf2e8560bb073e5d319604f22bb", + "job_name": "dev3-1017-pytorch-model-recovery-recovery-20261020" + } + ], + "log_dir": "jobs/dev3-1017-recovery-20261020-v2-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json": "310dfa6b358b8f2fa5144d4ac1b64822810f68f8b655335aa7d3f8fe2e39bfe4", + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/image-digests.json": "4d3e55e813dd5e0e32ec4bdfd7d361bf9171ee5d70cd12c7cddce33e7c2b3db2", + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/original-baseline-evidence.json": "67f77b67645c8be3d2ee9e56dd7a12475f7c03122e7b96715e0c906e60eb3bda" + }, + "review_policy": "Freeze all three new model outputs before examining their trajectories, artifacts, or official test contents. Monitor only phase metadata, infrastructure logs, and host resources while running.", + "interpretation": "New independently generated solutions with authorized recovery; these cannot regrade the deleted original workspaces or overwrite original baseline results. Tasks have prior development exposure; this is not a held-out evaluation.", + "supersedes": "experiment.json", + "amendment": "Before any model call, select configurations with Docker keep_containers=true; delete=false alone removes Docker containers. Accept the three successful native installation-only records by hash. Preserve the initial registration and all original configs.", + "accepted_preflight_trials": [ + { + "arm": "recovery", + "task": "terminal-bench/multi-source-data-merger", + "result_path": "jobs/runtime-source/jobs/dev3-1017-install-only-recovery-20261020/multi-source-data-merger__8HoSY39/result.json", + "result_sha256": "d9dffaaa8f4c470a7c71171d84b758e9a016e50aa929064fbf84b0c043628926", + "bootstrap_sha256": "33254eef8f18b233ce1017454a168303e97c29fc41d02a971740ec588a03302e", + "manifest_sha256": "5e02416f7b0dbc0548dbf72c93c3356300726c280d7667bfbc1e205c80bc9ba2", + "input_config_sha256": "c3aab71fe88ba6a3bba618c56934acbc43d6aa3d34027a1ff3d1f69d0523cc91" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-cli", + "result_path": "jobs/runtime-source/jobs/dev3-1017-install-only-recovery-20261020/pytorch-model-cli__7S8845w/result.json", + "result_sha256": "a1129937e99f5816fa94d27f1bd220111fe86b74a32d566a832fd5adc4210dca", + "bootstrap_sha256": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "manifest_sha256": "5e02416f7b0dbc0548dbf72c93c3356300726c280d7667bfbc1e205c80bc9ba2", + "input_config_sha256": "c3aab71fe88ba6a3bba618c56934acbc43d6aa3d34027a1ff3d1f69d0523cc91" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-recovery", + "result_path": "jobs/runtime-source/jobs/dev3-1017-install-only-recovery-20261020/pytorch-model-recovery__8JmHZuF/result.json", + "result_sha256": "11fadb581e74540d384ad8758192e2787f0163783fc99c91f82761d86ba4c300", + "bootstrap_sha256": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "manifest_sha256": "5e02416f7b0dbc0548dbf72c93c3356300726c280d7667bfbc1e205c80bc9ba2", + "input_config_sha256": "c3aab71fe88ba6a3bba618c56934acbc43d6aa3d34027a1ff3d1f69d0523cc91" + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment.json new file mode 100644 index 0000000..bfa14e7 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment.json @@ -0,0 +1,182 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-20T13:42:17.189951+00:00", + "experiment": "dev34-flash1017-recovery-20261020", + "separate_from": "dev34-flash1017-main-20261019", + "original_baseline_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "authorization": "The user explicitly selected three fresh agent trials, one new solution per task, retained workspaces, at most three verifier attempts, and separate reporting from the original baseline.", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-20T13:42:17.189959+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "arms": { + "recovery": { + "source_root": "jobs/runtime-source", + "source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385, + "core_agent_matches_ref": "ce28b1e826a292d796ba64f13e79199b66f65758" + } + }, + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "data-processing", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/pytorch-model-recovery:20260430", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "category": "model-training", + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "e080308a05703b40edd16f53aea41a0e6b9874750ecee35bb432029d0d44bf53", + "environment/Dockerfile": "8a6b7edbb9e7c241e447f15a71e13e5d043b25169be2fd9802b84f5be3d56eb6", + "environment/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "environment/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "instruction.md": "2bef07dc28be3ab2c35ae389584034c2e4dde92c2c31b3eb6bd829628c8c3f4a", + "solution/solve.sh": "175a7802777b43e0fa4f629d71f7c68fb40706813c342aaddc72108d89bd5b2c", + "task.toml": "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd", + "tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + } + } + ], + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "total_model_runs": 3, + "verifier_policy": { + "max_attempts": 3, + "timeout_seconds": 900, + "retry_dependency_errors": true, + "retry_collection_errors": true, + "stop_on_valid_score": true, + "same_container_and_output": true, + "test_scripts_unchanged": true + }, + "retention": { + "environment_delete": false, + "pre_verifier_artifacts": [ + "/app", + "/data for merger only" + ], + "raw_evidence": "Git-ignored jobs/; solution files, stopped containers, every verifier attempt, and input snapshots retained." + }, + "preflight": [ + { + "arm": "recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/install-only.json", + "config_sha256": "cf48a0614d74c2af65064071d45ee0be866157b22c332634793e07cf3b75c909", + "job_name": "dev3-1017-install-only-recovery-20261020" + } + ], + "queue": [ + { + "arm": "recovery", + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger.json", + "config_sha256": "114e9fddd1072c38a730cb9c60200db17b9470938a2acc52436512e9c631402d", + "job_name": "dev3-1017-multi-source-data-merger-recovery-20261020" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli.json", + "config_sha256": "7f734468d91f253547a47cc15f7876d49176bc3efac8a29c79cf43d1c8e8356d", + "job_name": "dev3-1017-pytorch-model-cli-recovery-20261020" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-recovery", + "config": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery.json", + "config_sha256": "d2768117c84a90260d88b2e443b88fcec70b8bdce778e7ffd64ae51bc53b29bb", + "job_name": "dev3-1017-pytorch-model-recovery-recovery-20261020" + } + ], + "log_dir": "jobs/dev3-1017-recovery-20261020-orchestration", + "additional_file_sha256": { + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json": "310dfa6b358b8f2fa5144d4ac1b64822810f68f8b655335aa7d3f8fe2e39bfe4", + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/apt-cache.conf": "d0e1ffb6dbd02c54978ba2df386326311a59f6d3774990180f22bcc6adfaedbc", + "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-release-manifest.json": "77dfda40e45b870a8ce7f07a1e058d416d94a04ab3d27c2f21bc370f22151394", + "benchmarks/harbor/scripts/apt_index_proxy.py": "bc15f7adf1d7968d627ceca4683ab86742a7188a1b1955ba1fef37ba48e8a936", + "benchmarks/harbor/scripts/uv_release_cache.py": "993cc35b1f3a9e8e171b662ab6d4524c5fe059f0d1f0e275a5bb1930111e0ff5", + "benchmarks/harbor/scripts/cache_image.py": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0" + }, + "review_policy": "Freeze all three new model outputs before examining their trajectories, artifacts, or official test contents. Monitor only phase metadata, infrastructure logs, and host resources while running.", + "interpretation": "New independently generated solutions with authorized recovery; these cannot regrade the deleted original workspaces or overwrite original baseline results. Tasks have prior development exposure; this is not a held-out evaluation." +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/image-digests.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/image-digests.json new file mode 100644 index 0000000..21f3514 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/image-digests.json @@ -0,0 +1,30 @@ +{ + "checked_at": "2026-10-20T13:45:35.955916+00:00", + "original_identities_match": true, + "images": [ + { + "task": "terminal-bench/multi-source-data-merger", + "image": "alexgshaw/multi-source-data-merger:20251031", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "repo_digests": [ + "alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e" + ] + }, + { + "task": "terminal-bench/pytorch-model-cli", + "image": "alexgshaw/pytorch-model-cli:20251031", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "repo_digests": [ + "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67" + ] + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "image": "alexgshaw/pytorch-model-recovery:20260430", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "repo_digests": [ + "alexgshaw/pytorch-model-recovery@sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e" + ] + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/install-only.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/install-only.json new file mode 100644 index 0000000..0c1d32a --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/install-only.json @@ -0,0 +1,46 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + }, + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": true + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/main-prompt.txt b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/main-prompt.txt new file mode 100644 index 0000000..56116f5 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/main-prompt.txt @@ -0,0 +1,9 @@ +You are nanoPyCodeAgent, a coding agent running non-interactively on a task handed to you up front. There is no user to reply to you: never ask a clarifying question, never stop to wait for confirmation, and never present a plan for approval — decide on your own and carry it out. Work the task through to the end, then check the result with the tools instead of assuming it worked. When it is done, answer with a short summary and no further tool calls: that reply is what ends the run. Runtime budget reminders report remaining time and model replies; when either is running low, prioritize saving the required deliverables, necessary verification, and a final summary. Do not start optional work after the requirements are satisfied. + +Identify the required deliverables, interfaces, and constraints. Treat examples as evidence, not as the full contract unless the task says so. Inspect relevant files, dependencies, and existing checks instead of guessing. Task-supplied workspace files, including source, executables, and tests, are available references unless the task restricts their use. Consult general documentation as needed. Do not seek or use task-specific reference solutions or hidden evaluation tests from outside the supplied materials unless the task explicitly requests them. Derive checks from the stated requirements. + +Verify the requested behavior through the real entry point when feasible. Check the content and meaning of outputs, not only command success or file existence. For logic or numerical work, choose a few representative and boundary cases justified by the task, using an independent reference, invariant, or consistency check when available. Do not derive every expectation from the implementation being tested or weaken requirements to make a check pass. Keep verification proportional to the task and remaining budget. + +When a check fails, inspect the error and relevant output, revise the underlying assumption or implementation, and rerun the affected check. Keep the task's constraints intact. Once appropriate checks support the required behavior, save the deliverables and finish. State what was verified and any remaining limitation without claiming unobserved success. + +Prefer the read tool for viewing files, the edit tool for changing part of an existing file, and the write tool for creating files or rewriting them whole. Use the bash tool to run commands, search with grep, and complete tasks that need real command output instead of guessing. \ No newline at end of file diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger-retained.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger-retained.json new file mode 100644 index 0000000..984f2ab --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger-retained.json @@ -0,0 +1,44 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": false, + "kwargs": { + "keep_containers": true + } + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [ + "/app", + "/data" + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger.json new file mode 100644 index 0000000..f26e679 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/multi-source-data-merger.json @@ -0,0 +1,41 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": false + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [ + "/app", + "/data" + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/original-baseline-evidence.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/original-baseline-evidence.json new file mode 100644 index 0000000..3657de8 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/original-baseline-evidence.json @@ -0,0 +1,177 @@ +{ + "checked_at": "2026-10-20T13:44:52.930641+00:00", + "original_baseline_path": "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/results/dev34-flash1017-main-20261019.json", + "original_baseline_sha256": "b107d3a48b7205bd092ceed12f5e6cf3ccdf8b65ffd7cddddeacb73ee0b9a0a5", + "original_baseline_publication_ref": "1ae6e05", + "original_aggregate": { + "tasks": 34, + "classifications": { + "passed": 26, + "scored_failure": 5, + "verifier_error": 3 + }, + "passed_fraction_of_pool": 0.7647058823529411, + "official_pytest_counts": { + "passed": 133, + "failed": 8, + "skipped": 0, + "errors": 1, + "xfailed": 0, + "xpassed": 0 + }, + "verifier_attempts": 37, + "verifier_retries": 3, + "wall_seconds": 15853.918955, + "estimated_known_cost_usd": 1.860998832, + "usage_incomplete_tasks": [], + "cost_incomplete_tasks": [], + "recorded_tokens": { + "prompt_tokens": 47814612, + "completion_tokens": 1153308, + "cached_tokens": 47167872 + }, + "assertion_cases": 141, + "collection_errors": 1, + "agent_trials_started": 34, + "host_launch_errors_before_model": 3 + }, + "workspace_inspection": "Docker reported no containers and no volumes before starting this supplemental experiment. Original artifact manifests contain empty /logs/artifacts directories only.", + "policy": "Preserve original baseline and native evidence unchanged. New solutions are separate trials, not original-output verifier retries.", + "trials": [ + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__ovUnaUM", + "original_classification": "verifier_error", + "original_raw_reward": 0.0, + "original_analytical_score": null, + "original_verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 327.15425308000704 + } + ], + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/result.json": "bd84053547e596ad0f3a8285a80da37e3a22f0eae72f9fe831fea2d526888583", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/config.json": "32879386a7e47a3832e34f78fc25b31375362c8a98f7e94c2f97a56288ab0fda", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/bootstrap.json": "5011ac5d6ab074bd66636cc77aae3a2d966293bcb051b929e7a9cb076f908948", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/trajectory.json": "652ff7db3dd1b8f75a080cb68a12e9a7bbcb73b9ae245b4d52485487e901a424", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/nanopycodeagent.txt": "cb5d1b6d5480a1e87a6236a04545f52a35296ede77de27c7fe9770edbfcc4acd", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/test-stdout.txt": "7690d05931b29fda94b3c858291f9a35e557ebbe826c715725fda790311dba46", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/retry-summary.json": "f637a457f9341eefc7e30edd19559bc31c0bca312791506f4320847b38cd5c53", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/manifest.json": "7fda413bc1c8c6a57d8ab89a72c7f81a9b16f7fdc0140b03cd9c9696584be5ae", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/config.json": "175d9eea47f437346fc37f27016c08a2fb591e4cd345044b6254f6e13ab7c701", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-multi-source-data-merger-main-20261019.json": "1fe2a8510681dc681b0d2145503b4bfb72b60fe667e6fd4c0fcd098324b3aa7e", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/attempts/1/test-stdout.txt": "7690d05931b29fda94b3c858291f9a35e557ebbe826c715725fda790311dba46", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "artifact_manifest_sha256": "c1db653f487fb773c4f9247162ab3314e7f0584a486914b9874c4b3d6949e42d", + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + } + ], + "original_workspace_available": false + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__t73XoLw", + "original_classification": "verifier_error", + "original_raw_reward": 0.0, + "original_analytical_score": null, + "original_verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 133.28830967300746 + } + ], + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/result.json": "4847cac28aefa9154528ed40833214b4401c0065e2eee64aeeb8bdec6ec0fbb1", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/config.json": "b80389d25c6b68223471bb82b4ca1c5f47bedd16e6de6c7ad08887957e8ab398", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/trajectory.json": "9baf56afbccfcb6f39ab2df5c5c7f21b7684783d5f32fee2693897f337bfe446", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/nanopycodeagent.txt": "c689bf519d6c856396aa49c03dfb0a1bb65ad9dd46eb905372df6f595ca00872", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/ctrf.json": "0383370176312a002520c110ff53a75664836ed3d627225fd93b76ccd89f9166", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/test-stdout.txt": "dad670fa7c0265c95408dc5f33e01b4dc9d164574ab5e6625b75ecb289bc306f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/retry-summary.json": "9d5e42ed1f72ed1b2bbe1de2724041d8f06eec6bc2e79cad5b52b49131d0ed5b", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/manifest.json": "84502d6134216bcf9b867ed4a206b237fd14dc7a204b226888e75a64fc11ce97", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/config.json": "1b9e5409dc8e2c45449188a96cd3b0399cd99a60900dda4569273848f19ee407", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-cli-main-20261019.json": "98e3548ee417240c814b98bb07b6dd53a772e83e30e6f555f32565c7dc18ebe1", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/test-stdout.txt": "dad670fa7c0265c95408dc5f33e01b4dc9d164574ab5e6625b75ecb289bc306f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/ctrf.json": "0383370176312a002520c110ff53a75664836ed3d627225fd93b76ccd89f9166", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "artifact_manifest_sha256": "c1db653f487fb773c4f9247162ab3314e7f0584a486914b9874c4b3d6949e42d", + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + } + ], + "original_workspace_available": false + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "trial": "pytorch-model-recovery__hogG9gA", + "original_classification": "verifier_error", + "original_raw_reward": null, + "original_analytical_score": null, + "original_verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0023410760041 + }, + { + "attempt": 2, + "status": "timeout", + "duration_seconds": 900.0023236230045 + }, + { + "attempt": 3, + "status": "timeout", + "duration_seconds": 900.00202244299 + } + ], + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/result.json": "8cd798fd024cae52b15271b61516416a3697d9bb4e834a4f03e46878ea9df759", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/config.json": "fedb449dfc06e5ee34620f44b69539990d5753834b8c0de95ac9a355a8146fb3", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/trajectory.json": "7e4230a06c24c1751916117887130adcfc70429a5f4995669df5fc4afc479b4f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/nanopycodeagent.txt": "5dee8328da08df739c2285b303ffc9349c305d1b803232d82d68e56d3a6a6925", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/test-stdout.txt": "d20478d9b2f2a8527862c484b10fd0140c8f86bfa1c587967131dd9c7e998c8d", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/retry-summary.json": "386028e2e0fa4b477bd4342345e31bd3cdf2377d60cad33e5218d09777e0f43a", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/manifest.json": "992ab17d7c24c8e82ee153e98c5aee1805a191faa57bfab0a5693481ee44cb6f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/config.json": "4f6cd8791bdb04ff12ed8d1ae1786a1f947298490305e509c0badc0fb02feaaf", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-recovery-main-20261019.json": "b71e4cca26a8670fcc6294e6744ac49a6d7cbb56661253122d15df8bbbde9da8", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/1/test-stdout.txt": "90a456f12a085a8bf4fc5c23c00a01426f771f06a0d62623cf805cf5e57e0c64", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/2/test-stdout.txt": "9cff11e64d1ca45fcd585def9798cb2aa91d8908fc88350f24af3a770b909927", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/3/test-stdout.txt": "d20478d9b2f2a8527862c484b10fd0140c8f86bfa1c587967131dd9c7e998c8d" + }, + "artifact_manifest_sha256": "c1db653f487fb773c4f9247162ab3314e7f0584a486914b9874c4b3d6949e42d", + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + } + ], + "original_workspace_available": false + } + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli-retained.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli-retained.json new file mode 100644 index 0000000..0e9961e --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli-retained.json @@ -0,0 +1,43 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": false, + "kwargs": { + "keep_containers": true + } + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [ + "/app" + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli.json new file mode 100644 index 0000000..defee02 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-cli.json @@ -0,0 +1,40 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": false + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [ + "/app" + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery-retained.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery-retained.json new file mode 100644 index 0000000..c5b966b --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery-retained.json @@ -0,0 +1,43 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": false, + "kwargs": { + "keep_containers": true + } + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [ + "/app" + ] +} diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery.json b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery.json new file mode 100644 index 0000000..d9708e0 --- /dev/null +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/pytorch-model-recovery.json @@ -0,0 +1,40 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-recovery", + "ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" + } + ], + "environment": { + "type": "docker", + "extra_docker_compose": [ + "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-recovery-20261020/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/dependency-cache.compose.json" + ], + "delete": false + }, + "verifier": { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": { + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + } + }, + "artifacts": [ + "/app" + ] +} From 80406d610ca7c08976d48185d478edf7ab93cc5e Mon Sep 17 00:00:00 2001 From: minixalpha Date: Tue, 20 Oct 2026 08:14:48 -0700 Subject: [PATCH 08/12] chore(benchmarks): publish flash 1017 recovery results --- .../README.md | 2 + .../en/dev34-flash1017-recovery-20261020.md | 47 + .../dev34-flash1017-recovery-20261020.md | 47 + .../dev34-flash1017-recovery-20261020.json | 1800 +++++++++++++++++ 4 files changed, 1896 insertions(+) create mode 100644 benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md create mode 100644 benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md create mode 100644 benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json diff --git a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md index 5fc651c..b0ee30e 100644 --- a/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md +++ b/benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/README.md @@ -1,5 +1,7 @@ # Flash 1017 verifier-recovery supplement +**Completed:** one pass, one scored failure, and one unscored verifier error across three fresh agent trials. The merger passed all three tests in one verifier attempt. The CLI timed out once, then completed with five passing tests and one failing test (10 of 50 predictions wrong); valid zero stopped recovery after two attempts. Model recovery exhausted three 900-second verifier attempts downloading PyTorch/CUDA dependencies before pytest. Original baseline results remain unchanged. See the [English report](../../reports/en/dev34-flash1017-recovery-20261020.md), [complete Simplified Chinese report](../../reports/zh-CN/dev34-flash1017-recovery-20261020.md), and [audited results](../../results/dev34-flash1017-recovery-20261020.json). All three main containers and pre-verifier solution archives are retained; all 50 temporary image-cache relocations were restored and SHA256-verified. + This is a separately authorized three-task experiment, not a regrade or replacement of the October 19 dev34 baseline. The original containers were deleted and no `/app` solution artifacts were archived. The user explicitly diff --git a/benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md b/benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md new file mode 100644 index 0000000..f9d08ba --- /dev/null +++ b/benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md @@ -0,0 +1,47 @@ +# Flash 1017: three-task verifier-recovery supplement + +This separately authorized supplement finished with **1 pass, 1 scored failure, and 1 unscored verifier error** across three fresh agent trials. The [original dev34 baseline](dev34-flash1017-main-20261019.md) remains **26 passes, five scored failures, and three unscored verifier errors**. The original containers had been deleted and no `/app` solutions were archived, so these new solutions cannot regrade the original outputs or fill their missing scores. The user explicitly chose one newly generated solution per task, at most three verifier attempts, retained workspaces, and separate reporting. + +[Machine-readable results](../../results/dev34-flash1017-recovery-20261020.json) contain native rewards, analytical classifications, all attempt records, accounting, artifact hashes, and original-baseline integrity evidence. The [registration](../../configs/dev34-flash1017-recovery-20261020/README.md) explains the immutable plans and preflight history. [Complete Simplified Chinese version](../zh-CN/dev34-flash1017-recovery-20261020.md). + +The core agent and 385-word prompt match baseline main `ce28b1e826a292d796ba64f13e79199b66f65758` byte for byte. The frozen runtime is `40fdd041c370a96dc06f86b436eacc01ce1566d7`; its verifier harness adds explicitly selected pytest collection-error recovery. The model is `deepseek/deepseek-flash` at `https://api.deepseek.com/anthropic`. [Official documentation](https://api-docs.deepseek.com/quick_start/agent_integrations/) checked on October 20 maps this mutable alias to DeepSeek-V4.1-Flash-1017; it does not pin an immutable checkpoint. Temperature and reasoning effort are unset. Each task retains 100 replies, 65,536 output tokens per reply, a 720-second work budget, 1,800-second setup limit, original images/resources/package sources/tests, and a 900-second verifier-attempt limit. Active plan SHA256: `a6c1a9433aafcdb9ce4d834b99d2c91aab2148488626b8b7fb31857f95a4b481`. + +All three fresh installation-only checks succeeded. Each task explicitly reports verifier dependency profile `not_configured`; installation success does not certify verifier dependencies. The initial plan performed only these checks. Before any model call, v2 selected `environment.kwargs.keep_containers=true` as well as `environment.delete=false`, because the latter alone still removes Docker containers in Harbor 0.21. The original plan and configs remain intact, and v2 binds the successful native preflights by hash. + +The existing comparison orchestrator used the standard `uv run --project benchmarks/harbor python -m harbor_adapter run --config ...` entry point. It ran merger and CLI as the first pair, then model recovery; observed maximum concurrency was 2. Each agent ran once. Timeouts, recognized uv network failures before pytest, and explicitly recognized aborted pytest collection shared a maximum of three attempts in the same container with the same agent output. Both `retry_dependency_errors` and `retry_collection_errors` were enabled for this authorized supplement. Any valid score, including zero, stopped immediately. Verifier filesystem changes and dependency caches were retained between attempts. There was no solution repair or test feedback to a model. + +The table gives completed official pytest counts as **passed / failed / errors / skipped**. A dash means no completed assertion result; `null` means unscored. Raw native rewards and all archived attempt outputs remain in the results and local evidence. + +| Task | Outcome | Analytical score | Pytest P/F/E/S | Verifier attempts | +| --- | --- | --- | --- | --- | +| `multi-source-data-merger` | pass | 1.0 | 3 / 0 / 0 / 0 | 1: completed (48.73s) | +| `pytorch-model-cli` | scored failure | 0.0 | 5 / 1 / 0 / 0 | 1: timeout (900.00s); 2: completed (492.35s) | +| `pytorch-model-recovery` | unscored verifier error | null | — | 1: timeout (900.00s); 2: timeout (900.00s); 3: timeout (900.00s) | + +**`multi-source-data-merger`:** The official verifier executed all three checks and passed. One verifier attempt; the new merged values and conflict report satisfy the official assertions. + +**`pytorch-model-cli`:** The first verifier attempt timed out after 900 seconds during dependency installation, before pytest. The second completed with five passed checks and one failed check: 10 of the 50 checked images produced incorrect predictions. The submitted cli_tool is an ELF executable (5,877,872 bytes). Its C++ source applies (pixel/255 - 0.1307)/0.3081, whereas the official dataset pipeline uses ToTensor() without that extra normalization. This is an observed preprocessing mismatch; no patched solution was tested. A valid scored zero stopped recovery after two attempts. + +**`pytorch-model-recovery`:** The agent reconstructs an encoder/decoder, tunes only output_layer with ridge regression, and exports a TorchScript model. The task does not require a particular optimization method. The submitted source and tool calls show no hidden-test introspection; final verifier status is collected separately. All three verifier attempts exhausted their original 900-second limits while downloading PyTorch/CUDA dependencies before pytest started. The native VerifierTimeoutError and null reward are retained. No official assertions executed, so correctness of this new TorchScript output remains unverified. + +The supplement used **6 verifier attempts**, including **3 retries**. Model-phase wall time was **82.90 minutes**, excluding installation-only preflight. Recorded input/output/cache tokens were **1,168,608 / 47,844 / 1,143,168**. Usage and cost records are complete for **3/3** and **3/3** trials respectively. The adapter-estimated total cost was **USD 0.071903808**; this is an estimate rather than billing reconciliation. + +All three agent executions and pre-verifier artifact collections were frozen at **2026-10-20T14:27:02.680316+00:00**, before reviewing any new trajectory, artifact content, or official test output. The final verifier continued after that freeze. The audit covered three ATIF trajectories, three agent stdout logs, all 68 tool-call arguments, and all 40 frozen files for exact configured credential matches (zero). No task-specific reference-solution retrieval or hidden-verifier content read was observed. The CLI used ordinary compiler/package tooling; model recovery used task-provided weights/data and output-layer ridge regression, which the task does not prohibit. These observations do not prove absence of prior exposure or training contamination. These are development tasks, independently regenerated solutions, and an altered recovery policy; no checkpoint-only or held-out conclusion follows. + +Harbor collected `/app` for every task and `/data` for the merger before verification. Every original main container below is retained and stopped; all three image identities still match the baseline, and no main container was OOM-killed. Full file hashes, symlink targets, manifests, source/wheel fingerprints and native evidence hashes are recorded in the JSON. Raw logs and solution files remain in Git-ignored `jobs/`. + +| Task | Retained main container | Container ID | Original image ID | +| --- | --- | --- | --- | +| `multi-source-data-merger` | `multi-source-data-merger__jnm6pzd__env-main-1` | `e16fca632789` | `sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e` | +| `pytorch-model-cli` | `pytorch-model-cli__p8vpnwh__env-main-1` | `5906632d1db2` | `sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67` | +| `pytorch-model-recovery` | `pytorch-model-recovery__ncs7y4h__env-main-1` | `e8f4a6bb804b` | `sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e` | + +| Trial | Task revision | Native result SHA256 | ATIF SHA256 | +| --- | --- | --- | --- | +| `multi-source-data-merger__jNM6pZd` | `sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49` | `a7b0dd5432c9c2039c843f48248ddb269ad759173031736da46ba96e6b7d7b79` | `b8d4e011e14fda92785c457dbb1dacfeec4f8bf70f180b39e65128a51c1429f6` | +| `pytorch-model-cli__P8vPNwH` | `sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79` | `1c4d0b6360600073b5dedd1ca089f4f8cff37819935549362f02ff06b1bd0056` | `f05243db52fdae75e29108803ba60fdc2bca54c9de9c997d6b1ff08789f080fa` | +| `pytorch-model-recovery__nCs7Y4H` | `sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc` | `2a61cc964ad2a684e2b4adb3e5d333f75093db51320f7e448a2cb83446998e34` | `73b4f374dbfb89dd3d3b94d572136c2f613bc1666aad87b8a6454c3316bb3f3d` | + +During verifier dependency installation, root free space fell to 2.99 GiB. Six duplicate OCI archives and 44 verified image-cache blobs (1,576,934,879 bytes) were temporarily relocated to memory-backed `/tmp`, preserving original paths as symlinks under the cache's single-writer lock. All 50 regular files were restored and SHA256-verified before publication. Original Docker images, solution artifacts, dependency caches, package sources and verifier scripts were unchanged by this host maintenance. + +Validation: 190 local Harbor adapter tests passed, including timeout bounds, opt-in collection/download recovery, valid-score stopping and log preservation. Frozen source/prompt/task/config/wheel/image checks passed; original baseline results and the three original trials' recorded evidence hashes remain unchanged. Both full report versions were generated together from the same result data and checked for matching outcomes, counts, identifiers, hashes, caveats and links. diff --git a/benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md b/benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md new file mode 100644 index 0000000..d5b006c --- /dev/null +++ b/benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md @@ -0,0 +1,47 @@ +# Flash 1017：三题 verifier 恢复补充实验 + +本次单独授权的补充实验完成了 3 个全新 agent 试次，结果为 **1 题通过、1 题评分失败、1 题 verifier 错误且未评分**。[原 dev34 baseline](dev34-flash1017-main-20261019.md) 仍为 **26 题通过、5 题评分失败、3 题 verifier 错误且未评分**。原容器已删除，且未归档 `/app` 解答，因此这些新解答无法重新评估原产物，也不能填补原试次缺失的评分。用户明确选择了每题重新生成一份解答、verifier 最多尝试 3 次、保留工作目录，并单独报告。 + +[机器可读结果](../../results/dev34-flash1017-recovery-20261020.json) 保存原生评分、分析分类、全部尝试记录、用量、产物哈希和原 baseline 完整性证据。[登记说明](../../configs/dev34-flash1017-recovery-20261020/README.md) 记录不可变计划和预检历史。[完整英文版](../en/dev34-flash1017-recovery-20261020.md)。 + +核心 agent 和 385 词提示词与 baseline main `ce28b1e826a292d796ba64f13e79199b66f65758` 逐字节一致。冻结的运行版本为 `40fdd041c370a96dc06f86b436eacc01ce1566d7`；其 verifier harness 增加了需显式选择的 pytest 收集错误恢复。模型为 `deepseek/deepseek-flash`，端点为 `https://api.deepseek.com/anthropic`。10 月 20 日核对的[官方文档](https://api-docs.deepseek.com/quick_start/agent_integrations/) 将此可变别名映射到 DeepSeek-V4.1-Flash-1017，无法固定不可变 checkpoint。temperature 和 reasoning effort 均未设置。每题保留 100 次回复、每次最多 65,536 个输出 token、720 秒工作预算、1,800 秒安装时限，以及原始镜像、资源、软件包来源、测试和单次 verifier 900 秒时限。有效计划 SHA256：`a6c1a9433aafcdb9ce4d834b99d2c91aab2148488626b8b7fb31857f95a4b481`。 + +3 个全新的仅安装检查均成功。每题的 verifier 依赖 profile 均明确记录为 `not_configured`；安装成功不能认证 verifier 依赖可用。初版计划只执行了这些检查。在任何模型调用之前，v2 同时选用了 `environment.kwargs.keep_containers=true` 和 `environment.delete=false`，因为在 Harbor 0.21 中仅设置后者仍会删除 Docker 容器。初版计划与配置保持原样，v2 通过哈希绑定成功的原生预检记录。 + +现有比较编排器调用标准入口 `uv run --project benchmarks/harbor python -m harbor_adapter run --config ...`。它先并行运行数据合并和 CLI，再运行模型恢复，观察到的最大并发为 2。每题 agent 只运行一次。超时、pytest 启动前识别到的 uv 网络错误，以及明确识别到的 pytest 收集中断，在同一容器和同一份 agent 产物上共享最多 3 次尝试。本次单独授权的补充实验显式开启了 `retry_dependency_errors` 和 `retry_collection_errors`。任何有效评分（包括 0 分）都会立即停止。各次尝试之间保留 verifier 对文件系统的修改和依赖缓存。没有修补解答，也没有向模型反馈测试结果。 + +下表的官方 pytest 完整结果按 **通过 / 失败 / 错误 / 跳过** 排列。横线表示没有完成的断言结果；`null` 表示未评分。原生评分及各次归档输出保存在结果与本地证据中。 + +| 题目 | 结果 | 分析评分 | Pytest P/F/E/S | Verifier 尝试 | +| --- | --- | --- | --- | --- | +| `multi-source-data-merger` | 通过 | 1.0 | 3 / 0 / 0 / 0 | 1: completed (48.73s) | +| `pytorch-model-cli` | 评分失败 | 0.0 | 5 / 1 / 0 / 0 | 1: timeout (900.00s); 2: completed (492.35s) | +| `pytorch-model-recovery` | verifier 错误，未评分 | null | — | 1: timeout (900.00s); 2: timeout (900.00s); 3: timeout (900.00s) | + +**`multi-source-data-merger`:** 官方 verifier 执行了全部 3 项检查并通过。只进行 1 次验证；新解答的合并值和冲突报告满足官方断言。 + +**`pytorch-model-cli`:** 第 1 次 verifier 在依赖安装阶段运行满 900 秒后超时，尚未进入 pytest。第 2 次完成了测试，5 项通过、1 项失败：检查的 50 张图中有 10 张预测错误。提交的 cli_tool 是 ELF 可执行文件（5,877,872 字节）。其 C++ 源码使用 (pixel/255 - 0.1307)/0.3081，而官方数据流程使用 ToTensor()，没有额外的均值/方差归一化。这是已观察到的预处理差异；没有测试任何修补后的解答。有效的 0 分使恢复在 2 次尝试后停止。 + +**`pytorch-model-recovery`:** agent 重建了编码器/解码器，仅通过岭回归调整 output_layer，并导出 TorchScript 模型。题目没有要求特定优化方法。提交源码和工具调用中未观察到读取隐藏测试的行为；最终 verifier 状态另行收集。 3 次 verifier 均在下载 PyTorch/CUDA 依赖时耗尽原定 900 秒时限，尚未进入 pytest。保留原生 VerifierTimeoutError 和 null 评分。没有执行官方断言，因此这份新 TorchScript 产物的正确性仍未验证。 + +本次补充实验共执行 **6 次 verifier**，其中 **3 次为重试**。模型阶段墙钟时间为 **82.90 分钟**，不含仅安装预检。记录的输入/输出/缓存 token 为 **1,168,608 / 47,844 / 1,143,168**。用量和费用记录分别有 **3/3** 和 **3/3** 个试次完整。adapter 估算总费用为 **USD 0.071903808**；这是估算，未进行账单对账。 + +3 个 agent 执行和验证前产物归档均于 **2026-10-20T14:27:02.680316+00:00** 冻结，之后才查看新轨迹、产物内容或官方测试输出。最后一个 verifier 在冻结后继续运行。审计覆盖 3 份 ATIF 轨迹、3 份 agent 标准输出、全部 68 次工具调用参数，并对全部 40 个冻结文件扫描已配置凭据的精确匹配（0 次）。未观察到获取题目专属参考解答或读取隐藏 verifier 内容的行为。CLI 使用常规编译器和软件包工具；模型恢复使用题目提供的权重/数据及输出层岭回归，题目未禁止这种方法。这些观察不能证明不存在先前接触或训练污染。这些是开发题目，解答为独立重新生成，恢复策略也有变化，因此不能得出仅由 checkpoint 变化导致的结论，也不属于留出集评测。 + +Harbor 在验证前为每题收集 `/app`，并为数据合并题收集 `/data`。下列原始主容器均已停止并保留；3 个镜像身份仍与 baseline 一致，且主容器均未被 OOM 杀死。JSON 记录完整文件哈希、符号链接目标、清单、源码/wheel 指纹和原生证据哈希。原始日志和解答文件保存在由 Git 忽略的 `jobs/` 中。 + +| 题目 | 保留的主容器 | 容器 ID | 原始镜像 ID | +| --- | --- | --- | --- | +| `multi-source-data-merger` | `multi-source-data-merger__jnm6pzd__env-main-1` | `e16fca632789` | `sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e` | +| `pytorch-model-cli` | `pytorch-model-cli__p8vpnwh__env-main-1` | `5906632d1db2` | `sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67` | +| `pytorch-model-recovery` | `pytorch-model-recovery__ncs7y4h__env-main-1` | `e8f4a6bb804b` | `sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e` | + +| 试次 | 题目版本 | 原生结果 SHA256 | ATIF SHA256 | +| --- | --- | --- | --- | +| `multi-source-data-merger__jNM6pZd` | `sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49` | `a7b0dd5432c9c2039c843f48248ddb269ad759173031736da46ba96e6b7d7b79` | `b8d4e011e14fda92785c457dbb1dacfeec4f8bf70f180b39e65128a51c1429f6` | +| `pytorch-model-cli__P8vPNwH` | `sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79` | `1c4d0b6360600073b5dedd1ca089f4f8cff37819935549362f02ff06b1bd0056` | `f05243db52fdae75e29108803ba60fdc2bca54c9de9c997d6b1ff08789f080fa` | +| `pytorch-model-recovery__nCs7Y4H` | `sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc` | `2a61cc964ad2a684e2b4adb3e5d333f75093db51320f7e448a2cb83446998e34` | `73b4f374dbfb89dd3d3b94d572136c2f613bc1666aad87b8a6454c3316bb3f3d` | + +verifier 安装依赖时，根分区可用空间降至 2.99 GiB。6 个重复 OCI 归档和 44 个已校验镜像缓存 blob（共 1,576,934,879 字节）被暂存到内存中的 `/tmp`，在持有缓存单写者锁期间以符号链接保留原路径。发布前，全部 50 个常规文件均已恢复并通过 SHA256 校验。此次主机维护没有改动 Docker 原始镜像、解答产物、依赖缓存、软件包来源或 verifier 脚本。 + +验证：190 项本地 Harbor adapter 测试通过，覆盖超时上限、显式收集/下载错误恢复、有效评分后停止和日志保留。冻结的源码/提示词/题目/配置/wheel/镜像检查通过；原 baseline 结果及原 3 个试次已记录的证据哈希保持不变。两份完整报告从同一结果数据同时生成，并核对了结果、计数、标识符、哈希、限制和链接的一致性。 diff --git a/benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json b/benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json new file mode 100644 index 0000000..550aa65 --- /dev/null +++ b/benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json @@ -0,0 +1,1800 @@ +{ + "schema_version": 1, + "experiment": "dev34-flash1017-recovery-20261020", + "collected_at": "2026-10-20T15:10:24.655866+00:00", + "cohort": "Three fresh, separately authorized recovery trials; one new model solution each; original baseline unchanged.", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "provider_checkpoint": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-20T13:42:17.189959+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "core_agent_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "runtime_source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "plan_path": "benchmarks/harbor/configs/dev34-flash1017-recovery-20261020/experiment-v2.json", + "plan_sha256": "a6c1a9433aafcdb9ce4d834b99d2c91aab2148488626b8b7fb31857f95a4b481", + "model_output_freeze": { + "recorded_at": "2026-10-20T14:27:02.680316+00:00", + "model_outputs": 3, + "policy": "All three agent executions and pre-verifier artifact collections finished before reviewing any new trajectory, artifact content or official test output. The final verifier remains active; no model reruns, solution edits or test feedback to a model are allowed.", + "trials": [ + { + "task": "terminal-bench/multi-source-data-merger", + "job": "dev3-1017-multi-source-data-merger-recovery-20261020", + "trial": "multi-source-data-merger__jNM6pZd", + "file_sha256": { + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/agent/trajectory.json": "b8d4e011e14fda92785c457dbb1dacfeec4f8bf70f180b39e65128a51c1429f6", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/agent/nanopycodeagent.txt": "d8ede0eed4bd3310ddd4e33573943d494fff33c717a9f7b67124d1211651eb54", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/config.json": "51c251981a4e2ed33dcf2287c86799de7fd945bb13085706496fb9e9e882a064", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/agent/bootstrap.json": "33254eef8f18b233ce1017454a168303e97c29fc41d02a971740ec588a03302e", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/app/conflicts.json": "b36e9898ecb5f1056780359f3cc5ab50f043f1e4e732972781d92aac78ada3ed", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/app/merge_users.py": "b0d94a6b07ab4eff9c0920cb96a37ae66e65a6c892d354411c7fdd6075f8a934", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/app/merged_users.parquet": "effad8725622325c3a3185038b610a2097d6337788bd5031435ecdee6e93b2ff", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/manifest.json": "afdc3cddfe46f408bbaea8e826dd0c1423ed4c97948d66029243a8a8c8accd71" + }, + "symlinks": {} + }, + { + "task": "terminal-bench/pytorch-model-cli", + "job": "dev3-1017-pytorch-model-cli-recovery-20261020", + "trial": "pytorch-model-cli__P8vPNwH", + "file_sha256": { + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/agent/trajectory.json": "f05243db52fdae75e29108803ba60fdc2bca54c9de9c997d6b1ff08789f080fa", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/agent/nanopycodeagent.txt": "5d4ef4d93898cf10f80442fe1f14908460553ff6fd0bac58cf1a9c54d786fc60", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/config.json": "f6040debc1dea9a8386da3bde6554edd5bfe6b6f3f08d4a4bae5bc4a6f9b649c", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/agent/bootstrap.json": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/cli_tool": "e6c80c3f6665ffc44ff2e7042b33235550c79a1ce5e56227827de104504d977b", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/cli_tool.cpp": "1f67600b6af1572e92fcb314124ee3014458cefd6f2ec54217078c8cf2cc1abe", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/cli_tool.py": "92295cdd7be35342e38da3b482db7266cfb7d27d7b64d9b317382ecf811b53e8", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/prediction.txt": "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/app/weights.json": "76b0b062f898598366154c2c4618534594e7bb0e69c60c49bcceec101cd83708", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6" + }, + "symlinks": {} + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "job": "dev3-1017-pytorch-model-recovery-recovery-20261020", + "trial": "pytorch-model-recovery__nCs7Y4H", + "file_sha256": { + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/agent/trajectory.json": "73b4f374dbfb89dd3d3b94d572136c2f613bc1666aad87b8a6454c3316bb3f3d", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/agent/nanopycodeagent.txt": "036c564ae66e993149b0a73d6de0e24292f3b06331b035c929d109e019d5025a", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/config.json": "125c1fdecd4add7009fa787adb79226e020e0f8598b12413864ac3c8f3502c83", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/agent/bootstrap.json": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/__pycache__/recovered_model.cpython-313.pyc": "c492a45a453bba7c9fc0db2bce81b3bac895ca649841b91e2919e13f4ac59071", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/model.pt": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/recovered_model.py": "602526444f3517d52c011a3291c3b492d6a86b4bd5200011b4027bc2ee86f3f7", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/tune_model.py": "e2e9e6bbdb6e13f9e7e57d28f8c4e66932bfeff8dc468df2ae2034b8b9e92e71", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/verify.py": "ed8ec4d7d52bbd849c15f8acc066825af6354fb729f265e9d223307835381c9e", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/app/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6" + }, + "symlinks": {} + } + ] + }, + "job_completion_evidence": { + "recorded_at": "2026-10-20T15:10:24.572582+00:00", + "last_job_finished_at": "2026-10-20T15:09:49.675410+00:00", + "finished_jobs": 3, + "policy": "All three new model outputs finished before review of new trajectories, artifacts or test output.", + "job_record_sha256": { + "jobs/dev3-1017-recovery-20261020-v2-orchestration/dev3-1017-multi-source-data-merger-recovery-20261020.json": "f9b1dcc365734193d2ce60bffc5db9c1e157413e9aa34b50e3e4ab300c673b9f", + "jobs/dev3-1017-recovery-20261020-v2-orchestration/dev3-1017-pytorch-model-cli-recovery-20261020.json": "4cf5114e0f080117e85bff28a796b06abd9bab5ffa6bd7eac306e86db36a07bd", + "jobs/dev3-1017-recovery-20261020-v2-orchestration/dev3-1017-pytorch-model-recovery-recovery-20261020.json": "5df563c0dfc936b5798e7e78ece5489ae105488a7fd373f1a19c902d946abc71" + } + }, + "verifier_policy": { + "max_attempts": 3, + "timeout_seconds": 900, + "retry_dependency_errors": true, + "retry_collection_errors": true, + "stop_on_valid_score": true, + "same_container_and_output": true, + "test_scripts_unchanged": true + }, + "retention": { + "environment_delete": false, + "pre_verifier_artifacts": [ + "/app", + "/data for merger only" + ], + "raw_evidence": "Git-ignored jobs/; solution files, stopped containers, every verifier attempt, and input snapshots retained.", + "docker_keep_containers": true + }, + "aggregate": { + "tasks": 3, + "classifications": { + "passed": 1, + "scored_failure": 1, + "verifier_error": 1 + }, + "verifier_attempts": 6, + "verifier_retries": 3, + "maximum_concurrent_jobs": 2, + "wall_seconds": 4974.050042, + "estimated_known_cost_usd": 0.071903808, + "recorded_tokens": { + "prompt_tokens": 1168608, + "completion_tokens": 47844, + "cached_tokens": 1143168 + }, + "completed_official_pytest_counts": { + "passed": 8, + "failed": 1, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + } + }, + "trials": [ + { + "order": 1, + "task": "terminal-bench/multi-source-data-merger", + "task_ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "job": "dev3-1017-multi-source-data-merger-recovery-20261020", + "trial": "multi-source-data-merger__jNM6pZd", + "classification": "passed", + "raw_reward": 1.0, + "analytical_score": 1.0, + "exception_type": null, + "official_pytest_counts": { + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792504127.8832994, + "stop": 1792504128.9687674 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 48.733129436004674 + } + ], + "started_at": "2026-10-20T13:46:59.559396Z", + "finished_at": "2026-10-20T13:49:10.291032Z", + "seconds": { + "environment_setup": 2.083707, + "agent_setup": 31.681899, + "agent_execution": 26.426609, + "verifier": 48.949854 + }, + "task_checksum": "33fa3b988ff60ec62b6ce40ee455208cb083ac1ac46ddc5247e954c88b9d5e8e", + "source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "7ab271344f8fda788e5562c96c7ce2d39f2db5137b629930c71e1e91fac016c9", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "86a92c078b9e6276e946aabc688e13ba267220b5d5b01c52944b7bb596ecd392" + }, + "wheel_sha256": "eb90a6fac1961222da55497a4012654f084919adae6cd25c6ab1c01d39ef202e", + "bootstrap_status": "ready", + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 27715, + "completion_tokens": 4127, + "cached_tokens": 25600 + }, + "estimated_known_cost_usd": 0.0057405, + "native_cost_matches_atif": true + }, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 6, + "tool_calls": 6, + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + }, + { + "source": "/app", + "destination": "artifacts/app", + "type": "directory", + "status": "ok", + "service": null + }, + { + "source": "/data", + "destination": "artifacts/data", + "type": "directory", + "status": "ok", + "service": null + } + ], + "artifact_file_sha256": { + "artifacts/app/conflicts.json": "b36e9898ecb5f1056780359f3cc5ab50f043f1e4e732972781d92aac78ada3ed", + "artifacts/app/merge_users.py": "b0d94a6b07ab4eff9c0920cb96a37ae66e65a6c892d354411c7fdd6075f8a934", + "artifacts/app/merged_users.parquet": "effad8725622325c3a3185038b610a2097d6337788bd5031435ecdee6e93b2ff", + "artifacts/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "artifacts/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "artifacts/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "artifacts/manifest.json": "afdc3cddfe46f408bbaea8e826dd0c1423ed4c97948d66029243a8a8c8accd71" + }, + "artifact_symlinks": {}, + "evidence_sha256": { + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/result.json": "a7b0dd5432c9c2039c843f48248ddb269ad759173031736da46ba96e6b7d7b79", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/config.json": "51c251981a4e2ed33dcf2287c86799de7fd945bb13085706496fb9e9e882a064", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/agent/bootstrap.json": "33254eef8f18b233ce1017454a168303e97c29fc41d02a971740ec588a03302e", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/agent/trajectory.json": "b8d4e011e14fda92785c457dbb1dacfeec4f8bf70f180b39e65128a51c1429f6", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/agent/nanopycodeagent.txt": "d8ede0eed4bd3310ddd4e33573943d494fff33c717a9f7b67124d1211651eb54", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/artifacts/manifest.json": "afdc3cddfe46f408bbaea8e826dd0c1423ed4c97948d66029243a8a8c8accd71", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/verifier/test-stdout.txt": "097b73203b93f1098cdfa430eeaebb06aec3a45147bea3888696d63371885498", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/verifier/ctrf.json": "245207047e0ca01be6ad6b921412dd4dfee3f587495e682fd20e4fc4bb6ee23c", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/verifier/retry-summary.json": "fd82f49cf1b5830375138e5d46acf56ef0c1344793a68f0b1077309064c63f10", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020-input/manifest.json": "6c7a1fce21019448c787c2e75726b292c96d5973f691037dbbe358a2d8fb3fe5", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020-input/config.json": "f5fb510d05495fe35a803969b82acec64e21bd9d075fc51fecb2f828a67db6ce", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev3-1017-recovery-20261020-v2-orchestration/dev3-1017-multi-source-data-merger-recovery-20261020.json": "f9b1dcc365734193d2ce60bffc5db9c1e157413e9aa34b50e3e4ab300c673b9f", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/verifier/attempts/1/ctrf.json": "245207047e0ca01be6ad6b921412dd4dfee3f587495e682fd20e4fc4bb6ee23c", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/runtime-source/jobs/dev3-1017-multi-source-data-merger-recovery-20261020/multi-source-data-merger__jNM6pZd/verifier/attempts/1/test-stdout.txt": "097b73203b93f1098cdfa430eeaebb06aec3a45147bea3888696d63371885498" + }, + "original_baseline_classification": "verifier_error", + "original_baseline_reward": 0.0, + "original_baseline_trials_unchanged": true, + "provenance_checks_passed": true, + "analysis_note": "The official verifier executed all three checks and passed. One verifier attempt; the new merged values and conflict report satisfy the official assertions." + }, + { + "order": 2, + "task": "terminal-bench/pytorch-model-cli", + "task_ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "job": "dev3-1017-pytorch-model-cli-recovery-20261020", + "trial": "pytorch-model-cli__P8vPNwH", + "classification": "scored_failure", + "raw_reward": 0.0, + "analytical_score": 0.0, + "exception_type": null, + "official_pytest_counts": { + "passed": 5, + "failed": 1, + "skipped": 0, + "errors": 0, + "xfailed": 0, + "xpassed": 0 + }, + "official_ctrf_summary": { + "tests": 6, + "passed": 5, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792506000.5999215, + "stop": 1792506032.724694 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0022991680016 + }, + { + "attempt": 2, + "status": "completed", + "return_code": 0, + "duration_seconds": 492.35444629899575 + } + ], + "started_at": "2026-10-20T13:46:59.283417Z", + "finished_at": "2026-10-20T14:20:54.582370Z", + "seconds": { + "environment_setup": 2.130641, + "agent_setup": 40.033522, + "agent_execution": 579.237753, + "verifier": 1392.571163 + }, + "task_checksum": "e8438573f503c2d98ad71905862638bfa485470010d15d5b092821e0096b9daa", + "source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "7ab271344f8fda788e5562c96c7ce2d39f2db5137b629930c71e1e91fac016c9", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "86a92c078b9e6276e946aabc688e13ba267220b5d5b01c52944b7bb596ecd392" + }, + "wheel_sha256": "eb90a6fac1961222da55497a4012654f084919adae6cd25c6ab1c01d39ef202e", + "bootstrap_status": "ready", + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 696822, + "completion_tokens": 24690, + "cached_tokens": 685568 + }, + "estimated_known_cost_usd": 0.037117608, + "native_cost_matches_atif": true + }, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 32, + "tool_calls": 36, + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + }, + { + "source": "/app", + "destination": "artifacts/app", + "type": "directory", + "status": "ok", + "service": null + } + ], + "artifact_file_sha256": { + "artifacts/app/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "artifacts/app/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "artifacts/app/cli_tool": "e6c80c3f6665ffc44ff2e7042b33235550c79a1ce5e56227827de104504d977b", + "artifacts/app/cli_tool.cpp": "1f67600b6af1572e92fcb314124ee3014458cefd6f2ec54217078c8cf2cc1abe", + "artifacts/app/cli_tool.py": "92295cdd7be35342e38da3b482db7266cfb7d27d7b64d9b317382ecf811b53e8", + "artifacts/app/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "artifacts/app/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "artifacts/app/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "artifacts/app/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "artifacts/app/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "artifacts/app/prediction.txt": "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", + "artifacts/app/weights.json": "76b0b062f898598366154c2c4618534594e7bb0e69c60c49bcceec101cd83708", + "artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6" + }, + "artifact_symlinks": {}, + "evidence_sha256": { + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/result.json": "1c4d0b6360600073b5dedd1ca089f4f8cff37819935549362f02ff06b1bd0056", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/config.json": "f6040debc1dea9a8386da3bde6554edd5bfe6b6f3f08d4a4bae5bc4a6f9b649c", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/agent/bootstrap.json": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/agent/trajectory.json": "f05243db52fdae75e29108803ba60fdc2bca54c9de9c997d6b1ff08789f080fa", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/agent/nanopycodeagent.txt": "5d4ef4d93898cf10f80442fe1f14908460553ff6fd0bac58cf1a9c54d786fc60", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/test-stdout.txt": "345378b644f0cf3a40244694d131d8bc90ea7c3bf4d057bff6326911aa476d07", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/ctrf.json": "5a03ac466fd168ee870d512b67f9c6cde14477ccbdcf808f927fbc40de4ff807", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/retry-summary.json": "cc4542a0e2fe235bc93d0a1dfd8a86a67bf76bf31fc5d29a94b64138dac737b8", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020-input/manifest.json": "c7db661f047da988aca3ba1ddc2b5267272b9feae8ae665883cfc7a254f540e8", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020-input/config.json": "2e2df24cacf45e04c35deb2587e20f5ed8a07117d6317d757c09ba27a35f6c58", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev3-1017-recovery-20261020-v2-orchestration/dev3-1017-pytorch-model-cli-recovery-20261020.json": "4cf5114e0f080117e85bff28a796b06abd9bab5ffa6bd7eac306e86db36a07bd", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/attempts/1/test-stdout.txt": "d8e593f88b8393d6c541842c18a04d8ec05d7bd4746783106570b33e64900426", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/attempts/2/ctrf.json": "5a03ac466fd168ee870d512b67f9c6cde14477ccbdcf808f927fbc40de4ff807", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/attempts/2/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-cli-recovery-20261020/pytorch-model-cli__P8vPNwH/verifier/attempts/2/test-stdout.txt": "345378b644f0cf3a40244694d131d8bc90ea7c3bf4d057bff6326911aa476d07" + }, + "original_baseline_classification": "verifier_error", + "original_baseline_reward": 0.0, + "original_baseline_trials_unchanged": true, + "provenance_checks_passed": true, + "analysis_note": "The first verifier attempt timed out after 900 seconds during dependency installation, before pytest. The second completed with five passed checks and one failed check: 10 of the 50 checked images produced incorrect predictions. The submitted cli_tool is an ELF executable (5,877,872 bytes). Its C++ source applies (pixel/255 - 0.1307)/0.3081, whereas the official dataset pipeline uses ToTensor() without that extra normalization. This is an observed preprocessing mismatch; no patched solution was tested. A valid scored zero stopped recovery after two attempts." + }, + { + "order": 3, + "task": "terminal-bench/pytorch-model-recovery", + "task_ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "job": "dev3-1017-pytorch-model-recovery-recovery-20261020", + "trial": "pytorch-model-recovery__nCs7Y4H", + "classification": "verifier_error", + "raw_reward": null, + "analytical_score": null, + "exception_type": "VerifierTimeoutError", + "official_pytest_counts": null, + "official_ctrf_summary": null, + "verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0020226070192 + }, + { + "attempt": 2, + "status": "timeout", + "duration_seconds": 900.0021604810026 + }, + { + "attempt": 3, + "status": "timeout", + "duration_seconds": 900.0021750649903 + } + ], + "started_at": "2026-10-20T14:21:06.869949Z", + "finished_at": "2026-10-20T15:09:44.704803Z", + "seconds": { + "environment_setup": 2.140067, + "agent_setup": 37.050461, + "agent_execution": 156.263275, + "verifier": 2700.964603 + }, + "task_checksum": "e321c0e7501b3f021cfca26520574b343dff3dee6919f2e9dd5ff9e28ffc3725", + "source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "7ab271344f8fda788e5562c96c7ce2d39f2db5137b629930c71e1e91fac016c9", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "86a92c078b9e6276e946aabc688e13ba267220b5d5b01c52944b7bb596ecd392" + }, + "wheel_sha256": "eb90a6fac1961222da55497a4012654f084919adae6cd25c6ab1c01d39ef202e", + "bootstrap_status": "ready", + "verifier_dependency_profile": { + "profile": null, + "status": "not_configured" + }, + "accounting": { + "usage_complete": true, + "cost_complete": true, + "cost_is_estimated": true, + "recorded_tokens": { + "prompt_tokens": 444071, + "completion_tokens": 19027, + "cached_tokens": 432000 + }, + "estimated_known_cost_usd": 0.0290457, + "native_cost_matches_atif": true + }, + "trajectory_terminal_outcome": "completed", + "recorded_agent_steps": 25, + "tool_calls": 26, + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + }, + { + "source": "/app", + "destination": "artifacts/app", + "type": "directory", + "status": "ok", + "service": null + } + ], + "artifact_file_sha256": { + "artifacts/app/__pycache__/recovered_model.cpython-313.pyc": "c492a45a453bba7c9fc0db2bce81b3bac895ca649841b91e2919e13f4ac59071", + "artifacts/app/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "artifacts/app/model.pt": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "artifacts/app/recovered_model.py": "602526444f3517d52c011a3291c3b492d6a86b4bd5200011b4027bc2ee86f3f7", + "artifacts/app/tune_model.py": "e2e9e6bbdb6e13f9e7e57d28f8c4e66932bfeff8dc468df2ae2034b8b9e92e71", + "artifacts/app/verify.py": "ed8ec4d7d52bbd849c15f8acc066825af6354fb729f265e9d223307835381c9e", + "artifacts/app/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6" + }, + "artifact_symlinks": {}, + "evidence_sha256": { + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/result.json": "2a61cc964ad2a684e2b4adb3e5d333f75093db51320f7e448a2cb83446998e34", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/config.json": "125c1fdecd4add7009fa787adb79226e020e0f8598b12413864ac3c8f3502c83", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/agent/bootstrap.json": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/agent/trajectory.json": "73b4f374dbfb89dd3d3b94d572136c2f613bc1666aad87b8a6454c3316bb3f3d", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/agent/nanopycodeagent.txt": "036c564ae66e993149b0a73d6de0e24292f3b06331b035c929d109e019d5025a", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/verifier/test-stdout.txt": "034e509e6d5e1e587085482f987d108a25588d854b1404f6f0a3b424ba2c29d8", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/verifier/retry-summary.json": "8609c874195c9b06450a4a0667270aac906444ee1adfa675f486aa19450b55ea", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020-input/manifest.json": "733df586f30505c9e3900e0bf0af80c7a97fe3d6af61a16677e7704c8fc9f2cf", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020-input/config.json": "3aef21d9d14d2c895cc08cfd755a5375687cc2bbeb8103aeb051ff321d017343", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev3-1017-recovery-20261020-v2-orchestration/dev3-1017-pytorch-model-recovery-recovery-20261020.json": "5df563c0dfc936b5798e7e78ece5489ae105488a7fd373f1a19c902d946abc71", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/verifier/attempts/1/test-stdout.txt": "4d00e96829b6291622f1a138c491042c44d5d2790880e9253ac72d363d906864", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/verifier/attempts/2/test-stdout.txt": "e951235c90184be58c1908b676fd33ddbf8e9199c8fd61bf416f0d1e80934690", + "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H/verifier/attempts/3/test-stdout.txt": "034e509e6d5e1e587085482f987d108a25588d854b1404f6f0a3b424ba2c29d8" + }, + "original_baseline_classification": "verifier_error", + "original_baseline_reward": null, + "original_baseline_trials_unchanged": true, + "provenance_checks_passed": true, + "analysis_note": "The agent reconstructs an encoder/decoder, tunes only output_layer with ridge regression, and exports a TorchScript model. The task does not require a particular optimization method. The submitted source and tool calls show no hidden-test introspection; final verifier status is collected separately. All three verifier attempts exhausted their original 900-second limits while downloading PyTorch/CUDA dependencies before pytest started. The native VerifierTimeoutError and null reward are retained. No official assertions executed, so correctness of this new TorchScript output remains unverified." + } + ], + "original_baseline_integrity": { + "checked_at": "2026-10-20T13:44:52.930641+00:00", + "original_baseline_path": "/home/minix/projects/nanoPyCodeAgent/jobs/dev34-flash1017-main-20261019/benchmarks/harbor/results/dev34-flash1017-main-20261019.json", + "original_baseline_sha256": "b107d3a48b7205bd092ceed12f5e6cf3ccdf8b65ffd7cddddeacb73ee0b9a0a5", + "original_baseline_publication_ref": "1ae6e05", + "original_aggregate": { + "tasks": 34, + "classifications": { + "passed": 26, + "scored_failure": 5, + "verifier_error": 3 + }, + "passed_fraction_of_pool": 0.7647058823529411, + "official_pytest_counts": { + "passed": 133, + "failed": 8, + "skipped": 0, + "errors": 1, + "xfailed": 0, + "xpassed": 0 + }, + "verifier_attempts": 37, + "verifier_retries": 3, + "wall_seconds": 15853.918955, + "estimated_known_cost_usd": 1.860998832, + "usage_incomplete_tasks": [], + "cost_incomplete_tasks": [], + "recorded_tokens": { + "prompt_tokens": 47814612, + "completion_tokens": 1153308, + "cached_tokens": 47167872 + }, + "assertion_cases": 141, + "collection_errors": 1, + "agent_trials_started": 34, + "host_launch_errors_before_model": 3 + }, + "workspace_inspection": "Docker reported no containers and no volumes before starting this supplemental experiment. Original artifact manifests contain empty /logs/artifacts directories only.", + "policy": "Preserve original baseline and native evidence unchanged. New solutions are separate trials, not original-output verifier retries.", + "trials": [ + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__ovUnaUM", + "original_classification": "verifier_error", + "original_raw_reward": 0.0, + "original_analytical_score": null, + "original_verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 327.15425308000704 + } + ], + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/result.json": "bd84053547e596ad0f3a8285a80da37e3a22f0eae72f9fe831fea2d526888583", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/config.json": "32879386a7e47a3832e34f78fc25b31375362c8a98f7e94c2f97a56288ab0fda", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/bootstrap.json": "5011ac5d6ab074bd66636cc77aae3a2d966293bcb051b929e7a9cb076f908948", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/trajectory.json": "652ff7db3dd1b8f75a080cb68a12e9a7bbcb73b9ae245b4d52485487e901a424", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/agent/nanopycodeagent.txt": "cb5d1b6d5480a1e87a6236a04545f52a35296ede77de27c7fe9770edbfcc4acd", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/test-stdout.txt": "7690d05931b29fda94b3c858291f9a35e557ebbe826c715725fda790311dba46", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/retry-summary.json": "f637a457f9341eefc7e30edd19559bc31c0bca312791506f4320847b38cd5c53", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/manifest.json": "7fda413bc1c8c6a57d8ab89a72c7f81a9b16f7fdc0140b03cd9c9696584be5ae", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/config.json": "175d9eea47f437346fc37f27016c08a2fb591e4cd345044b6254f6e13ab7c701", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-multi-source-data-merger-main-20261019.json": "1fe2a8510681dc681b0d2145503b4bfb72b60fe667e6fd4c0fcd098324b3aa7e", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/attempts/1/test-stdout.txt": "7690d05931b29fda94b3c858291f9a35e557ebbe826c715725fda790311dba46", + "jobs/main-source/jobs/dev34-1017-multi-source-data-merger-main-20261019/multi-source-data-merger__ovUnaUM/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "artifact_manifest_sha256": "c1db653f487fb773c4f9247162ab3314e7f0584a486914b9874c4b3d6949e42d", + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + } + ], + "original_workspace_available": false + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__t73XoLw", + "original_classification": "verifier_error", + "original_raw_reward": 0.0, + "original_analytical_score": null, + "original_verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 133.28830967300746 + } + ], + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/result.json": "4847cac28aefa9154528ed40833214b4401c0065e2eee64aeeb8bdec6ec0fbb1", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/config.json": "b80389d25c6b68223471bb82b4ca1c5f47bedd16e6de6c7ad08887957e8ab398", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/trajectory.json": "9baf56afbccfcb6f39ab2df5c5c7f21b7684783d5f32fee2693897f337bfe446", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/agent/nanopycodeagent.txt": "c689bf519d6c856396aa49c03dfb0a1bb65ad9dd46eb905372df6f595ca00872", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/ctrf.json": "0383370176312a002520c110ff53a75664836ed3d627225fd93b76ccd89f9166", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/test-stdout.txt": "dad670fa7c0265c95408dc5f33e01b4dc9d164574ab5e6625b75ecb289bc306f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/retry-summary.json": "9d5e42ed1f72ed1b2bbe1de2724041d8f06eec6bc2e79cad5b52b49131d0ed5b", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/manifest.json": "84502d6134216bcf9b867ed4a206b237fd14dc7a204b226888e75a64fc11ce97", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/config.json": "1b9e5409dc8e2c45449188a96cd3b0399cd99a60900dda4569273848f19ee407", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-cli-main-20261019.json": "98e3548ee417240c814b98bb07b6dd53a772e83e30e6f555f32565c7dc18ebe1", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/test-stdout.txt": "dad670fa7c0265c95408dc5f33e01b4dc9d164574ab5e6625b75ecb289bc306f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/ctrf.json": "0383370176312a002520c110ff53a75664836ed3d627225fd93b76ccd89f9166", + "jobs/main-source/jobs/dev34-1017-pytorch-model-cli-main-20261019/pytorch-model-cli__t73XoLw/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "artifact_manifest_sha256": "c1db653f487fb773c4f9247162ab3314e7f0584a486914b9874c4b3d6949e42d", + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + } + ], + "original_workspace_available": false + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "trial": "pytorch-model-recovery__hogG9gA", + "original_classification": "verifier_error", + "original_raw_reward": null, + "original_analytical_score": null, + "original_verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0023410760041 + }, + { + "attempt": 2, + "status": "timeout", + "duration_seconds": 900.0023236230045 + }, + { + "attempt": 3, + "status": "timeout", + "duration_seconds": 900.00202244299 + } + ], + "evidence_sha256": { + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/result.json": "8cd798fd024cae52b15271b61516416a3697d9bb4e834a4f03e46878ea9df759", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/config.json": "fedb449dfc06e5ee34620f44b69539990d5753834b8c0de95ac9a355a8146fb3", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/bootstrap.json": "8b0838a69434e306576e34c0c21d5a7cfc4d98960b8a69a0b1bc58c387078567", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/trajectory.json": "7e4230a06c24c1751916117887130adcfc70429a5f4995669df5fc4afc479b4f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/agent/nanopycodeagent.txt": "5dee8328da08df739c2285b303ffc9349c305d1b803232d82d68e56d3a6a6925", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/test-stdout.txt": "d20478d9b2f2a8527862c484b10fd0140c8f86bfa1c587967131dd9c7e998c8d", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/retry-summary.json": "386028e2e0fa4b477bd4342345e31bd3cdf2377d60cad33e5218d09777e0f43a", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/manifest.json": "992ab17d7c24c8e82ee153e98c5aee1805a191faa57bfab0a5693481ee44cb6f", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/config.json": "4f6cd8791bdb04ff12ed8d1ae1786a1f947298490305e509c0badc0fb02feaaf", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev34-1017-main-20261019-v2-orchestration/dev34-1017-pytorch-model-recovery-main-20261019.json": "b71e4cca26a8670fcc6294e6744ac49a6d7cbb56661253122d15df8bbbde9da8", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/1/test-stdout.txt": "90a456f12a085a8bf4fc5c23c00a01426f771f06a0d62623cf805cf5e57e0c64", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/2/test-stdout.txt": "9cff11e64d1ca45fcd585def9798cb2aa91d8908fc88350f24af3a770b909927", + "jobs/main-source/jobs/dev34-1017-pytorch-model-recovery-main-20261019/pytorch-model-recovery__hogG9gA/verifier/attempts/3/test-stdout.txt": "d20478d9b2f2a8527862c484b10fd0140c8f86bfa1c587967131dd9c7e998c8d" + }, + "artifact_manifest_sha256": "c1db653f487fb773c4f9247162ab3314e7f0584a486914b9874c4b3d6949e42d", + "artifact_manifest": [ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "empty", + "service": null + } + ], + "original_workspace_available": false + } + ] + }, + "post_freeze_audit": { + "started_at": "2026-10-20T14:27:52.592426+00:00", + "agent_output_freeze_sha256": "15e56557516eb4e78a878efd34b69ce5704ab3248e58db69931d128ece73c359", + "trials": [ + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__jNM6pZd", + "tool_calls": 6, + "configured_secret_exact_matches": [], + "candidates": [], + "analysis_note": "The official verifier executed all three checks and passed. One verifier attempt; the new merged values and conflict report satisfy the official assertions." + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__P8vPNwH", + "tool_calls": 36, + "configured_secret_exact_matches": [], + "candidates": [], + "analysis_note": "The first verifier attempt timed out after 900 seconds during dependency installation, before pytest. The second completed with five passed checks and one failed check: 10 of the 50 checked images produced incorrect predictions. The submitted cli_tool is an ELF executable (5,877,872 bytes). Its C++ source applies (pixel/255 - 0.1307)/0.3081, whereas the official dataset pipeline uses ToTensor() without that extra normalization. This is an observed preprocessing mismatch; no patched solution was tested. A valid scored zero stopped recovery after two attempts." + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "trial": "pytorch-model-recovery__nCs7Y4H", + "tool_calls": 26, + "configured_secret_exact_matches": [], + "candidates": [], + "analysis_note": "The agent reconstructs an encoder/decoder, tunes only output_layer with ridge regression, and exports a TorchScript model. The task does not require a particular optimization method. The submitted source and tool calls show no hidden-test introspection; final verifier status is collected separately. All three verifier attempts exhausted their original 900-second limits while downloading PyTorch/CUDA dependencies before pytest started. The native VerifierTimeoutError and null reward are retained. No official assertions executed, so correctness of this new TorchScript output remains unverified." + } + ], + "status": "complete", + "completed_at": "2026-10-20T15:11:54.439665+00:00", + "scope": "Scanned all three ATIF trajectories and agent stdout, checked all 68 tool-call arguments and their paths/URLs, reviewed submitted model-recovery source and CLI preprocessing, and scanned all 40 frozen files for exact configured credential values. Artifact hashes were frozen before this review.", + "tool_calls_reviewed": 68, + "frozen_files_scanned": 40, + "configured_credential_exact_matches": 0, + "observed_task_specific_reference_solution_retrieval": false, + "observed_hidden_verifier_content_reads": false, + "limitations": "No observed retrieval is not proof of absence of prior exposure or training contamination. The development tasks have prior exposure. No original solution was restored or regraded.", + "trajectory_review_completed_at": "2026-10-20T14:32:41.358517+00:00", + "final_verifier_review": "All three recovery-task verifier attempts timed out before pytest during PyTorch/CUDA dependency installation. The CLI completed valid assertions on its second attempt and stopped on scored zero." + }, + "documentation_sync": { + "status": "complete", + "checked_at": "2026-10-20T15:13:38.395847+00:00", + "english_path": "benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md", + "simplified_chinese_path": "benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md", + "report_sha256": { + "benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md": "e94129568aae507b54f5731abeebda397359d326c1501df79477d753485f91ed", + "benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md": "c30b6cb3caf0eccf0fdccebdb6e3b3e60b52aa238f232595a83c4407e08d2e0c" + }, + "checks": "Both complete versions have identical result rows (with translated outcome labels), counts, code identifiers, hashes, evidence/container rows and caveats; all local links resolve." + }, + "container_retention": { + "checked_at": "2026-10-20T15:10:56.567300+00:00", + "all_three_main_containers_retained": true, + "all_main_containers_stopped": true, + "all_original_image_identities_match": true, + "main_container_oom_kills": 0, + "trials": [ + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__jNM6pZd", + "container_name": "multi-source-data-merger__jnm6pzd__env-main-1", + "container_id": "e16fca6327890edd6055c9815cd7f0b5823664d30759df914c591e7460f09cc4", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.8784315Z", + "FinishedAt": "2026-10-20T13:49:10.129622197Z" + }, + "native_resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "effective_memory_bytes": 2147483648, + "effective_nano_cpus": 1000000000 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__P8vPNwH", + "container_name": "pytorch-model-cli__p8vpnwh__env-main-1", + "container_id": "5906632d1db211dbb8c61963bf42befb9ba8252bdfb5efe6a6dbb56e17d528ea", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.649251362Z", + "FinishedAt": "2026-10-20T14:20:54.147226314Z" + }, + "native_resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "effective_memory_bytes": 2147483648, + "effective_nano_cpus": 1000000000 + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "trial": "pytorch-model-recovery__nCs7Y4H", + "container_name": "pytorch-model-recovery__ncs7y4h__env-main-1", + "container_id": "e8f4a6bb804bc8ef8f5e0acde059fd6205efc9cd4966b0029149873af31619fd", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T14:21:07.251168654Z", + "FinishedAt": "2026-10-20T15:09:44.189596443Z" + }, + "native_resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "effective_memory_bytes": 2147483648, + "effective_nano_cpus": 1000000000 + } + ], + "retained_services": [ + { + "name": "multi-source-data-merger__jnm6pzd__env-apt-index-cache-1", + "id": "4b65baf1e8d17db261cb3f8586cb0e9323e9f13150b785bc300e9e92e02b7d46", + "service": "apt-index-cache", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.981803547Z", + "FinishedAt": "2026-10-20T13:48:59.540705831Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-20T06:48:54.525155802-07:00", + "End": "2026-10-20T06:48:54.621362256-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:55.62176899-07:00", + "End": "2026-10-20T06:48:55.722619757-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:56.723896092-07:00", + "End": "2026-10-20T06:48:56.821166835-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:57.822164961-07:00", + "End": "2026-10-20T06:48:57.921429044-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:58.922528853-07:00", + "End": "2026-10-20T06:48:59.023702263-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "name": "multi-source-data-merger__jnm6pzd__env-uv-release-cache-1", + "id": "5984d22da76d851fe107c320d253d75064e23d6b60a5cd5034174309b4c29272", + "service": "uv-release-cache", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:47:00.044021975Z", + "FinishedAt": "2026-10-20T13:48:59.540687214Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-20T06:48:54.369340846-07:00", + "End": "2026-10-20T06:48:54.468518931-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:55.469907396-07:00", + "End": "2026-10-20T06:48:55.577191344-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:56.577494173-07:00", + "End": "2026-10-20T06:48:56.666421426-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:57.66772146-07:00", + "End": "2026-10-20T06:48:57.772570149-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T06:48:58.77358243-07:00", + "End": "2026-10-20T06:48:58.880741212-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "name": "multi-source-data-merger__jnm6pzd__env-main-1", + "id": "e16fca6327890edd6055c9815cd7f0b5823664d30759df914c591e7460f09cc4", + "service": "main", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.8784315Z", + "FinishedAt": "2026-10-20T13:49:10.129622197Z" + } + }, + { + "name": "pytorch-model-cli__p8vpnwh__env-apt-index-cache-1", + "id": "e89d74c25667fbd60ba97883a4e83e20f632e3ec9bb09aada42ec530cd5435e5", + "service": "apt-index-cache", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.74086331Z", + "FinishedAt": "2026-10-20T14:20:44.05611658Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-20T07:20:38.903612349-07:00", + "End": "2026-10-20T07:20:38.998831059-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:39.999677703-07:00", + "End": "2026-10-20T07:20:40.094269997-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:41.095479113-07:00", + "End": "2026-10-20T07:20:41.193911616-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:42.194830528-07:00", + "End": "2026-10-20T07:20:42.290258673-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:43.290793369-07:00", + "End": "2026-10-20T07:20:43.388917902-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "name": "pytorch-model-cli__p8vpnwh__env-uv-release-cache-1", + "id": "ee1a43f50b1aff3c93daa4da08c7fdf3ca350d8faa3203a1811ecc4f2e2d321c", + "service": "uv-release-cache", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.807393322Z", + "FinishedAt": "2026-10-20T14:20:44.05586415Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-20T07:20:39.249579801-07:00", + "End": "2026-10-20T07:20:39.342406095-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:40.343027248-07:00", + "End": "2026-10-20T07:20:40.439525331-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:41.440617885-07:00", + "End": "2026-10-20T07:20:41.5406096-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:42.542014509-07:00", + "End": "2026-10-20T07:20:42.643189646-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T07:20:43.644615773-07:00", + "End": "2026-10-20T07:20:43.738489694-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "name": "pytorch-model-cli__p8vpnwh__env-main-1", + "id": "5906632d1db211dbb8c61963bf42befb9ba8252bdfb5efe6a6dbb56e17d528ea", + "service": "main", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T13:46:59.649251362Z", + "FinishedAt": "2026-10-20T14:20:54.147226314Z" + } + }, + { + "name": "pytorch-model-recovery__ncs7y4h__env-apt-index-cache-1", + "id": "85ece21e3a35b6db0e4452943ead6ad0581681a74051ad71d1e8579784189a50", + "service": "apt-index-cache", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T14:21:07.415229706Z", + "FinishedAt": "2026-10-20T15:09:34.068706928Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-20T08:09:28.983676349-07:00", + "End": "2026-10-20T08:09:29.087415097-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:30.087911476-07:00", + "End": "2026-10-20T08:09:30.183785603-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:31.184928031-07:00", + "End": "2026-10-20T08:09:31.283601165-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:32.284375745-07:00", + "End": "2026-10-20T08:09:32.38488485-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:33.385962183-07:00", + "End": "2026-10-20T08:09:33.486826714-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "name": "pytorch-model-recovery__ncs7y4h__env-uv-release-cache-1", + "id": "c0f5a78624e44c79f078c5dd77fc6a925c0a435611941374206d77283c0d7976", + "service": "uv-release-cache", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T14:21:07.345031936Z", + "FinishedAt": "2026-10-20T15:09:34.068639679Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-20T08:09:29.049204838-07:00", + "End": "2026-10-20T08:09:29.145990812-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:30.14720016-07:00", + "End": "2026-10-20T08:09:30.250318879-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:31.250881953-07:00", + "End": "2026-10-20T08:09:31.34806937-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:32.348688474-07:00", + "End": "2026-10-20T08:09:32.451285454-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-20T08:09:33.452098539-07:00", + "End": "2026-10-20T08:09:33.547382141-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "name": "pytorch-model-recovery__ncs7y4h__env-main-1", + "id": "e8f4a6bb804bc8ef8f5e0acde059fd6205efc9cd4966b0029149873af31619fd", + "service": "main", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-20T14:21:07.251168654Z", + "FinishedAt": "2026-10-20T15:09:44.189596443Z" + } + } + ], + "original_baseline_unchanged": true, + "original_trial_evidence_files_reverified": 42, + "original_trial_wheels_reverified": 3, + "native_cpu_and_memory_limits_match": true + }, + "host_storage_maintenance": { + "started_at": "2026-10-20T14:51:06.082891+00:00", + "reason": "Temporarily relocate duplicate image download caches during observed verifier disk pressure. Original Docker images, task data, package sources, verifier scripts, solution artifacts and dependency caches are unchanged. Retain byte-identical original cache paths via symlinks until restoration.", + "state": "restored", + "entries": [ + { + "kind": "archives", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/alexgshaw_constraints-scheduling_20251031.oci.tar", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/alexgshaw_constraints-scheduling_20251031.oci.tar", + "size": 49868800, + "sha256": "7329eecdba814c7c5a24bb464f719b19e6a748893bdc98f0429654b934b20c30", + "moved": true, + "restored": true + }, + { + "kind": "archives", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/alexgshaw_db-wal-recovery_20251031.oci.tar", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/alexgshaw_db-wal-recovery_20251031.oci.tar", + "size": 175298560, + "sha256": "8dcd5819d2892c069ae844bc473fbf6f436f02f1be03e3bcfb7aaff41ef72e9e", + "moved": true, + "restored": true + }, + { + "kind": "archives", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/alexgshaw_git-multibranch_20251031.oci.tar", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/alexgshaw_git-multibranch_20251031.oci.tar", + "size": 222156800, + "sha256": "9b4edde0286a626b9e8c0ca61927c7409f350b0ed8ae5e06e9a75d30552a3dd8", + "moved": true, + "restored": true + }, + { + "kind": "archives", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/alexgshaw_multi-source-data-merger_20251031.oci.tar", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/alexgshaw_multi-source-data-merger_20251031.oci.tar", + "size": 220907520, + "sha256": "0724586c62ae743426dd15a73a8bc03a3073d78a8f4dca7b95d2b3eead8fe42e", + "moved": true, + "restored": true + }, + { + "kind": "archives", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/alexgshaw_pytorch-model-cli_20251031.oci.tar", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/alexgshaw_pytorch-model-cli_20251031.oci.tar", + "size": 44369920, + "sha256": "014e5de3abd1f13e842b75bcaa181aba349858f5d5e243a9c40b2a1acff5107d", + "moved": true, + "restored": true + }, + { + "kind": "archives", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/alexgshaw_query-optimize_20251031.oci.tar", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/alexgshaw_query-optimize_20251031.oci.tar", + "size": 120514560, + "sha256": "9436aa5926f36f0f0cd5e25feb24ffb3728e0089d2ef65f4336b8f8b13e834aa", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/057ca2a0dba2dddca0cd4a348ab5e530d411b59d57f17ac999e9291daf503707", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/057ca2a0dba2dddca0cd4a348ab5e530d411b59d57f17ac999e9291daf503707", + "size": 23288765, + "sha256": "057ca2a0dba2dddca0cd4a348ab5e530d411b59d57f17ac999e9291daf503707", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "size": 1362, + "sha256": "0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/0ff6d2977f94533d8a7a0cdc5a16efd42cb3ccfeb2f5684cdc8db0836b43b9e3", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/0ff6d2977f94533d8a7a0cdc5a16efd42cb3ccfeb2f5684cdc8db0836b43b9e3", + "size": 2310, + "sha256": "0ff6d2977f94533d8a7a0cdc5a16efd42cb3ccfeb2f5684cdc8db0836b43b9e3", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/1351034b0738666abd9b361bcce533e9c3bfd78ea38445fd30ed4b21d63e8fdf", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/1351034b0738666abd9b361bcce533e9c3bfd78ea38445fd30ed4b21d63e8fdf", + "size": 92, + "sha256": "1351034b0738666abd9b361bcce533e9c3bfd78ea38445fd30ed4b21d63e8fdf", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/14dd762cfec1bf2eb97dfc87325a73b87a3e0d61fbd86d7cd72b4cb773132a73", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/14dd762cfec1bf2eb97dfc87325a73b87a3e0d61fbd86d7cd72b4cb773132a73", + "size": 523, + "sha256": "14dd762cfec1bf2eb97dfc87325a73b87a3e0d61fbd86d7cd72b4cb773132a73", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/19ba1edfa8aca3ca931727a9beb36396299d4076c3ced9574490a51f5be15fbc", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/19ba1edfa8aca3ca931727a9beb36396299d4076c3ced9574490a51f5be15fbc", + "size": 192403950, + "sha256": "19ba1edfa8aca3ca931727a9beb36396299d4076c3ced9574490a51f5be15fbc", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/19fb8589da0207a0e7d3baa0c1b71a67136b1ad06c4b2e65cc771664592e6d9e", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/19fb8589da0207a0e7d3baa0c1b71a67136b1ad06c4b2e65cc771664592e6d9e", + "size": 249, + "sha256": "19fb8589da0207a0e7d3baa0c1b71a67136b1ad06c4b2e65cc771664592e6d9e", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/20929a28fe1e5af2d88306abb0017058246d012bf454261a9ba22c0658f31078", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/20929a28fe1e5af2d88306abb0017058246d012bf454261a9ba22c0658f31078", + "size": 20125351, + "sha256": "20929a28fe1e5af2d88306abb0017058246d012bf454261a9ba22c0658f31078", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/216effbc080728f5f3d64828761bd28c78808cd43c9bd3c75bea5094cb30a000", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/216effbc080728f5f3d64828761bd28c78808cd43c9bd3c75bea5094cb30a000", + "size": 91, + "sha256": "216effbc080728f5f3d64828761bd28c78808cd43c9bd3c75bea5094cb30a000", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/24e9f2af0382d814e60f76f5b219bf4eb0b5795d1b18822549e02f05913f4f71", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/24e9f2af0382d814e60f76f5b219bf4eb0b5795d1b18822549e02f05913f4f71", + "size": 177531, + "sha256": "24e9f2af0382d814e60f76f5b219bf4eb0b5795d1b18822549e02f05913f4f71", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "size": 1789, + "sha256": "28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/2d19236650cc866ba441762e3d89edfa452756e0e4833cfdb984257d50b446e7", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/2d19236650cc866ba441762e3d89edfa452756e0e4833cfdb984257d50b446e7", + "size": 6012, + "sha256": "2d19236650cc866ba441762e3d89edfa452756e0e4833cfdb984257d50b446e7", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/2ed2e0894df4529528cbd34c879f9a0a9767ded49a43b6771ae3a53310b2453c", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/2ed2e0894df4529528cbd34c879f9a0a9767ded49a43b6771ae3a53310b2453c", + "size": 2326, + "sha256": "2ed2e0894df4529528cbd34c879f9a0a9767ded49a43b6771ae3a53310b2453c", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/31ab2b12569dc3e523dfc7492dbf9f764a2201458a630ab1af4f741a6dce308d", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/31ab2b12569dc3e523dfc7492dbf9f764a2201458a630ab1af4f741a6dce308d", + "size": 627, + "sha256": "31ab2b12569dc3e523dfc7492dbf9f764a2201458a630ab1af4f741a6dce308d", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/38513bd7256313495cdd83b3b0915a633cfa475dc2a07072ab2c8d191020ca5d", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/38513bd7256313495cdd83b3b0915a633cfa475dc2a07072ab2c8d191020ca5d", + "size": 29777924, + "sha256": "38513bd7256313495cdd83b3b0915a633cfa475dc2a07072ab2c8d191020ca5d", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/47dff70e3eac1e1529d8a120abdbac81ea1c6f3cc4fe037e974cc124831676f8", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/47dff70e3eac1e1529d8a120abdbac81ea1c6f3cc4fe037e974cc124831676f8", + "size": 1573663, + "sha256": "47dff70e3eac1e1529d8a120abdbac81ea1c6f3cc4fe037e974cc124831676f8", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/4b3ffd8ccb5201a0fc03585952effb4ed2d1ea5e704d2e7330212fb8b16c86a3", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/4b3ffd8ccb5201a0fc03585952effb4ed2d1ea5e704d2e7330212fb8b16c86a3", + "size": 29723147, + "sha256": "4b3ffd8ccb5201a0fc03585952effb4ed2d1ea5e704d2e7330212fb8b16c86a3", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/5096d3409e5c692afd62a136f90a2e52dd87653aa6637a3e1ca31ad30fab1096", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/5096d3409e5c692afd62a136f90a2e52dd87653aa6637a3e1ca31ad30fab1096", + "size": 6640, + "sha256": "5096d3409e5c692afd62a136f90a2e52dd87653aa6637a3e1ca31ad30fab1096", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/516306c0434b835b3b409a324d1a37714c87f6149e765e721d13c3ad78566dc9", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/516306c0434b835b3b409a324d1a37714c87f6149e765e721d13c3ad78566dc9", + "size": 91, + "sha256": "516306c0434b835b3b409a324d1a37714c87f6149e765e721d13c3ad78566dc9", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/5654e241057913a0f1a234583c62e12ed7bfab560d9c3f0dc5da985d9f3514eb", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/5654e241057913a0f1a234583c62e12ed7bfab560d9c3f0dc5da985d9f3514eb", + "size": 2854, + "sha256": "5654e241057913a0f1a234583c62e12ed7bfab560d9c3f0dc5da985d9f3514eb", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "size": 1155, + "sha256": "567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/57376dfe58013c9d6be8be41c7abeb7ba9f9fe65f6a5e5e70ae0650aa06992d8", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/57376dfe58013c9d6be8be41c7abeb7ba9f9fe65f6a5e5e70ae0650aa06992d8", + "size": 42615364, + "sha256": "57376dfe58013c9d6be8be41c7abeb7ba9f9fe65f6a5e5e70ae0650aa06992d8", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "size": 1570, + "sha256": "5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/63ff33ad641b9f09475780c18fcae5aa35c92fed4207585d4e93c6070d6c2e0e", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/63ff33ad641b9f09475780c18fcae5aa35c92fed4207585d4e93c6070d6c2e0e", + "size": 92, + "sha256": "63ff33ad641b9f09475780c18fcae5aa35c92fed4207585d4e93c6070d6c2e0e", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/65694556a87deb6f2c0f643841d4d4ff3fabd1e794999978694d891ccb24e703", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/65694556a87deb6f2c0f643841d4d4ff3fabd1e794999978694d891ccb24e703", + "size": 2092, + "sha256": "65694556a87deb6f2c0f643841d4d4ff3fabd1e794999978694d891ccb24e703", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/656e54030aa62a7d0a25b9789570bf2df27d160b4d1ad31e28caefc1c980955e", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/656e54030aa62a7d0a25b9789570bf2df27d160b4d1ad31e28caefc1c980955e", + "size": 334, + "sha256": "656e54030aa62a7d0a25b9789570bf2df27d160b4d1ad31e28caefc1c980955e", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/7a195517a906da1d8081fa0ff799cf6af1ef716a0067307c70ef63bc8bb6dd32", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/7a195517a906da1d8081fa0ff799cf6af1ef716a0067307c70ef63bc8bb6dd32", + "size": 397, + "sha256": "7a195517a906da1d8081fa0ff799cf6af1ef716a0067307c70ef63bc8bb6dd32", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/7afc971ff391e7867c7ef4ebd87302be1bba2cd0ade63f82f22f7373022110ec", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/7afc971ff391e7867c7ef4ebd87302be1bba2cd0ade63f82f22f7373022110ec", + "size": 130, + "sha256": "7afc971ff391e7867c7ef4ebd87302be1bba2cd0ade63f82f22f7373022110ec", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/7e88bf276da094b3c1c5d62775d85ae6e63c58165a5c759570bcecf8b52a4ce3", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/7e88bf276da094b3c1c5d62775d85ae6e63c58165a5c759570bcecf8b52a4ce3", + "size": 3515854, + "sha256": "7e88bf276da094b3c1c5d62775d85ae6e63c58165a5c759570bcecf8b52a4ce3", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "size": 1998, + "sha256": "8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/9040135e24f8def558e39d9b4c5b54b721c749c33610ae4233bfab3ccb2c63b6", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/9040135e24f8def558e39d9b4c5b54b721c749c33610ae4233bfab3ccb2c63b6", + "size": 93, + "sha256": "9040135e24f8def558e39d9b4c5b54b721c749c33610ae4233bfab3ccb2c63b6", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/99b12d2635194ce40491d1dcd713ca38107c6a363b0702341240b2a0e576baa0", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/99b12d2635194ce40491d1dcd713ca38107c6a363b0702341240b2a0e576baa0", + "size": 145548211, + "sha256": "99b12d2635194ce40491d1dcd713ca38107c6a363b0702341240b2a0e576baa0", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/a9ffe18d7fdb9bb2f5b878fdc08887ef2d9644c86f5d4e07cc2e80b783fbea04", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/a9ffe18d7fdb9bb2f5b878fdc08887ef2d9644c86f5d4e07cc2e80b783fbea04", + "size": 1292681, + "sha256": "a9ffe18d7fdb9bb2f5b878fdc08887ef2d9644c86f5d4e07cc2e80b783fbea04", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/b1badc6e50664185acfaa0ca255d8076061c2a9d881cecaaad281ae11af000ce", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/b1badc6e50664185acfaa0ca255d8076061c2a9d881cecaaad281ae11af000ce", + "size": 28230255, + "sha256": "b1badc6e50664185acfaa0ca255d8076061c2a9d881cecaaad281ae11af000ce", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "size": 1575, + "sha256": "c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/ce5b55dfeca415b971cd9ee0a4c50591feb1c7d06cefce8c6654d9f9589d68a1", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/ce5b55dfeca415b971cd9ee0a4c50591feb1c7d06cefce8c6654d9f9589d68a1", + "size": 23288765, + "sha256": "ce5b55dfeca415b971cd9ee0a4c50591feb1c7d06cefce8c6654d9f9589d68a1", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/d37114942cef1c97956f7ef87c75eca327b22718932547e74dd54d8a92c13a7a", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/d37114942cef1c97956f7ef87c75eca327b22718932547e74dd54d8a92c13a7a", + "size": 13646886, + "sha256": "d37114942cef1c97956f7ef87c75eca327b22718932547e74dd54d8a92c13a7a", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/d55dfff74d98ac204fa78f18533eecc39a8bd69bd9384a73572c6d09f06b11fc", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/d55dfff74d98ac204fa78f18533eecc39a8bd69bd9384a73572c6d09f06b11fc", + "size": 12418042, + "sha256": "d55dfff74d98ac204fa78f18533eecc39a8bd69bd9384a73572c6d09f06b11fc", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/dacaa4c4b4567b83fe1f6a497ad6d562a963eea3b0ce7e4419d97af5c213526b", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/dacaa4c4b4567b83fe1f6a497ad6d562a963eea3b0ce7e4419d97af5c213526b", + "size": 1114, + "sha256": "dacaa4c4b4567b83fe1f6a497ad6d562a963eea3b0ce7e4419d97af5c213526b", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/e03ee4aed2ebd0b03b8e49a7d5d2ceab71ea9fce72b7b2e95ab79389fcc4180c", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/e03ee4aed2ebd0b03b8e49a7d5d2ceab71ea9fce72b7b2e95ab79389fcc4180c", + "size": 161793587, + "sha256": "e03ee4aed2ebd0b03b8e49a7d5d2ceab71ea9fce72b7b2e95ab79389fcc4180c", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/e73850a50582f63498f7551a987cc493e848413fcae176379acff9144341f77f", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/e73850a50582f63498f7551a987cc493e848413fcae176379acff9144341f77f", + "size": 14358125, + "sha256": "e73850a50582f63498f7551a987cc493e848413fcae176379acff9144341f77f", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/e97853488aa5b877aa1248a8d146c75752543b00dced77518e40164d1badafd3", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/e97853488aa5b877aa1248a8d146c75752543b00dced77518e40164d1badafd3", + "size": 250, + "sha256": "e97853488aa5b877aa1248a8d146c75752543b00dced77518e40164d1badafd3", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/ec6eeb02cecf35faf5badd75646ab0dd2e02cab484638b413ccd4606a2e2ef7d", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/ec6eeb02cecf35faf5badd75646ab0dd2e02cab484638b413ccd4606a2e2ef7d", + "size": 2905, + "sha256": "ec6eeb02cecf35faf5badd75646ab0dd2e02cab484638b413ccd4606a2e2ef7d", + "moved": true, + "restored": true + }, + { + "kind": "blobs", + "original": "/home/minix/projects/nanoPyCodeAgent/.cache/nanopy-harbor/images/blobs/sha256/eed93e57e79fbfdcee5b29167010ad27dd4ed321cd678f5226834ba46ef43eb0", + "temporary": "/tmp/nanopy-dev3-flash1017-20261020-image-cache/blobs/sha256/eed93e57e79fbfdcee5b29167010ad27dd4ed321cd678f5226834ba46ef43eb0", + "size": 1947, + "sha256": "eed93e57e79fbfdcee5b29167010ad27dd4ed321cd678f5226834ba46ef43eb0", + "moved": true, + "restored": true + } + ], + "free_before": 3208425472, + "pending_cleanup": false, + "moved_at": "2026-10-20T14:51:08.463012+00:00", + "moved_bytes": 1576934879, + "free_after_move": 3208425472, + "restored_at": "2026-10-20T15:10:11.896775+00:00", + "restored_files": 50, + "restoration_hashes_verified": true, + "free_after_restore": 4127641600 + }, + "preflight_history": [ + { + "arm": "recovery", + "task": "terminal-bench/multi-source-data-merger", + "result_path": "jobs/runtime-source/jobs/dev3-1017-install-only-recovery-20261020/multi-source-data-merger__8HoSY39/result.json", + "result_sha256": "d9dffaaa8f4c470a7c71171d84b758e9a016e50aa929064fbf84b0c043628926", + "bootstrap_sha256": "33254eef8f18b233ce1017454a168303e97c29fc41d02a971740ec588a03302e", + "manifest_sha256": "5e02416f7b0dbc0548dbf72c93c3356300726c280d7667bfbc1e205c80bc9ba2", + "input_config_sha256": "c3aab71fe88ba6a3bba618c56934acbc43d6aa3d34027a1ff3d1f69d0523cc91" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-cli", + "result_path": "jobs/runtime-source/jobs/dev3-1017-install-only-recovery-20261020/pytorch-model-cli__7S8845w/result.json", + "result_sha256": "a1129937e99f5816fa94d27f1bd220111fe86b74a32d566a832fd5adc4210dca", + "bootstrap_sha256": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "manifest_sha256": "5e02416f7b0dbc0548dbf72c93c3356300726c280d7667bfbc1e205c80bc9ba2", + "input_config_sha256": "c3aab71fe88ba6a3bba618c56934acbc43d6aa3d34027a1ff3d1f69d0523cc91" + }, + { + "arm": "recovery", + "task": "terminal-bench/pytorch-model-recovery", + "result_path": "jobs/runtime-source/jobs/dev3-1017-install-only-recovery-20261020/pytorch-model-recovery__8JmHZuF/result.json", + "result_sha256": "11fadb581e74540d384ad8758192e2787f0163783fc99c91f82761d86ba4c300", + "bootstrap_sha256": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "manifest_sha256": "5e02416f7b0dbc0548dbf72c93c3356300726c280d7667bfbc1e205c80bc9ba2", + "input_config_sha256": "c3aab71fe88ba6a3bba618c56934acbc43d6aa3d34027a1ff3d1f69d0523cc91" + } + ], + "integrity": { + "original_baseline_and_evidence_unchanged": true, + "all_frozen_agent_outputs_unchanged": true, + "native_cpu_and_memory_limits_match": true, + "all_task_image_identities_match": true, + "all_three_containers_retained": true, + "all_temporary_image_cache_copies_restored": true + } +} From 34dedc4f6da3b512a1b7a5e9d22fa79738ec57ff Mon Sep 17 00:00:00 2001 From: minixalpha Date: Wed, 21 Oct 2026 07:05:41 -0700 Subject: [PATCH 09/12] feat(benchmarks): verify retained solutions in separate batches --- benchmarks/harbor/README.md | 23 ++ benchmarks/harbor/README.zh-CN.md | 19 ++ ...orch-model-recovery-verifier-20261021.json | 21 ++ ...ytorch-model-recovery-verifier-20261021.md | 76 +++++ ...ytorch-model-recovery-verifier-20261021.md | 63 ++++ ...orch-model-recovery-verifier-20261021.json | 308 ++++++++++++++++++ .../harbor/src/harbor_adapter/__main__.py | 11 + .../src/harbor_adapter/_verifier_runner.py | 38 ++- .../harbor/src/harbor_adapter/retained.py | 179 ++++++++++ benchmarks/harbor/tests/test_retained.py | 139 ++++++++ 10 files changed, 876 insertions(+), 1 deletion(-) create mode 100644 benchmarks/harbor/configs/pytorch-model-recovery-verifier-20261021.json create mode 100644 benchmarks/harbor/reports/en/pytorch-model-recovery-verifier-20261021.md create mode 100644 benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-verifier-20261021.md create mode 100644 benchmarks/harbor/results/pytorch-model-recovery-verifier-20261021.json create mode 100644 benchmarks/harbor/src/harbor_adapter/retained.py create mode 100644 benchmarks/harbor/tests/test_retained.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 39cb137..5cf2dd7 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -311,6 +311,29 @@ the registered files before verification. Keep both the artifact manifest and file hashes with the run evidence; trajectories alone do not preserve binaries or the original system state. +For an explicitly authorized additional verifier batch on a retained Docker +trial, use the repository entry point without starting an agent: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter verify-retained \ + --trial-dir jobs/example/task__trial \ + --task-dir /path/to/the/original/task \ + --container task__trial__env-main-1 \ + --output-dir jobs/example-additional-verifier-batch +``` + +All original project services must be stopped. The command starts those same +containers, waits for service health checks, verifies frozen `/app` artifacts +and original test hashes, and runs at most three attempts with the native task +timeout. The optional `--retry-dependency-errors` and +`--retry-collection-errors` flags require the same separate authorization as +their verifier kwargs. Any completed score, including zero, stops the batch. +The original trial logs are temporarily set aside inside the retained container +and restored afterward. New attempt logs, implementation sources, policy, +hashes, and integrity checks go into the separate output directory. Containers +are stopped and retained after the batch. This does not change the original +trial result or authorize repeated batches automatically. + ## Benchmark reports Reports are available in [English](reports/en/) and diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 5ecd9b8..dd01a5e 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -250,6 +250,25 @@ Harbor 原生的 provider 凭证和已经配置的 base URL 也会转换为 nano 容器。Harbor 会在验证前收集已登记的文件。 将产物清单和文件哈希与运行证据一起保存；仅有轨迹无法保留二进制文件和原始系统状态。 +对于明确授权、使用已保留 Docker trial 的额外 verifier 批次，可通过仓库入口执行， +无需启动 agent： + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter verify-retained \ + --trial-dir jobs/example/task__trial \ + --task-dir /path/to/the/original/task \ + --container task__trial__env-main-1 \ + --output-dir jobs/example-additional-verifier-batch +``` + +原项目的所有服务必须处于停止状态。该命令启动原来的容器，等待服务健康检查通过， +核对冻结的 `/app` 产物和原始测试的哈希，并使用题目原生时限执行最多三次尝试。 +可选的 `--retry-dependency-errors` 和 `--retry-collection-errors` 标志与对应的 +verifier 参数一样，需要单独授权。任何已完成的评分都会结束该批次，包括得分为 0。 +原 trial 日志会暂存在保留容器内，并在结束后恢复。新增尝试日志、执行源码、策略、 +哈希和完整性检查写入单独的输出目录。批次结束后，容器会被停止并继续保留。 +该命令不会修改原 trial 的结果，也不会自动授权后续重复批次。 + ## Benchmark 报告 报告提供[英文](reports/en/)和[简体中文](reports/zh-CN/)两个版本，文件名一一对应。 diff --git a/benchmarks/harbor/configs/pytorch-model-recovery-verifier-20261021.json b/benchmarks/harbor/configs/pytorch-model-recovery-verifier-20261021.json new file mode 100644 index 0000000..1abab61 --- /dev/null +++ b/benchmarks/harbor/configs/pytorch-model-recovery-verifier-20261021.json @@ -0,0 +1,21 @@ +{ + "kind": "separately authorized verifier-only continuation", + "authorization": "User requested three additional verifier attempts for pytorch-model-recovery on 2026-10-21.", + "original_supplement": "benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json", + "original_supplement_sha256": "37249473185caa2980fd5a322a375ef5da6bde1a7dd43e2964160def218a93b6", + "trial_dir": "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H", + "task_dir": "/home/minix/.cache/harbor/tasks/packages/terminal-bench/pytorch-model-recovery/2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "container": "pytorch-model-recovery__ncs7y4h__env-main-1", + "output_dir": "jobs/pytorch-model-recovery-verifier-20261021", + "model_calls": 0, + "same_agent_output": true, + "max_attempts_this_batch": 3, + "previous_attempts_on_this_output": 3, + "native_timeout_seconds_per_attempt": 900, + "retry_dependency_errors": true, + "retry_collection_errors": true, + "stop_after_any_completed_score": true, + "dependency_profile": "not_configured; previous native dependency preflight is not a verifier dependency preflight", + "keep_container": true, + "preserve_original_trial_evidence": true +} diff --git a/benchmarks/harbor/reports/en/pytorch-model-recovery-verifier-20261021.md b/benchmarks/harbor/reports/en/pytorch-model-recovery-verifier-20261021.md new file mode 100644 index 0000000..435c311 --- /dev/null +++ b/benchmarks/harbor/reports/en/pytorch-model-recovery-verifier-20261021.md @@ -0,0 +1,76 @@ +# PyTorch model recovery: retained-solution verifier continuation + +Date: 2026-10-21. This separately authorized batch retries the existing +`pytorch-model-recovery__nCs7Y4H` solution from the +[October 20 supplement](dev34-flash1017-recovery-20261020.md). + +**All three additional verifier attempts timed out before pytest started. The +score remains `null`: this is a verifier error, with no scored pass or failure.** +No model was called and no new solution was generated. This solution has now +received six verifier attempts across two separately recorded batches. + +| Batch attempt | Attempt on this solution | Result | Duration | Last observed stage | +| --- | --- | --- | --- | --- | +| 1 | 4 | Timeout; unscored | 900.002 s | PyTorch/CUDA downloads | +| 2 | 5 | Timeout; unscored | 900.002 s | Torch download after CUDA dependencies completed | +| 3 | 6 | Timeout; unscored | 900.002 s | Torch download; completed dependencies reused | + +The new verifier execution time was approximately 45 minutes. No attempt +produced a CTRF test report or valid reward. New model calls and model cost were +both zero. The original dev34 baseline remains **26 passes, five scored +failures, three unscored verifier errors**. Neither that baseline nor the +October 20 supplement was rewritten, and no composite dev34 score is reported. + +## Policy and preserved identity + +The batch used the repository's reusable `verify-retained` command, the same +Docker container, the same `/app` solution, the original test scripts and package +sources, and the task's native 900-second per-attempt limit. The separately +authorized budget was at most three additional attempts, with opt-in dependency +download and collection recovery. Any completed valid score, including zero, +would have stopped the batch. Whole-trial retries and model calls were disabled. + +The original solution used core agent commit +`ce28b1e826a292d796ba64f13e79199b66f65758` and runtime commit +`40fdd041c370a96dc06f86b436eacc01ce1566d7`. Its recorded model identity is +DeepSeek-V4.1-Flash-1017 through the mutable `deepseek-flash` alias; this batch +makes no new claim about the alias's current mapping. The frozen solution +`/app/model.pt` SHA256 remains +`098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e`. + +No fresh agent setup or dependency preflight was run. This task has no registered +verifier dependency profile; one is not claimed to have passed. + +## Storage and execution observations + +The first attempt briefly reported APT cache proxy connection refusals during +service startup, used existing indexes and installed curl, then entered the +original uv downloads. Both sidecars were healthy for the later attempts. +The published host command now waits for service health checks. The actual +executed source snapshot is archived; this hardening did not change the running +attempt runner, time limits, solution, or retry policy. + +To prevent disk exhaustion, 50 duplicate image-cache files totaling +1,576,934,879 bytes were temporarily relocated and subsequently restored with +SHA256 verification. Separately, 37 abandoned partial-download directories +totaling 5,753,708,793 bytes were archived after checking timestamps, open file +handles, and hashes. Their archives remain under the two dated +`/tmp/nanopy-pytorch-recovery-20261021-stale-downloads*` directories. Active +downloads, completed dependency caches, original Docker images, and task files +were not changed by this maintenance. + +## Evidence and validation + +The original trial files were restored byte for byte. Frozen solution and test +hashes passed checks before and after verification. All three original project +containers are stopped and retained, with unchanged container and image +identities and no container OOM flag. The local Harbor suite passed **203 tests**, +including log restoration, cancellation, service readiness, and valid-zero +handling. + +- [Machine-readable result](../../results/pytorch-model-recovery-verifier-20261021.json) +- [Registered continuation plan](../../configs/pytorch-model-recovery-verifier-20261021.json) +- Local raw evidence: `jobs/pytorch-model-recovery-verifier-20261021/`, including + registration, executed sources, every attempt's logs, integrity checks, + container retention, storage history, and partial-download archive manifests. +- [Complete Simplified Chinese version](../zh-CN/pytorch-model-recovery-verifier-20261021.md) diff --git a/benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-verifier-20261021.md b/benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-verifier-20261021.md new file mode 100644 index 0000000..ac6e4b6 --- /dev/null +++ b/benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-verifier-20261021.md @@ -0,0 +1,63 @@ +# PyTorch 模型恢复：保留解答的 verifier 追加验证 + +日期：2026-10-21。本批次经过单独授权，继续验证 +[10 月 20 日补充实验](dev34-flash1017-recovery-20261020.md)中保留的 +`pytorch-model-recovery__nCs7Y4H` 解答。 + +**新增 3 次 verifier 尝试全部在 pytest 启动前超时。评分仍为 `null`：这是 +verifier 错误，尚无有效的成功或失败评分。** 本次没有调用模型，也没有生成新解答。 +同一份解答在两个分开记录的批次中，累计接受了 6 次 verifier 尝试。 + +| 本批次尝试 | 此解答累计尝试 | 结果 | 耗时 | 最后观察到的阶段 | +| --- | --- | --- | --- | --- | +| 1 | 4 | 超时；未评分 | 900.002 秒 | 下载 PyTorch/CUDA 依赖 | +| 2 | 5 | 超时；未评分 | 900.002 秒 | CUDA 依赖完成后仍在下载 Torch | +| 3 | 6 | 超时；未评分 | 900.002 秒 | 复用已完成的依赖，仍在下载 Torch | + +新增 verifier 执行耗时约 45 分钟。没有任何一次产生 CTRF 测试报告或有效评分。 +新增模型调用次数及模型费用均为零。原 dev34 baseline 仍为 **26 题成功、5 题有效 +评分失败、3 题 verifier 错误未评分**。原 baseline 和 10 月 20 日补充实验均未被 +改写，也不报告混合后的 dev34 分数。 + +## 策略与身份保留 + +本批次通过仓库内可复用的 `verify-retained` 命令执行，使用同一 Docker 容器、同一 +份 `/app` 解答、原始测试脚本及包来源，并保留题目原生的每次 900 秒时限。单独授权 +的预算为最多新增 3 次尝试，并显式启用依赖下载及测试收集错误恢复。任何有效的已 +完成评分，包括 0 分，都会立即结束批次。没有整题重试，也没有模型调用。 + +原解答使用核心 agent 提交 +`ce28b1e826a292d796ba64f13e79199b66f65758`，运行时提交为 +`40fdd041c370a96dc06f86b436eacc01ce1566d7`。其已记录的模型身份是通过可变别名 +`deepseek-flash` 使用的 DeepSeek-V4.1-Flash-1017；本批次没有对该别名当前指向作出 +新的断言。冻结解答 `/app/model.pt` 的 SHA256 仍为 +`098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e`。 + +本次没有重新执行 agent 安装或依赖预检。该题没有已登记的 verifier 依赖配置， +因此没有声称这类依赖预检已通过。 + +## 磁盘维护与执行观察 + +第 1 次尝试在服务启动时短暂出现 APT 缓存代理连接被拒绝，随后使用已有索引和已 +安装的 curl，进入原始 uv 下载流程。后续尝试中，两个 sidecar 均已健康运行。 +已发布的宿主命令现在会等待服务健康检查。实际执行的源码快照已归档；这一加固 +没有改变正在运行的尝试执行器、时限、解答或重试策略。 + +为避免磁盘耗尽，临时迁移了 50 个重复镜像缓存文件，共 1,576,934,879 字节；之后 +已全部恢复，并通过 SHA256 校验。另外，在核对时间戳、打开的文件句柄和哈希后， +归档了 37 个已废弃的下载临时目录，共 5,753,708,793 字节。归档保存在两个带日期 +的 `/tmp/nanopy-pytorch-recovery-20261021-stale-downloads*` 目录中。维护操作没有 +更改正在进行的下载、已完成的依赖缓存、原始 Docker 镜像或任务文件。 + +## 证据与验证 + +原 trial 文件已逐字节恢复。冻结解答和测试的哈希在验证前后均通过核对。原项目的 +3 个容器均已停止并继续保留，容器及镜像身份不变，且没有容器 OOM 标记。本地 +Harbor 测试套件 **203 项通过**，覆盖日志恢复、取消执行、服务就绪及有效 0 分处理。 + +- [机器可读结果](../../results/pytorch-model-recovery-verifier-20261021.json) +- [已登记的追加验证计划](../../configs/pytorch-model-recovery-verifier-20261021.json) +- 本地原始证据：`jobs/pytorch-model-recovery-verifier-20261021/`，包括登记记录、 + 实际执行的源码、每次尝试日志、完整性检查、容器保留记录、磁盘历史和下载临时文件 + 归档清单。 +- [完整英文版本](../en/pytorch-model-recovery-verifier-20261021.md) diff --git a/benchmarks/harbor/results/pytorch-model-recovery-verifier-20261021.json b/benchmarks/harbor/results/pytorch-model-recovery-verifier-20261021.json new file mode 100644 index 0000000..2be5141 --- /dev/null +++ b/benchmarks/harbor/results/pytorch-model-recovery-verifier-20261021.json @@ -0,0 +1,308 @@ +{ + "schema_version": 1, + "experiment": "pytorch-model-recovery-verifier-20261021", + "collected_at": "2026-10-21T14:04:39.704238+00:00", + "kind": "Separately authorized verifier-only continuation of the retained 2026-10-20 solution.", + "task": "terminal-bench/pytorch-model-recovery", + "trial": "pytorch-model-recovery__nCs7Y4H", + "task_ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "model_for_original_solution": "deepseek/deepseek-flash", + "provider_checkpoint_for_original_solution": { + "api_model": "deepseek-flash", + "reported_checkpoint": "DeepSeek-V4.1-Flash-1017", + "verification": "Official documentation checked before execution maps deepseek-flash to this checkpoint; the API alias is not an immutable checkpoint pin.", + "checked_at": "2026-10-20T13:42:17.189959+00:00", + "sources": [ + "https://api-docs.deepseek.com/quick_start/agent_integrations/", + "https://api-docs.deepseek.com/news/news261017/" + ] + }, + "core_agent_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "original_runtime_source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "new_model_calls": 0, + "new_model_cost_usd": 0, + "prior_supplement": { + "path": "benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json", + "sha256": "37249473185caa2980fd5a322a375ef5da6bde1a7dd43e2964160def218a93b6", + "original_verifier_attempts": 3, + "original_reward": null + }, + "plan": { + "path": "benchmarks/harbor/configs/pytorch-model-recovery-verifier-20261021.json", + "sha256": "5ce1cc0ea847209b32c1216f0c0dd6a9f6a1f07855d7cc75c8a9f04d16f903c6" + }, + "policy": { + "native_timeout_seconds": 900.0, + "max_attempts": 3, + "retry_dependency_errors": true, + "retry_collection_errors": true + }, + "verifier_result": { + "status": "verifier_error", + "rewards": null, + "last_attempt": { + "attempt": 3, + "status": "timeout", + "duration_seconds": 900.0021111120004 + }, + "error": "timeout" + }, + "attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0023519049864, + "overall_attempt_on_this_solution": 4, + "raw_archived_reward": null, + "pytest_session_observed": false, + "ctrf_summary": null, + "log_sha256": { + "test-stdout.txt": "d995ee6332259ba48b14ca31fb499580871ba5c59fc64aff160e5b917a73898b" + } + }, + { + "attempt": 2, + "status": "timeout", + "duration_seconds": 900.0024621869961, + "overall_attempt_on_this_solution": 5, + "raw_archived_reward": null, + "pytest_session_observed": false, + "ctrf_summary": null, + "log_sha256": { + "test-stdout.txt": "55c18cf5e336d18234b1efaed4ca4c99d1af22abe6355b38931be0603a10ee53" + } + }, + { + "attempt": 3, + "status": "timeout", + "duration_seconds": 900.0021111120004, + "overall_attempt_on_this_solution": 6, + "raw_archived_reward": null, + "pytest_session_observed": false, + "ctrf_summary": null, + "log_sha256": { + "test-stdout.txt": "bb67828a56c69a8b469f3b21ec09f908ba73eba916ef167d27c507f652756550" + } + } + ], + "overall_verifier_attempts_on_this_solution": 6, + "verifier_seconds_this_batch": 2700.006925203983, + "retained_container": [ + { + "id": "85ece21e3a35b6db0e4452943ead6ad0581681a74051ad71d1e8579784189a50", + "name": "pytorch-model-recovery__ncs7y4h__env-apt-index-cache-1", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-21T13:17:06.610491279Z", + "FinishedAt": "2026-10-21T14:02:26.45335586Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-21T07:02:21.279207299-07:00", + "End": "2026-10-21T07:02:21.375029931-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:22.376356703-07:00", + "End": "2026-10-21T07:02:22.475384192-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:23.476451432-07:00", + "End": "2026-10-21T07:02:23.574333533-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:24.574996172-07:00", + "End": "2026-10-21T07:02:24.680012342-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:25.680436283-07:00", + "End": "2026-10-21T07:02:25.775505581-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "id": "c0f5a78624e44c79f078c5dd77fc6a925c0a435611941374206d77283c0d7976", + "name": "pytorch-model-recovery__ncs7y4h__env-uv-release-cache-1", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-21T13:17:06.687241526Z", + "FinishedAt": "2026-10-21T14:02:16.343893305Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-21T07:02:11.159655249-07:00", + "End": "2026-10-21T07:02:11.261993952-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:12.262851803-07:00", + "End": "2026-10-21T07:02:12.355403643-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:13.356021632-07:00", + "End": "2026-10-21T07:02:13.458588952-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:14.459721514-07:00", + "End": "2026-10-21T07:02:14.568135008-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-21T07:02:15.56934414-07:00", + "End": "2026-10-21T07:02:15.659450418-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "id": "e8f4a6bb804bc8ef8f5e0acde059fd6205efc9cd4966b0029149873af31619fd", + "name": "pytorch-model-recovery__ncs7y4h__env-main-1", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-21T13:17:06.50181432Z", + "FinishedAt": "2026-10-21T14:02:36.563205749Z" + } + } + ], + "same_container_id": "e8f4a6bb804bc8ef8f5e0acde059fd6205efc9cd4966b0029149873af31619fd", + "same_image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "solution_model_sha256": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "frozen_solution_and_original_tests_sha256": { + "/app/__pycache__/recovered_model.cpython-313.pyc": "c492a45a453bba7c9fc0db2bce81b3bac895ca649841b91e2919e13f4ac59071", + "/app/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "/app/model.pt": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "/app/recovered_model.py": "602526444f3517d52c011a3291c3b492d6a86b4bd5200011b4027bc2ee86f3f7", + "/app/tune_model.py": "e2e9e6bbdb6e13f9e7e57d28f8c4e66932bfeff8dc468df2ae2034b8b9e92e71", + "/app/verify.py": "ed8ec4d7d52bbd849c15f8acc066825af6354fb729f265e9d223307835381c9e", + "/app/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "/tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "/tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "/tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + }, + "integrity": { + "original_trial_unchanged": true, + "frozen_solution_unchanged": true, + "original_tests_unchanged": true, + "finished_at": "2026-10-21T14:02:06.276461+00:00" + }, + "original_trial_sha256": { + "agent/bootstrap.json": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "agent/nanopycodeagent.txt": "036c564ae66e993149b0a73d6de0e24292f3b06331b035c929d109e019d5025a", + "agent/trajectory.json": "73b4f374dbfb89dd3d3b94d572136c2f613bc1666aad87b8a6454c3316bb3f3d", + "artifacts/app/__pycache__/recovered_model.cpython-313.pyc": "c492a45a453bba7c9fc0db2bce81b3bac895ca649841b91e2919e13f4ac59071", + "artifacts/app/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "artifacts/app/model.pt": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "artifacts/app/recovered_model.py": "602526444f3517d52c011a3291c3b492d6a86b4bd5200011b4027bc2ee86f3f7", + "artifacts/app/tune_model.py": "e2e9e6bbdb6e13f9e7e57d28f8c4e66932bfeff8dc468df2ae2034b8b9e92e71", + "artifacts/app/verify.py": "ed8ec4d7d52bbd849c15f8acc066825af6354fb729f265e9d223307835381c9e", + "artifacts/app/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6", + "config.json": "125c1fdecd4add7009fa787adb79226e020e0f8598b12413864ac3c8f3502c83", + "exception.txt": "56305954e60464b84bd02c84ce5d5373a0bab576da8c5d926ed395a9978e7f95", + "lock.json": "5dc26291064142102bbfc4f3a90176a232d1824518915e9ad6da782c755635f5", + "result.json": "2a61cc964ad2a684e2b4adb3e5d333f75093db51320f7e448a2cb83446998e34", + "trial.log": "6504d89623d40eeab818172245fa3795d77f6349871df0f88d363e6674cde42e", + "verifier/attempts/1/test-stdout.txt": "4d00e96829b6291622f1a138c491042c44d5d2790880e9253ac72d363d906864", + "verifier/attempts/2/test-stdout.txt": "e951235c90184be58c1908b676fd33ddbf8e9199c8fd61bf416f0d1e80934690", + "verifier/attempts/3/test-stdout.txt": "034e509e6d5e1e587085482f987d108a25588d854b1404f6f0a3b424ba2c29d8", + "verifier/retry-summary.json": "8609c874195c9b06450a4a0667270aac906444ee1adfa675f486aa19450b55ea", + "verifier/test-stdout.txt": "034e509e6d5e1e587085482f987d108a25588d854b1404f6f0a3b424ba2c29d8" + }, + "executed_implementation_sha256": { + "retained.py": "8d7641607c660bf50d1104228d5a15f2a2b8ec04761c52e9445338217e125dad", + "_verifier_runner.py": "be2f51d137fe97b65d8e1ef4e0134fd6f3f1f379f234e1de3f3a705d3a28a632" + }, + "implementation_note": "The executed source snapshot is archived. Service health waiting and result classification were hardened in the published host command after this batch started; the in-container attempt runner and attempt policy were unchanged.", + "startup_observation": "Attempt 1 reported brief APT cache proxy connection refusals at service startup, used existing package indexes and installed curl, then reached the original uv downloads. Later attempts ran after both sidecars became healthy.", + "storage": { + "minimum_sampled_root_free_bytes": 2745761792, + "verified_duplicate_image_cache_bytes_temporarily_relocated": 1576934879, + "image_cache_files_restored": 50, + "image_cache_restoration_hashes_verified": true, + "stale_partial_download_bytes_archived": 5753708793, + "stale_partial_download_directories": 37, + "stale_archive_dirs": [ + "/tmp/nanopy-pytorch-recovery-20261021-stale-downloads-after-attempt2", + "/tmp/nanopy-pytorch-recovery-20261021-stale-downloads" + ], + "stale_archive_hashes_verified": true, + "active_downloads_and_completed_packages_modified_by_maintenance": false + }, + "documentation_sync": { + "english_report": "benchmarks/harbor/reports/en/pytorch-model-recovery-verifier-20261021.md", + "chinese_report": "benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-verifier-20261021.md", + "status": "full report pair" + }, + "evidence_sha256": { + "jobs/pytorch-model-recovery-verifier-20261021/completion.json": "da28436999984826b327a43833536ae1a6f6c005bfdfa2493ff648ec1616ecc6", + "jobs/pytorch-model-recovery-verifier-20261021/container-retention.json": "712d18ee20a4ebc53d9acfb7880b12285990cc76d982f56bf02e105c255e0602", + "jobs/pytorch-model-recovery-verifier-20261021/integrity.json": "3313ccb0479dc64bf79c523ff70115bc7c501563753e634c1e17394f1b559ae5", + "jobs/pytorch-model-recovery-verifier-20261021/registration.json": "3bcff0ae00373f1e0291dd2d50b8acb636c9a9d2288913bf076e73330cddb09e", + "jobs/pytorch-model-recovery-verifier-20261021/result.json": "06ff86ee7d172ebf2460e981202f1d290d973d140a42d5a01de7a7baea62102d", + "jobs/pytorch-model-recovery-verifier-20261021/runner-stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/pytorch-model-recovery-verifier-20261021/runner-stdout.txt": "5fdd1ea303c5803ab38bbd197634ce0afca04301900c828abf7355a67a66eb79", + "jobs/pytorch-model-recovery-verifier-20261021/source/_verifier_runner.py": "be2f51d137fe97b65d8e1ef4e0134fd6f3f1f379f234e1de3f3a705d3a28a632", + "jobs/pytorch-model-recovery-verifier-20261021/source/retained.py": "8d7641607c660bf50d1104228d5a15f2a2b8ec04761c52e9445338217e125dad", + "jobs/pytorch-model-recovery-verifier-20261021/stale-download-archive-after-attempt2.json": "ee803db9c1a629f5bf40fe6eb6f1bc8961be35ab686bbdebadf4bc5c455cb66e", + "jobs/pytorch-model-recovery-verifier-20261021/stale-download-archive.json": "315f82f1dfd51bc2af2221a22b36a61fc45792bd3ea76c5cc1a858ce46f6f2d2", + "jobs/pytorch-model-recovery-verifier-20261021/storage-and-phase-history.jsonl": "9cd5ec31c5f92ee3d175a77ea5d7275d38f865c66cb89d9c8d985580672d52fe", + "jobs/pytorch-model-recovery-verifier-20261021/verifier/attempts/1/test-stdout.txt": "d995ee6332259ba48b14ca31fb499580871ba5c59fc64aff160e5b917a73898b", + "jobs/pytorch-model-recovery-verifier-20261021/verifier/attempts/2/test-stdout.txt": "55c18cf5e336d18234b1efaed4ca4c99d1af22abe6355b38931be0603a10ee53", + "jobs/pytorch-model-recovery-verifier-20261021/verifier/attempts/3/test-stdout.txt": "bb67828a56c69a8b469f3b21ec09f908ba73eba916ef167d27c507f652756550", + "jobs/pytorch-model-recovery-verifier-20261021/verifier/retry-summary.json": "5effb74466c22e41e4f4b59a5d8ec0ec33dd79b325fc219cabac1d889fee4656", + "jobs/pytorch-model-recovery-verifier-20261021/verifier/test-stdout.txt": "bb67828a56c69a8b469f3b21ec09f908ba73eba916ef167d27c507f652756550", + "jobs/pytorch-model-recovery-verifier-20261021/working-tree.patch": "c03b33c3b5c15899163909c5239aa228cfdaa8196391c6c5ea57cc6d0bcb370f" + }, + "scope_note": "Both the 2026-10-19 dev34 baseline and the 2026-10-20 three-task supplement remain unchanged. This batch does not create a new Harbor native trial result or a composite dev34 score." +} diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py index d41cf39..e665df5 100644 --- a/benchmarks/harbor/src/harbor_adapter/__main__.py +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -170,7 +170,18 @@ def main() -> int: cache = commands.add_parser("import-apt", help="Import a manifest and its sibling packages/ directory") cache.add_argument("manifest", type=Path) cache.add_argument("--cache-dir", type=Path, default=default_cache_dir()) + retained = commands.add_parser("verify-retained", help="Run an authorized verifier batch on an unchanged retained solution, without model calls") + retained.add_argument("--trial-dir", type=Path, required=True) + retained.add_argument("--task-dir", type=Path, required=True) + retained.add_argument("--container", required=True) + retained.add_argument("--output-dir", type=Path, required=True) + retained.add_argument("--max-attempts", type=int, default=3) + retained.add_argument("--retry-dependency-errors", action="store_true") + retained.add_argument("--retry-collection-errors", action="store_true") args = parser.parse_args() + if args.command == "verify-retained": + from .retained import verify_retained + return verify_retained(args) if args.command == "import-apt": count = AptCache(args.cache_dir.expanduser().resolve()).import_manifest(args.manifest) print(f"Imported {count} verified APT packages into {args.cache_dir}") diff --git a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py index ce3e766..3c78b74 100644 --- a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py +++ b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py @@ -202,11 +202,45 @@ def cancel(control): raise RuntimeError("Verifier runner did not stop after cancellation") +def run_batch(command, timeout_sec, max_attempts, logs, control, archive, + retry_dependency_errors=False, retry_collection_errors=False): + """Keep a new batch separate and restore the previous verifier evidence.""" + logs, archive = Path(logs).resolve(), Path(archive).resolve() + if logs == archive or logs in archive.parents or archive in logs.parents: + raise ValueError("Batch archive must be outside the verifier directory") + archive.mkdir(parents=True, exist_ok=False) + original, current = archive / "original", archive / "verifier" + original.mkdir() + current.mkdir() + moved = [] + try: + for path in list(logs.iterdir()): + shutil.move(str(path), original / path.name) + moved.append(path.name) + except BaseException: + for name in moved: + shutil.move(str(original / name), logs / name) + raise + try: + return run(command, timeout_sec, max_attempts, logs, control, + retry_dependency_errors, retry_collection_errors) + finally: + try: + for path in list(logs.iterdir()): + shutil.move(str(path), current / path.name) + finally: + for name in moved: + shutil.move(str(original / name), logs / name) + + if __name__ == "__main__": if sys.argv[1] == "cancel": cancel(sys.argv[2]) else: arguments = sys.argv[1:] + archive = arguments.pop(1) if arguments[0] == "batch" else None + if archive is not None: + arguments.pop(0) if len(arguments) == 5: command, timeout, count, logs, control = arguments retry_dependencies = False @@ -219,7 +253,9 @@ def cancel(control): command, timeout, count, retry, collection, logs, control = arguments retry_dependencies = retry == "1" retry_collection = collection == "1" - print(json.dumps(run( + runner = run if archive is None else lambda *args: run_batch( + *args[:5], archive, *args[5:]) + print(json.dumps(runner( command, float(timeout), int(count), logs, control, retry_dependencies, retry_collection, ))) diff --git a/benchmarks/harbor/src/harbor_adapter/retained.py b/benchmarks/harbor/src/harbor_adapter/retained.py new file mode 100644 index 0000000..742370b --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/retained.py @@ -0,0 +1,179 @@ +"""Verify an unchanged solution in a stopped, retained Docker trial.""" + +from datetime import datetime, timezone +import hashlib +import json +import math +from pathlib import Path +import subprocess +import time +import uuid + +from harbor.models.task.config import TaskOS +from harbor.models.task.task import Task +from harbor.utils.scripts import build_execution_command + +from .verifier import _RUNNER + + +def hashes(root): + return {p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(root.rglob("*")) if p.is_file()} + + +def docker(*args): + return subprocess.check_output(["docker", *map(str, args)], text=True) + + +def check_container_files(container, expected): + script = ( + "import hashlib,json,sys; from pathlib import Path; " + "expected=json.loads(sys.argv[1]); " + "actual={p:hashlib.sha256(Path(p).read_bytes()).hexdigest() for p in expected}; " + "assert actual==expected, 'Retained task files differ from frozen evidence'" + ) + docker("exec", "--user", "root", container, "python3", "-c", script, + json.dumps(expected)) + + +def wait_for_services(services, timeout=30): + deadline = time.monotonic() + timeout + while True: + states = [m["State"] for m in json.loads(docker("inspect", *services))] + if any(not s["Running"] or s.get("Health", {}).get("Status") == "unhealthy" + for s in states): + raise RuntimeError("A retained project service failed to start") + if all(s.get("Health", {}).get("Status", "healthy") == "healthy" for s in states): + return + if time.monotonic() >= deadline: + raise TimeoutError("Retained project health checks did not become ready") + time.sleep(0.2) + + +def batch_result(output, record): + result = {"status": "verifier_error", "rewards": None, + "last_attempt": record} + if record["status"] != "completed": + result["error"] = record["status"] + return result + try: + logs = output / "verifier" + if (logs / "reward.txt").exists(): + rewards = {"reward": float((logs / "reward.txt").read_text())} + else: + rewards = json.loads((logs / "reward.json").read_text()) + if not isinstance(rewards, dict) or not rewards or any( + type(value) not in (int, float) or not math.isfinite(value) + for value in rewards.values() + ): + raise ValueError("Rewards must be a nonempty mapping of finite numbers") + except (OSError, ValueError, TypeError) as exc: + result["error"] = type(exc).__name__ + return result + return result | {"status": "scored", "rewards": rewards} + + +def verify_retained(args): + """Run no agent, preserve native evidence, and stop on any completed score.""" + trial, task_dir, output = (p.resolve() for p in + (args.trial_dir, args.task_dir, args.output_dir)) + if trial == output or trial in output.parents or output in trial.parents: + raise ValueError("Use an output directory outside the original trial") + if not 1 <= args.max_attempts <= 3: + raise ValueError("max_attempts must be between 1 and 3 per authorized batch") + task = Task(task_dir) + if task.config.environment.os != TaskOS.LINUX: + raise ValueError("Retained verification supports Linux tasks") + timeout = task.config.verifier.timeout_sec + if not 0 < timeout < float("inf"): + raise ValueError("A finite, positive native verifier timeout is required") + info, = json.loads(docker("inspect", args.container)) + if info["State"]["Running"]: + raise ValueError("Stop the retained trial before verifying it") + mounts = {m["Destination"]: m["Source"] for m in info["Mounts"]} + if Path(mounts.get("/logs/verifier", "")).resolve() != trial / "verifier": + raise ValueError("Container verifier mount does not match the original trial") + project = info["Config"]["Labels"]["com.docker.compose.project"] + services = docker("ps", "-aq", "--filter", f"label=com.docker.compose.project={project}").split() + members = json.loads(docker("inspect", *services)) + if any(m["State"]["Running"] for m in members): + raise ValueError("All retained project services must be stopped") + expected = {"/app/" + p: digest for p, digest in hashes(trial / "artifacts/app").items()} + if not expected: + raise ValueError("Retained verification requires frozen /app artifacts") + expected.update({"/tests/" + p: digest for p, digest in hashes(task_dir / "tests").items()}) + original = hashes(trial) + output.mkdir(parents=True, exist_ok=False) + source = Path(__file__).parent + manifest = { + "started_at": datetime.now(timezone.utc).isoformat(), + "kind": "separate retained-solution verifier batch", "model_calls": 0, + "trial_dir": str(trial), "task_dir": str(task_dir), + "task_toml_sha256": hashlib.sha256((task_dir / "task.toml").read_bytes()).hexdigest(), + "container_id": info["Id"], "image_id": info["Image"], + "project_services": services, "native_timeout_seconds": timeout, + "max_attempts": args.max_attempts, + "retry_dependency_errors": args.retry_dependency_errors, + "retry_collection_errors": args.retry_collection_errors, + "original_trial_sha256": original, "expected_container_sha256": expected, + "source_sha256": {name: hashlib.sha256((source / name).read_bytes()).hexdigest() + for name in ("retained.py", "_verifier_runner.py")}, + } + archive = f"/tmp/nanopy-retained-{uuid.uuid4().hex}" + control = archive + ".pid" + manifest["container_batch_archive"] = archive + (output / "source").mkdir() + for name in manifest["source_sha256"]: + (output / "source" / name).write_bytes((source / name).read_bytes()) + (output / "registration.json").write_text(json.dumps(manifest, indent=2) + "\n") + command = build_execution_command( + "/tests/" + task.paths.test_path.relative_to(task.paths.tests_dir).as_posix(), + stdout_path="/logs/verifier/test-stdout.txt", task_os=TaskOS.LINUX, + ) + started = [] + try: + for container in [info["Id"], *[s for s in services if s != info["Id"]]]: + # docker ps returns short IDs; compare against inspected IDs. + if any(previous.startswith(container) for previous in started): + continue + docker("start", container) + started.append(container) + wait_for_services(services) + check_container_files(info["Id"], expected) + with (output / "runner-stdout.txt").open("w") as stdout, (output / "runner-stderr.txt").open("w") as stderr: + process = subprocess.Popen([ + "docker", "exec", "--user", "root", info["Id"], "python3", "-c", _RUNNER, + "batch", archive, command, str(timeout), str(args.max_attempts), + str(int(args.retry_dependency_errors)), str(int(args.retry_collection_errors)), + "/logs/verifier", control, + ], stdout=stdout, stderr=stderr) + try: + code = process.wait(timeout=timeout * args.max_attempts + 120) + except BaseException: + docker("exec", "--user", "root", info["Id"], "python3", "-c", _RUNNER, "cancel", control) + process.wait(timeout=30) + raise + docker("cp", info["Id"] + ":" + archive + "/verifier", output / "verifier") + if code: + raise RuntimeError(f"Verifier runner exited {code}; see {output}") + check_container_files(info["Id"], expected) + if hashes(trial) != original: + raise RuntimeError("Original trial evidence was not restored byte for byte") + (output / "integrity.json").write_text(json.dumps({ + "original_trial_unchanged": True, "frozen_solution_unchanged": True, + "original_tests_unchanged": True, + "finished_at": datetime.now(timezone.utc).isoformat(), + }, indent=2) + "\n") + finally: + try: + for container in reversed(started): + docker("stop", "--time", "10", container) + finally: + (output / "completion.json").write_text(json.dumps({ + "status": "finished", "finished_at": datetime.now(timezone.utc).isoformat(), + }, indent=2) + "\n") + record = json.loads((output / "runner-stdout.txt").read_text()) + result = batch_result(output, record) + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({"output_dir": str(output), **result}), flush=True) + return int(result["status"] != "scored") diff --git a/benchmarks/harbor/tests/test_retained.py b/benchmarks/harbor/tests/test_retained.py new file mode 100644 index 0000000..3b6ebbb --- /dev/null +++ b/benchmarks/harbor/tests/test_retained.py @@ -0,0 +1,139 @@ +"""Retained verifier batches must preserve previously published evidence.""" + +import asyncio +import json +import shlex +import subprocess +import sys +from unittest.mock import Mock + +import pytest + +from harbor_adapter.retained import hashes +from harbor_adapter.verifier import _RUNNER + + +@pytest.mark.parametrize("timeout", [False, True]) +def test_batch_restores_old_attempts_and_keeps_new_logs_separate(tmp_path, timeout): + logs = tmp_path / "logs" + (logs / "attempts/1").mkdir(parents=True) + (logs / "attempts/1/test-stdout.txt").write_text("original attempt\n") + (logs / "retry-summary.json").write_text('{"old": true}\n') + (logs / "reward.txt").write_text("1\n") + before = hashes(logs) + archive = tmp_path / "batch" + command = f"echo 0 > {shlex.quote(str(logs / 'reward.txt'))}" + if timeout: + command += "; sleep 10" + result = subprocess.run([ + sys.executable, "-c", _RUNNER, "batch", str(archive), command, + "0.1", "3", "0", "0", str(logs), str(tmp_path / "control.pid"), + ], capture_output=True, text=True, check=True) + assert hashes(logs) == before + summary = json.loads((archive / "verifier/retry-summary.json").read_text()) + assert len(summary["attempts"]) == (3 if timeout else 1) + assert json.loads(result.stdout)["status"] == ("timeout" if timeout else "completed") + assert (archive / "verifier/reward.txt").exists() is not timeout + assert (archive / "verifier/attempts/1/reward.txt").read_text() == "0\n" + + +def test_batch_restores_original_logs_if_runner_raises(tmp_path): + from harbor_adapter._verifier_runner import run_batch + + logs = tmp_path / "logs" + logs.mkdir() + (logs / "old.txt").write_text("original evidence") + before = hashes(logs) + # An unavailable control directory fails before any verifier process starts. + with pytest.raises(FileNotFoundError): + run_batch("true", 1, 3, logs, tmp_path / "absent/control.pid", tmp_path / "batch") + assert hashes(logs) == before + + +def test_batch_refuses_to_overwrite_an_existing_archive(tmp_path): + from harbor_adapter._verifier_runner import run_batch + + logs = tmp_path / "logs" + logs.mkdir() + (logs / "old.txt").write_text("original evidence") + before = hashes(logs) + archive = tmp_path / "batch" + archive.mkdir() + with pytest.raises(FileExistsError): + run_batch("true", 1, 3, logs, tmp_path / "control.pid", archive) + assert hashes(logs) == before + + +def test_cancelled_batch_restores_original_evidence(tmp_path): + logs = tmp_path / "logs" + logs.mkdir() + (logs / "old.txt").write_text("original evidence") + before = hashes(logs) + archive, control = tmp_path / "batch", tmp_path / "control.pid" + ready = tmp_path / "ready" + command = f"touch {shlex.quote(str(ready))}; sleep 30" + + async def cancel(): + process = await asyncio.create_subprocess_exec( + sys.executable, "-c", _RUNNER, "batch", str(archive), command, + "10", "3", "0", "0", str(logs), str(control), + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + async with asyncio.timeout(5): + while not ready.exists(): + await asyncio.sleep(0.01) + cleanup = await asyncio.create_subprocess_exec( + sys.executable, "-c", _RUNNER, "cancel", str(control), + ) + assert await cleanup.wait() == 0 + await asyncio.wait_for(process.communicate(), 5) + assert process.returncode != 0 + + asyncio.run(cancel()) + assert hashes(logs) == before + summary = json.loads((archive / "verifier/retry-summary.json").read_text()) + assert [a["status"] for a in summary["attempts"]] == ["cancelled"] + + +def test_retained_batch_waits_for_dependency_services(monkeypatch): + from harbor_adapter.retained import wait_for_services + + inspect = Mock(side_effect=[ + json.dumps([{"State": {"Running": True, "Health": {"Status": "starting"}}}]), + json.dumps([{"State": {"Running": True, "Health": {"Status": "healthy"}}}]), + ]) + monkeypatch.setattr("harbor_adapter.retained.docker", inspect) + monkeypatch.setattr("harbor_adapter.retained.time.sleep", lambda _: None) + wait_for_services(["cache-service"]) + assert inspect.call_count == 2 + + +def test_retained_batch_rejects_failed_services(monkeypatch): + from harbor_adapter.retained import wait_for_services + + monkeypatch.setattr("harbor_adapter.retained.docker", lambda *args: json.dumps([ + {"State": {"Running": False}}, + ])) + with pytest.raises(RuntimeError, match="failed to start"): + wait_for_services(["cache-service"]) + + +@pytest.mark.parametrize("reward", ["0", "1", "invalid", "nan", None]) +def test_retained_result_distinguishes_valid_scores_from_verifier_errors(tmp_path, reward): + from harbor_adapter.retained import batch_result + + (tmp_path / "verifier").mkdir() + if reward is not None: + (tmp_path / "verifier/reward.txt").write_text(reward) + result = batch_result(tmp_path, {"status": "completed"}) + valid = reward in {"0", "1"} + assert result["status"] == ("scored" if valid else "verifier_error") + assert result["rewards"] == ({"reward": float(reward)} if valid else None) + + +def test_retained_timeout_does_not_publish_an_archived_reward(tmp_path): + from harbor_adapter.retained import batch_result + + (tmp_path / "verifier").mkdir() + (tmp_path / "verifier/reward.txt").write_text("1") + assert batch_result(tmp_path, {"status": "timeout"})["rewards"] is None From f7383dd69cec928ce61e12c49f3850c2edb34be5 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sat, 24 Oct 2026 06:31:12 -0700 Subject: [PATCH 10/12] feat(benchmarks): prepare retained verifier dependencies separately --- benchmarks/harbor/README.md | 18 +++++ benchmarks/harbor/README.zh-CN.md | 13 ++++ ...orch-model-recovery-prepared-20261024.json | 32 +++++++++ .../harbor/src/harbor_adapter/__main__.py | 1 + .../src/harbor_adapter/_verifier_runner.py | 36 ++++++++-- .../harbor/src/harbor_adapter/bootstrap.py | 16 +++++ .../harbor/src/harbor_adapter/retained.py | 45 +++++++++++-- benchmarks/harbor/tests/test_bootstrap.py | 16 +++++ benchmarks/harbor/tests/test_retained.py | 66 +++++++++++++++++++ 9 files changed, 232 insertions(+), 11 deletions(-) create mode 100644 benchmarks/harbor/configs/pytorch-model-recovery-prepared-20261024.json diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 5cf2dd7..d784f9f 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -82,6 +82,12 @@ assertions or solution code. Unknown QEMU revisions fail setup pending profile review; tasks without a profile explicitly report `not_configured`. Add a profile and regression tests when another task needs verifier preflight. +The reviewed `pytorch-model-recovery` revision also has a dependency-only +profile. It uses the original uv 0.9.5 installer and the verifier's Python 3.13, +pytest 8.4.1, torch 2.7.1, and pytest-json-ctrf 0.3.5 requirements, then runs only +`pytest --version` from `/tmp`. It does not import the submitted model or run +the task's tests. Other revisions require profile review. + Every installation writes `agent/bootstrap.json`, including partial progress on failure. Preflight failure stops the trial before any model call. The official verifier script remains unchanged and may still access the network @@ -334,6 +340,18 @@ hashes, and integrity checks go into the separate output directory. Containers are stopped and retained after the batch. This does not change the original trial result or authorize repeated batches automatically. +To give dependency preparation its own budget on a retained solution, add +`--dependency-timeout-sec 3600`. This requires a reviewed profile for the exact +task revision and checks its registered task-file hashes. Preparation runs once +in the same container before verification; its logs and status are recorded +under `dependency-preparation/`. A preparation timeout or nonzero exit is a +setup failure with no score, and verification does not start. On success, the +unchanged official verifier script runs with the original per-attempt timeout +(900 seconds for `pytorch-model-recovery`). Its own remaining dependency checks +are still included in those 900 seconds. The larger preparation deadline never +extends the verifier deadline. Record this preparation policy separately from +earlier baseline results, even when the solution is identical. + ## Benchmark reports Reports are available in [English](reports/en/) and diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index dd01a5e..7ceeda3 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -66,6 +66,11 @@ uv 0.9.5、Python 3.13、pytest 8.4.1 和 pytest-json-ctrf 0.3.5，不执行题 参考解。未知的 QEMU 版本会在安装阶段报错，要求先检查对应预检配置；没有预检配置 的其他题目明确记录为 `not_configured`。其他题目需要预检时，在这里新增配置及回归测试。 +已检查的 `pytorch-model-recovery` 版本也有仅准备依赖的配置。它使用原始 uv 0.9.5 +安装器和验证器所需的 Python 3.13、pytest 8.4.1、torch 2.7.1、pytest-json-ctrf +0.3.5，随后仅在 `/tmp` 执行 `pytest --version`。它不导入提交的模型，也不运行 +题目测试。其他题目版本需要重新检查对应配置。 + 每次安装都会写入 `agent/bootstrap.json`，失败时也保留已完成的步骤。预检失败会 在模型调用前终止 trial。官方验证脚本保持原样，执行时仍可能访问网络，所以预检 通过不能保证后续没有网络故障。完整标准运行默认启用下述验证超时重试，整题重试 @@ -269,6 +274,14 @@ verifier 参数一样，需要单独授权。任何已完成的评分都会结 哈希和完整性检查写入单独的输出目录。批次结束后，容器会被停止并继续保留。 该命令不会修改原 trial 的结果，也不会自动授权后续重复批次。 +若要为保留解答的依赖准备单独分配时间，可添加 `--dependency-timeout-sec 3600`。 +这要求精确题目版本已有检查过的依赖配置，并核对其中登记的任务文件哈希。依赖准备 +在同一容器中、正式验证前执行一次，日志和状态单独记录到 `dependency-preparation/`。 +准备超时或以非零状态退出时，结果为安装失败、不评分，也不会启动正式验证。准备 +成功后，保持原样的官方验证脚本按原生单次时限执行（`pytorch-model-recovery` 为 +900 秒）。脚本内剩余的依赖检查仍计入这 900 秒。更长的准备时限不会延长正式验证 +时限。即使解答相同，也应将这一准备策略与先前 baseline 分开记录。 + ## Benchmark 报告 报告提供[英文](reports/en/)和[简体中文](reports/zh-CN/)两个版本，文件名一一对应。 diff --git a/benchmarks/harbor/configs/pytorch-model-recovery-prepared-20261024.json b/benchmarks/harbor/configs/pytorch-model-recovery-prepared-20261024.json new file mode 100644 index 0000000..c56a524 --- /dev/null +++ b/benchmarks/harbor/configs/pytorch-model-recovery-prepared-20261024.json @@ -0,0 +1,32 @@ +{ + "kind": "separately authorized dependency preparation followed by native-timeout verification", + "authorization": "User approved giving dependency preparation more time while retaining the original formal verification limit on 2026-10-24.", + "task": "terminal-bench/pytorch-model-recovery", + "task_ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "trial_dir": "jobs/runtime-source/jobs/dev3-1017-pytorch-model-recovery-recovery-20261020/pytorch-model-recovery__nCs7Y4H", + "task_dir": "/home/minix/.cache/harbor/tasks/packages/terminal-bench/pytorch-model-recovery/2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "container": "pytorch-model-recovery__ncs7y4h__env-main-1", + "output_dir": "jobs/pytorch-model-recovery-prepared-20261024", + "dependency_preparation": { + "profile": "pytorch-model-recovery-v1", + "timeout_seconds": 3600, + "max_attempts": 1, + "executes_task_tests": false, + "imports_submitted_model": false + }, + "formal_verification": { + "native_timeout_seconds": 900, + "max_attempts": 1, + "official_script_unchanged": true + }, + "model_calls": 0, + "frozen_model_sha256": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "previous_verifier_attempts_on_this_solution": 6, + "preserve_original_trial_evidence": true, + "keep_containers": true, + "earlier_experiment_sha256": { + "benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json": "37249473185caa2980fd5a322a375ef5da6bde1a7dd43e2964160def218a93b6", + "benchmarks/harbor/results/pytorch-model-recovery-verifier-20261021.json": "30a91d03fc2af479205d33be9b1ad9d340accca50216f49c5a7fd492ba75703f" + }, + "comparison_note": "Additional dependency preparation changes the evaluation setup; report this continuation separately without rewriting earlier baseline or supplement results." +} diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py index e665df5..11eed91 100644 --- a/benchmarks/harbor/src/harbor_adapter/__main__.py +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -176,6 +176,7 @@ def main() -> int: retained.add_argument("--container", required=True) retained.add_argument("--output-dir", type=Path, required=True) retained.add_argument("--max-attempts", type=int, default=3) + retained.add_argument("--dependency-timeout-sec", type=float, help="Prepare a reviewed dependency profile before verification, with a separate deadline") retained.add_argument("--retry-dependency-errors", action="store_true") retained.add_argument("--retry-collection-errors", action="store_true") args = parser.parse_args() diff --git a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py index 3c78b74..a2620b7 100644 --- a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py +++ b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py @@ -9,6 +9,7 @@ from pathlib import Path import re import shutil +import shlex import signal import subprocess import sys @@ -233,14 +234,35 @@ def run_batch(command, timeout_sec, max_attempts, logs, control, archive, shutil.move(str(original / name), logs / name) +def run_prepared_batch(preparation_command, preparation_timeout, command, + timeout_sec, max_attempts, logs, control, archive, + retry_dependency_errors=False, retry_collection_errors=False): + """Prepare dependencies separately, then preserve the native verifier budget.""" + preparation_logs = Path(str(archive) + "-dependencies") + preparation_logs.mkdir(parents=True, exist_ok=False) + preparation = run( + f"({preparation_command}) > {shlex.quote(str(preparation_logs / 'test-stdout.txt'))} 2>&1", + preparation_timeout, 1, preparation_logs, control, + ) + result = {"preparation": preparation, "verification": None} + if preparation["status"] == "completed" and preparation["return_code"] == 0: + result["verification"] = run_batch( + command, timeout_sec, max_attempts, logs, control, archive, + retry_dependency_errors, retry_collection_errors, + ) + return result + + if __name__ == "__main__": if sys.argv[1] == "cancel": cancel(sys.argv[2]) else: arguments = sys.argv[1:] - archive = arguments.pop(1) if arguments[0] == "batch" else None - if archive is not None: - arguments.pop(0) + mode = arguments.pop(0) if arguments[0] in {"batch", "prepared-batch"} else None + archive = arguments.pop(0) if mode else None + if mode == "prepared-batch": + preparation_command, preparation_timeout = arguments[:2] + del arguments[:2] if len(arguments) == 5: command, timeout, count, logs, control = arguments retry_dependencies = False @@ -253,8 +275,12 @@ def run_batch(command, timeout_sec, max_attempts, logs, control, archive, command, timeout, count, retry, collection, logs, control = arguments retry_dependencies = retry == "1" retry_collection = collection == "1" - runner = run if archive is None else lambda *args: run_batch( - *args[:5], archive, *args[5:]) + if mode == "prepared-batch": + runner = lambda *args: run_prepared_batch( + preparation_command, float(preparation_timeout), *args[:5], archive, *args[5:]) + else: + runner = run if archive is None else lambda *args: run_batch( + *args[:5], archive, *args[5:]) print(json.dumps(runner( command, float(timeout), int(count), logs, control, retry_dependencies, retry_collection, diff --git a/benchmarks/harbor/src/harbor_adapter/bootstrap.py b/benchmarks/harbor/src/harbor_adapter/bootstrap.py index 412315c..5a0f550 100644 --- a/benchmarks/harbor/src/harbor_adapter/bootstrap.py +++ b/benchmarks/harbor/src/harbor_adapter/bootstrap.py @@ -145,6 +145,16 @@ async def install( "command -v sshpass; " "uvx -p 3.13 -w pytest==8.4.1 -w pytest-json-ctrf==0.3.5 pytest --version" ) +PYTORCH_RECOVERY_TASK = "terminal-bench/pytorch-model-recovery" +PYTORCH_RECOVERY_REF = "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc" +PYTORCH_RECOVERY_PREFLIGHT = ( + "set -euo pipefail; cd /tmp; command -v curl; " + "curl -LsSf https://astral.sh/uv/0.9.5/install.sh | sh; " + 'export PATH="$HOME/.local/bin:$PATH"; ' + "uv --version | grep -Eq '^uv 0\\.9\\.5( |$)'; " + "uvx -p 3.13 -w pytest==8.4.1 -w torch==2.7.1 " + "-w pytest-json-ctrf==0.3.5 pytest --version" +) @dataclass(frozen=True) @@ -152,12 +162,18 @@ class VerifierProfile: name: str apt_packages: tuple[str, ...] command: str + task_files_sha256: tuple[tuple[str, str], ...] = () VERIFIER_PROFILES = { (QEMU_TASK, QEMU_REF): VerifierProfile( "qemu-alpine-ssh-v1", ("curl", "sshpass"), QEMU_PREFLIGHT, ), + (PYTORCH_RECOVERY_TASK, PYTORCH_RECOVERY_REF): VerifierProfile( + "pytorch-model-recovery-v1", ("curl",), PYTORCH_RECOVERY_PREFLIGHT, + (("task.toml", "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd"), + ("tests/test.sh", "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e")), + ), } diff --git a/benchmarks/harbor/src/harbor_adapter/retained.py b/benchmarks/harbor/src/harbor_adapter/retained.py index 742370b..efd60ec 100644 --- a/benchmarks/harbor/src/harbor_adapter/retained.py +++ b/benchmarks/harbor/src/harbor_adapter/retained.py @@ -14,6 +14,7 @@ from harbor.utils.scripts import build_execution_command from .verifier import _RUNNER +from .bootstrap import verifier_profile def hashes(root): @@ -73,6 +74,24 @@ def batch_result(output, record): return result | {"status": "scored", "rewards": rewards} +def dependency_plan(trial, task_dir, timeout): + if timeout is None: + return None + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("Dependency preparation timeout must be finite and positive") + config = json.loads((trial / "config.json").read_text()) + name = config["task"].get("name", "").split("/")[-1] + profile = verifier_profile(trial / "agent", name) + if profile is None: + raise ValueError("Dependency preparation requires a reviewed task-revision profile") + for filename, digest in profile.task_files_sha256: + if hashlib.sha256((task_dir / filename).read_bytes()).hexdigest() != digest: + raise ValueError(f"Reviewed dependency profile mismatch: {filename}") + return {"profile": profile.name, "timeout_seconds": timeout, + "command": profile.command, "apt_prerequisites": list(profile.apt_packages), + "task_files_sha256": dict(profile.task_files_sha256)} + + def verify_retained(args): """Run no agent, preserve native evidence, and stop on any completed score.""" trial, task_dir, output = (p.resolve() for p in @@ -87,6 +106,7 @@ def verify_retained(args): timeout = task.config.verifier.timeout_sec if not 0 < timeout < float("inf"): raise ValueError("A finite, positive native verifier timeout is required") + preparation = dependency_plan(trial, task_dir, args.dependency_timeout_sec) info, = json.loads(docker("inspect", args.container)) if info["State"]["Running"]: raise ValueError("Stop the retained trial before verifying it") @@ -115,9 +135,10 @@ def verify_retained(args): "max_attempts": args.max_attempts, "retry_dependency_errors": args.retry_dependency_errors, "retry_collection_errors": args.retry_collection_errors, + "dependency_preparation": preparation, "original_trial_sha256": original, "expected_container_sha256": expected, "source_sha256": {name: hashlib.sha256((source / name).read_bytes()).hexdigest() - for name in ("retained.py", "_verifier_runner.py")}, + for name in ("retained.py", "_verifier_runner.py", "bootstrap.py", "__main__.py")}, } archive = f"/tmp/nanopy-retained-{uuid.uuid4().hex}" control = archive + ".pid" @@ -140,22 +161,32 @@ def verify_retained(args): started.append(container) wait_for_services(services) check_container_files(info["Id"], expected) + mode = (["prepared-batch", archive, preparation["command"], + str(preparation["timeout_seconds"])] if preparation else ["batch", archive]) with (output / "runner-stdout.txt").open("w") as stdout, (output / "runner-stderr.txt").open("w") as stderr: process = subprocess.Popen([ "docker", "exec", "--user", "root", info["Id"], "python3", "-c", _RUNNER, - "batch", archive, command, str(timeout), str(args.max_attempts), + *mode, command, str(timeout), str(args.max_attempts), str(int(args.retry_dependency_errors)), str(int(args.retry_collection_errors)), "/logs/verifier", control, ], stdout=stdout, stderr=stderr) try: - code = process.wait(timeout=timeout * args.max_attempts + 120) + code = process.wait(timeout=timeout * args.max_attempts + 120 + + (preparation["timeout_seconds"] if preparation else 0)) except BaseException: docker("exec", "--user", "root", info["Id"], "python3", "-c", _RUNNER, "cancel", control) process.wait(timeout=30) raise - docker("cp", info["Id"] + ":" + archive + "/verifier", output / "verifier") if code: raise RuntimeError(f"Verifier runner exited {code}; see {output}") + record = json.loads((output / "runner-stdout.txt").read_text()) + if preparation: + docker("cp", info["Id"] + ":" + archive + "-dependencies", output / "dependency-preparation") + verification = record["verification"] + else: + verification = record + if verification is not None: + docker("cp", info["Id"] + ":" + archive + "/verifier", output / "verifier") check_container_files(info["Id"], expected) if hashes(trial) != original: raise RuntimeError("Original trial evidence was not restored byte for byte") @@ -172,8 +203,10 @@ def verify_retained(args): (output / "completion.json").write_text(json.dumps({ "status": "finished", "finished_at": datetime.now(timezone.utc).isoformat(), }, indent=2) + "\n") - record = json.loads((output / "runner-stdout.txt").read_text()) - result = batch_result(output, record) + result = (batch_result(output, verification) if verification is not None else + {"status": "setup_failed", "rewards": None, "last_attempt": None}) + if preparation: + result["dependency_preparation"] = record["preparation"] (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps({"output_dir": str(output), **result}), flush=True) return int(result["status"] != "scored") diff --git a/benchmarks/harbor/tests/test_bootstrap.py b/benchmarks/harbor/tests/test_bootstrap.py index 7c5ff81..882ead5 100644 --- a/benchmarks/harbor/tests/test_bootstrap.py +++ b/benchmarks/harbor/tests/test_bootstrap.py @@ -135,6 +135,22 @@ def test_profile_is_scoped_to_reviewed_task_revision(tmp_path): verifier_profile(logs_dir, "qemu-alpine-ssh") +def test_pytorch_recovery_profile_pins_the_original_verifier_dependencies(tmp_path): + from harbor_adapter.bootstrap import PYTORCH_RECOVERY_REF, PYTORCH_RECOVERY_TASK + + (tmp_path / "config.json").write_text(json.dumps({"task": { + "name": PYTORCH_RECOVERY_TASK, "ref": PYTORCH_RECOVERY_REF, + }})) + profile = verifier_profile(tmp_path / "agent", "pytorch-model-recovery") + assert profile.name == "pytorch-model-recovery-v1" + for value in ["uv/0.9.5/install.sh", "-p 3.13", "pytest==8.4.1", "torch==2.7.1", + "pytest-json-ctrf==0.3.5", "pytest --version", "cd /tmp"]: + assert value in profile.command + assert "/tests" not in profile.command and "/app" not in profile.command + assert "--index" not in profile.command and "--extra-index" not in profile.command + assert set(dict(profile.task_files_sha256)) == {"task.toml", "tests/test.sh"} + + @pytest.mark.parametrize("task", [ {"path": "/tasks/qemu-alpine-ssh"}, {"path": "/dataset/qemu-alpine-ssh", "source": "local-dataset"}, diff --git a/benchmarks/harbor/tests/test_retained.py b/benchmarks/harbor/tests/test_retained.py index 3b6ebbb..faf2e86 100644 --- a/benchmarks/harbor/tests/test_retained.py +++ b/benchmarks/harbor/tests/test_retained.py @@ -2,6 +2,7 @@ import asyncio import json +import hashlib import shlex import subprocess import sys @@ -137,3 +138,68 @@ def test_retained_timeout_does_not_publish_an_archived_reward(tmp_path): (tmp_path / "verifier").mkdir() (tmp_path / "verifier/reward.txt").write_text("1") assert batch_result(tmp_path, {"status": "timeout"})["rewards"] is None + + +@pytest.mark.parametrize("preparation", ["sleep 0.2", "exit 2", "sleep 10"]) +def test_dependency_preparation_has_its_own_budget_and_gates_verification(tmp_path, preparation): + logs = tmp_path / "logs" + logs.mkdir() + (logs / "old.txt").write_text("original evidence") + before = hashes(logs) + archive = tmp_path / "batch" + marker = tmp_path / "verification-started" + command = f"touch {shlex.quote(str(marker))}; echo 0 > {shlex.quote(str(logs / 'reward.txt'))}" + result = subprocess.run([ + sys.executable, "-c", _RUNNER, "prepared-batch", str(archive), preparation, + "0.5", command, "0.1", "1", "0", "0", str(logs), str(tmp_path / "control.pid"), + ], capture_output=True, text=True, check=True) + record = json.loads(result.stdout) + success = preparation == "sleep 0.2" + assert marker.exists() is success + assert hashes(logs) == before + if success: + assert record["preparation"]["duration_seconds"] > 0.1 + assert record["verification"]["status"] == "completed" + assert (archive / "verifier/reward.txt").read_text() == "0\n" + else: + assert record["verification"] is None + assert not archive.exists() + summary = json.loads((tmp_path / "batch-dependencies/retry-summary.json").read_text()) + assert summary["timeout_sec"] == 0.5 + assert len(summary["attempts"]) == 1 + + +def test_long_preparation_budget_does_not_extend_the_formal_verifier(tmp_path): + logs = tmp_path / "logs" + logs.mkdir() + (logs / "old.txt").write_text("original evidence") + before = hashes(logs) + result = subprocess.run([ + sys.executable, "-c", _RUNNER, "prepared-batch", str(tmp_path / "batch"), + "true", "10", "sleep 5", "0.1", "1", "0", "0", str(logs), str(tmp_path / "control.pid"), + ], capture_output=True, text=True, check=True) + record = json.loads(result.stdout) + assert record["verification"]["status"] == "timeout" + assert record["verification"]["duration_seconds"] < 2 + assert hashes(logs) == before + + +def test_dependency_preparation_requires_a_reviewed_revision_and_matching_files(tmp_path, monkeypatch): + from harbor_adapter.bootstrap import VERIFIER_PROFILES, VerifierProfile + from harbor_adapter.retained import dependency_plan + + trial, task = tmp_path / "trial", tmp_path / "task" + trial.mkdir() + task.mkdir() + (trial / "config.json").write_text(json.dumps({"task": {"name": "terminal-bench/reviewed", "ref": "sha256:abc"}})) + (task / "task.toml").write_text("reviewed task") + digest = hashlib.sha256((task / "task.toml").read_bytes()).hexdigest() + monkeypatch.setitem(VERIFIER_PROFILES, ("terminal-bench/reviewed", "sha256:abc"), + VerifierProfile("reviewed-v1", (), "true", (("task.toml", digest),))) + assert dependency_plan(trial, task, 3600)["timeout_seconds"] == 3600 + (task / "task.toml").write_text("modified task") + with pytest.raises(ValueError, match="profile mismatch"): + dependency_plan(trial, task, 3600) + (trial / "config.json").write_text(json.dumps({"task": {"name": "terminal-bench/reviewed", "ref": "latest"}})) + with pytest.raises(ValueError, match="Unknown"): + dependency_plan(trial, task, 3600) From 2215db6ab3248f6b6b6ce98cf3f070cd07b4a840 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sat, 24 Oct 2026 06:56:46 -0700 Subject: [PATCH 11/12] chore(benchmarks): record dependency-prepared recovery Record the retained solution passing five tests with independent preparation and native verifier deadlines. Give the timing regression fixture more scheduling tolerance without changing benchmark time limits. --- ...ytorch-model-recovery-prepared-20261024.md | 86 +++++ ...ytorch-model-recovery-prepared-20261024.md | 74 ++++ ...orch-model-recovery-prepared-20261024.json | 355 ++++++++++++++++++ benchmarks/harbor/tests/test_retained.py | 10 +- 4 files changed, 520 insertions(+), 5 deletions(-) create mode 100644 benchmarks/harbor/reports/en/pytorch-model-recovery-prepared-20261024.md create mode 100644 benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-prepared-20261024.md create mode 100644 benchmarks/harbor/results/pytorch-model-recovery-prepared-20261024.json diff --git a/benchmarks/harbor/reports/en/pytorch-model-recovery-prepared-20261024.md b/benchmarks/harbor/reports/en/pytorch-model-recovery-prepared-20261024.md new file mode 100644 index 0000000..951f3a2 --- /dev/null +++ b/benchmarks/harbor/reports/en/pytorch-model-recovery-prepared-20261024.md @@ -0,0 +1,86 @@ +# PyTorch model recovery with separate dependency preparation + +Date: 2026-10-24. This authorized continuation reuses the frozen +`pytorch-model-recovery__nCs7Y4H` solution from the +[October 20 supplement](dev34-flash1017-recovery-20261020.md), following the +[October 21 verifier continuation](pytorch-model-recovery-verifier-20261021.md). + +**The retained solution passed all five official tests and received reward 1.0.** +Dependency preparation took 1,127.589 seconds, exceeding the original 900-second +limit. The unchanged official verifier then finished in 49.934 seconds within +its original 900-second limit. No model was called or solution regenerated. + +| Phase | Independent limit | Actual duration | Outcome | +| --- | --- | --- | --- | +| Dependency preparation | 3,600 s | 1,127.589 s | Passed; dependencies installed, `pytest --version` succeeded | +| Official verification | 900 s | 49.934 s | Passed; reward 1.0; five tests passed | + +The pytest session itself reported 5.00 seconds. The 49.934-second verifier +duration also includes the original script's APT refresh, uv installer, and +remaining dependency checks. Those steps were not removed or given extra time. +Preparation and formal verification each ran once. This was the seventh +verifier attempt on this solution; the previous six never reached pytest. + +## Official test results + +| Test | Result | +| --- | --- | +| `test_weights_file_unchanged` | Passed | +| `test_model_file_exists` | Passed | +| `test_model_loads_weights` | Passed | +| `test_state_dicts_match` | Passed | +| `test_model_loss` | Passed | + +There were zero failed or skipped tests. Pytest emitted two non-failing warnings: +NumPy was unavailable, and the Transformer's layout did not enable nested-tensor +optimization. The dependency requirements were left unchanged. + +## Preparation policy and implementation + +The new `--dependency-timeout-sec 3600` option uses the reviewed +`pytorch-model-recovery-v1` profile for task revision +`sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc`. +It checks the registered task configuration and test-script hashes, uses the +original uv 0.9.5 installer, Python 3.13, pytest 8.4.1, torch 2.7.1, and +pytest-json-ctrf 0.3.5 requirements, and runs only `pytest --version` from `/tmp`. +It does not import the submitted model or execute task assertions. + +A failed preparation would have been recorded as a setup failure with no score +and no official verification. Successful preparation starts a separate native +verifier deadline; the 3,600-second budget cannot extend that deadline. The +original task image, resources, package sources, and official scripts were +preserved. The native test log reports Python 3.13.12, pytest 8.4.1, pluggy 1.7.0, +and pytest-json-ctrf 0.3.5. + +The executed implementation was commit +`f7383dd69cec928ce61e12c49f3850c2edb34be5`. Original model generation used core +agent `ce28b1e826a292d796ba64f13e79199b66f65758` and runtime +`40fdd041c370a96dc06f86b436eacc01ce1566d7`. The frozen `/app/model.pt` SHA256 is +still `098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e`. + +## Integrity and comparison limits + +The same retained container and solution were used. Frozen solution and test +hashes passed checks before and after execution, and original trial files were +restored byte for byte. All three project containers are stopped and retained, +with unchanged identities and no container OOM flag. All 50 temporarily relocated +duplicate image-cache files were restored and SHA256-verified. The completed +dependency environment is retained. The local Harbor suite passed **209 tests**, +including independent deadlines, failed-preparation gating, and evidence +restoration. + +This preparation policy changes the evaluation setup. Live dependency resolution +and cache state also limit comparisons. The October 19 baseline remains +**26 passes, five scored failures, three unscored verifier errors**. The October +20 and October 21 records remain unchanged. This passed continuation is recorded +separately; it does not replace earlier results or create a composite dev34 score. +New model calls and model cost were both zero. + +- [Machine-readable result](../../results/pytorch-model-recovery-prepared-20261024.json) +- [Registered preparation and verification plan](../../configs/pytorch-model-recovery-prepared-20261024.json) +- Local evidence: `jobs/pytorch-model-recovery-prepared-20261024/`, including + separate preparation/verifier logs, CTRF test results, executed sources, + source and artifact hashes, integrity checks, and container retention. +- Live canonical verifier logs can refer to the original trial before and after + the new batch; the separately archived batch logs determine this result. +- [Complete Simplified Chinese version](../zh-CN/pytorch-model-recovery-prepared-20261024.md) diff --git a/benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-prepared-20261024.md b/benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-prepared-20261024.md new file mode 100644 index 0000000..8e4ae0c --- /dev/null +++ b/benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-prepared-20261024.md @@ -0,0 +1,74 @@ +# PyTorch 模型恢复：单独准备依赖后的验证 + +日期：2026-10-24。本次经过授权的追加实验复用 +[10 月 20 日补充实验](dev34-flash1017-recovery-20261020.md)中冻结的 +`pytorch-model-recovery__nCs7Y4H` 解答，接续 +[10 月 21 日 verifier 追加验证](pytorch-model-recovery-verifier-20261021.md)。 + +**保留解答通过了全部 5 项官方测试，得分为 1.0。** 依赖准备用时 1,127.589 秒， +超过了原来的 900 秒限制。随后，保持原样的官方验证器在原有 900 秒时限内，用 +49.934 秒完成验证。本次没有调用模型，也没有重新生成解答。 + +| 阶段 | 独立时限 | 实际耗时 | 结果 | +| --- | --- | --- | --- | +| 依赖准备 | 3,600 秒 | 1,127.589 秒 | 成功；依赖已安装，`pytest --version` 成功 | +| 官方验证 | 900 秒 | 49.934 秒 | 成功；得分 1.0；5 项测试通过 | + +pytest 会话本身报告耗时 5.00 秒。验证器的 49.934 秒还包括原脚本中的 APT 索引 +更新、uv 安装器和剩余依赖检查。这些步骤均未被删除，也没有获得额外时间。 +依赖准备和正式验证各执行一次。这是该解答的第 7 次 verifier 尝试，之前 6 次均 +未进入 pytest。 + +## 官方测试结果 + +| 测试 | 结果 | +| --- | --- | +| `test_weights_file_unchanged` | 通过 | +| `test_model_file_exists` | 通过 | +| `test_model_loads_weights` | 通过 | +| `test_state_dicts_match` | 通过 | +| `test_model_loss` | 通过 | + +没有失败或跳过的测试。pytest 输出了两项不影响通过的警告：NumPy 不可用，以及 +Transformer 的布局未启用嵌套张量优化。依赖要求保持不变。 + +## 准备策略与实现 + +新增的 `--dependency-timeout-sec 3600` 选项使用已检查的 +`pytorch-model-recovery-v1` 配置，绑定题目版本 +`sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc`。 +它核对已登记的题目配置及测试脚本哈希，使用原始 uv 0.9.5 安装器，以及 Python +3.13、pytest 8.4.1、torch 2.7.1 和 pytest-json-ctrf 0.3.5 的原有依赖要求，仅在 +`/tmp` 执行 `pytest --version`。它不导入提交的模型，也不执行题目断言。 + +若准备失败，会记为安装失败，不评分，也不进入官方验证。准备成功后，才启动独立 +的原生 verifier 时限；3,600 秒的准备预算不会延长该时限。原始任务镜像、资源、 +包来源和官方脚本均被保留。原生测试日志记录的版本为 Python 3.13.12、pytest +8.4.1、pluggy 1.7.0 和 pytest-json-ctrf 0.3.5。 + +实际执行的实现提交为 `f7383dd69cec928ce61e12c49f3850c2edb34be5`。原解答生成时 +使用的核心 agent 提交为 `ce28b1e826a292d796ba64f13e79199b66f65758`，运行时提交为 +`40fdd041c370a96dc06f86b436eacc01ce1566d7`。冻结的 `/app/model.pt` SHA256 仍为 +`098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e`。 + +## 完整性与比较限制 + +本次使用同一保留容器和同一份解答。冻结解答和测试的哈希在执行前后均通过核对， +原 trial 文件已逐字节恢复。项目的 3 个容器均已停止并继续保留，身份不变，也没有 +容器 OOM 标记。临时迁移的 50 个重复镜像缓存文件已全部恢复，并通过 SHA256 校验。 +已准备完成的依赖环境继续保留。本地 Harbor 测试套件 **209 项通过**，覆盖独立 +时限、准备失败时阻止验证，以及证据恢复。 + +这一准备策略改变了评估环境；实时依赖解析及缓存状态也限制了可比性。10 月 19 日 +baseline 仍为 **26 题成功、5 题有效评分失败、3 题 verifier 错误未评分**。 +10 月 20 日和 10 月 21 日记录均保持不变。本次通过结果单独记录，不替换旧结果， +也不生成混合后的 dev34 分数。新增模型调用次数和模型费用均为零。 + +- [机器可读结果](../../results/pytorch-model-recovery-prepared-20261024.json) +- [已登记的依赖准备与验证计划](../../configs/pytorch-model-recovery-prepared-20261024.json) +- 本地证据：`jobs/pytorch-model-recovery-prepared-20261024/`，包括分开的依赖准备与 + verifier 日志、CTRF 测试结果、实际执行的源码、源码和产物哈希、完整性检查及 + 容器保留记录。 +- 在新批次前后，实时的当前 verifier 日志可能属于原 trial；本结果以单独归档的 + 新批次日志为准。 +- [完整英文版本](../en/pytorch-model-recovery-prepared-20261024.md) diff --git a/benchmarks/harbor/results/pytorch-model-recovery-prepared-20261024.json b/benchmarks/harbor/results/pytorch-model-recovery-prepared-20261024.json new file mode 100644 index 0000000..e1b2330 --- /dev/null +++ b/benchmarks/harbor/results/pytorch-model-recovery-prepared-20261024.json @@ -0,0 +1,355 @@ +{ + "schema_version": 1, + "experiment": "pytorch-model-recovery-prepared-20261024", + "collected_at": "2026-10-24T13:54:30.566833+00:00", + "kind": "Separately authorized dependency preparation followed by native-timeout verification of the retained solution.", + "task": "terminal-bench/pytorch-model-recovery", + "task_ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "trial": "pytorch-model-recovery__nCs7Y4H", + "source_ref": "f7383dd69cec928ce61e12c49f3850c2edb34be5", + "original_core_agent_ref": "ce28b1e826a292d796ba64f13e79199b66f65758", + "original_runtime_source_ref": "40fdd041c370a96dc06f86b436eacc01ce1566d7", + "new_model_calls": 0, + "new_model_cost_usd": 0, + "earlier_experiment_sha256": { + "benchmarks/harbor/results/dev34-flash1017-recovery-20261020.json": "37249473185caa2980fd5a322a375ef5da6bde1a7dd43e2964160def218a93b6", + "benchmarks/harbor/results/pytorch-model-recovery-verifier-20261021.json": "30a91d03fc2af479205d33be9b1ad9d340accca50216f49c5a7fd492ba75703f" + }, + "plan": { + "path": "benchmarks/harbor/configs/pytorch-model-recovery-prepared-20261024.json", + "sha256": "d306487694979a679bab9d4df3a5944b2f16c2af266868ae2b639d38ce5040e4" + }, + "dependency_preparation": { + "profile": "pytorch-model-recovery-v1", + "timeout_seconds": 3600.0, + "command": "set -euo pipefail; cd /tmp; command -v curl; curl -LsSf https://astral.sh/uv/0.9.5/install.sh | sh; export PATH=\"$HOME/.local/bin:$PATH\"; uv --version | grep -Eq '^uv 0\\.9\\.5( |$)'; uvx -p 3.13 -w pytest==8.4.1 -w torch==2.7.1 -w pytest-json-ctrf==0.3.5 pytest --version", + "apt_prerequisites": [ + "curl" + ], + "task_files_sha256": { + "task.toml": "851a51204f53482a31f47cdf8833128cf421113b0331011286940fc857c37dfd", + "tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e" + }, + "result": { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 1127.5886361699959 + }, + "status": "passed", + "task_tests_executed": false, + "submitted_model_imported": false + }, + "formal_verifier_policy": { + "native_timeout_seconds": 900, + "max_attempts": 1, + "official_script_unchanged": true, + "remaining_script_bootstrap_included_in_native_budget": true + }, + "verifier_result": { + "status": "scored", + "rewards": { + "reward": 1.0 + }, + "last_attempt": { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 49.933707160002086 + }, + "dependency_preparation": { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 1127.5886361699959 + } + }, + "formal_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 49.933707160002086, + "overall_attempt_on_this_solution": 7, + "pytest_session_observed": true, + "raw_reward": "1", + "ctrf_summary": { + "tests": 5, + "passed": 5, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1792849927.1654904, + "stop": 1792849932.1679828 + }, + "ctrf_tests": [ + { + "name": "test_outputs.py::test_weights_file_unchanged", + "status": "passed", + "duration": 0.039344387012533844, + "start": 1792849929.2241943, + "stop": 1792849929.2638535, + "retries": 0, + "file_path": "test_outputs.py" + }, + { + "name": "test_outputs.py::test_model_file_exists", + "status": "passed", + "duration": 0.0003131080011371523, + "start": 1792849929.2640839, + "stop": 1792849929.2645166, + "retries": 0, + "file_path": "test_outputs.py" + }, + { + "name": "test_outputs.py::test_model_loads_weights", + "status": "passed", + "duration": 0.3143234669987578, + "start": 1792849929.264651, + "stop": 1792849929.5791852, + "retries": 0, + "file_path": "test_outputs.py" + }, + { + "name": "test_outputs.py::test_state_dicts_match", + "status": "passed", + "duration": 0.2818363909900654, + "start": 1792849929.5794127, + "stop": 1792849929.8615518, + "retries": 0, + "file_path": "test_outputs.py" + }, + { + "name": "test_outputs.py::test_model_loss", + "status": "passed", + "duration": 2.3055918629979715, + "start": 1792849929.8618186, + "stop": 1792849932.1676106, + "retries": 0, + "file_path": "test_outputs.py" + } + ], + "log_sha256": { + "ctrf.json": "de772fb927affbed04f1d82bfa64cc89d4dda5b1901e6bdfafdfbe3b617a5e21", + "reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "test-stdout.txt": "144b38a364b60e2a201c580ee50469cd510de46e58c541aa5ec4f5d3d12a67e2" + } + } + ], + "overall_verifier_attempts_on_this_solution": 7, + "frozen_solution_sha256": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "frozen_solution_and_test_sha256": { + "/app/__pycache__/recovered_model.cpython-313.pyc": "c492a45a453bba7c9fc0db2bce81b3bac895ca649841b91e2919e13f4ac59071", + "/app/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "/app/model.pt": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "/app/recovered_model.py": "602526444f3517d52c011a3291c3b492d6a86b4bd5200011b4027bc2ee86f3f7", + "/app/tune_model.py": "e2e9e6bbdb6e13f9e7e57d28f8c4e66932bfeff8dc468df2ae2034b8b9e92e71", + "/app/verify.py": "ed8ec4d7d52bbd849c15f8acc066825af6354fb729f265e9d223307835381c9e", + "/app/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "/tests/test.sh": "6005f3a75ccfe00d7bed1005264b4a84a1f8e30cd780f8a97b3a804be7cc7a8e", + "/tests/test_outputs.py": "5d4d57c43e6913bf3d0081da764ad36d1b7275e3bb114a05abda529afe563905", + "/tests/weights_gtruth.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080" + }, + "original_trial_sha256": { + "agent/bootstrap.json": "2acec152ff538811be8b32bbec76c38b2d5e9b3b285e0d600c4c49af6970a9b6", + "agent/nanopycodeagent.txt": "036c564ae66e993149b0a73d6de0e24292f3b06331b035c929d109e019d5025a", + "agent/trajectory.json": "73b4f374dbfb89dd3d3b94d572136c2f613bc1666aad87b8a6454c3316bb3f3d", + "artifacts/app/__pycache__/recovered_model.cpython-313.pyc": "c492a45a453bba7c9fc0db2bce81b3bac895ca649841b91e2919e13f4ac59071", + "artifacts/app/dataset.pt": "6db65e4ec1c0760209c2dd997ef582e4a1519ab38e0312339999c460a696960d", + "artifacts/app/model.pt": "098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e", + "artifacts/app/recovered_model.py": "602526444f3517d52c011a3291c3b492d6a86b4bd5200011b4027bc2ee86f3f7", + "artifacts/app/tune_model.py": "e2e9e6bbdb6e13f9e7e57d28f8c4e66932bfeff8dc468df2ae2034b8b9e92e71", + "artifacts/app/verify.py": "ed8ec4d7d52bbd849c15f8acc066825af6354fb729f265e9d223307835381c9e", + "artifacts/app/weights.pt": "71471e71c5a311a897362dd5b77413e7ca2ca59abd00a110bdc2cd3f8584d080", + "artifacts/manifest.json": "502e6df5db116b0d94d1f1c7cd9a32405cab81979a51f3dbecb38c424c665aa6", + "config.json": "125c1fdecd4add7009fa787adb79226e020e0f8598b12413864ac3c8f3502c83", + "exception.txt": "56305954e60464b84bd02c84ce5d5373a0bab576da8c5d926ed395a9978e7f95", + "lock.json": "5dc26291064142102bbfc4f3a90176a232d1824518915e9ad6da782c755635f5", + "result.json": "2a61cc964ad2a684e2b4adb3e5d333f75093db51320f7e448a2cb83446998e34", + "trial.log": "6504d89623d40eeab818172245fa3795d77f6349871df0f88d363e6674cde42e", + "verifier/attempts/1/test-stdout.txt": "4d00e96829b6291622f1a138c491042c44d5d2790880e9253ac72d363d906864", + "verifier/attempts/2/test-stdout.txt": "e951235c90184be58c1908b676fd33ddbf8e9199c8fd61bf416f0d1e80934690", + "verifier/attempts/3/test-stdout.txt": "034e509e6d5e1e587085482f987d108a25588d854b1404f6f0a3b424ba2c29d8", + "verifier/retry-summary.json": "8609c874195c9b06450a4a0667270aac906444ee1adfa675f486aa19450b55ea", + "verifier/test-stdout.txt": "034e509e6d5e1e587085482f987d108a25588d854b1404f6f0a3b424ba2c29d8" + }, + "integrity": { + "original_trial_unchanged": true, + "frozen_solution_unchanged": true, + "original_tests_unchanged": true, + "finished_at": "2026-10-24T13:52:13.051848+00:00" + }, + "retained_containers": [ + { + "id": "85ece21e3a35b6db0e4452943ead6ad0581681a74051ad71d1e8579784189a50", + "name": "pytorch-model-recovery__ncs7y4h__env-apt-index-cache-1", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-24T13:32:33.039408218Z", + "FinishedAt": "2026-10-24T13:52:33.237887449Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-24T06:52:27.821406816-07:00", + "End": "2026-10-24T06:52:27.918941886-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:28.91935326-07:00", + "End": "2026-10-24T06:52:29.019166792-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:30.020248952-07:00", + "End": "2026-10-24T06:52:30.117255175-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:31.118601761-07:00", + "End": "2026-10-24T06:52:31.218089856-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:32.219416666-07:00", + "End": "2026-10-24T06:52:32.321832994-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "id": "c0f5a78624e44c79f078c5dd77fc6a925c0a435611941374206d77283c0d7976", + "name": "pytorch-model-recovery__ncs7y4h__env-uv-release-cache-1", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-24T13:32:33.132520459Z", + "FinishedAt": "2026-10-24T13:52:23.116543156Z", + "Health": { + "Status": "unhealthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-24T06:52:17.976422201-07:00", + "End": "2026-10-24T06:52:18.083696921-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:19.085021225-07:00", + "End": "2026-10-24T06:52:19.175401171-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:20.176313232-07:00", + "End": "2026-10-24T06:52:20.281013918-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:21.282156914-07:00", + "End": "2026-10-24T06:52:21.373813343-07:00", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-24T06:52:22.374700737-07:00", + "End": "2026-10-24T06:52:22.47747711-07:00", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + { + "id": "e8f4a6bb804bc8ef8f5e0acde059fd6205efc9cd4966b0029149873af31619fd", + "name": "pytorch-model-recovery__ncs7y4h__env-main-1", + "image_id": "sha256:7c2bb52851cef25bdc3939e473496d25fbf11fd0cb7ab58bcf3cefcb751cdc0e", + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-24T13:32:32.926729841Z", + "FinishedAt": "2026-10-24T13:52:43.353900192Z" + } + } + ], + "executed_implementation_sha256": { + "retained.py": "2f1bc92128ade805e9fa6cbf02bb8aae3da60486423785f422152f837b57e36d", + "_verifier_runner.py": "1733a2681bd9d13a30fcc1ab5317df1025d28d2d082b8cc9de85c0d9d95e723c", + "bootstrap.py": "7e79712812e3554a4f3912448c095f5966f59b35f4e5dca285dc186487956a13", + "__main__.py": "c5a12867226b94c679c162bfac2a098de1af4255c2964b4f9be9bc65355260ee" + }, + "storage_maintenance": { + "duplicate_image_cache_bytes_temporarily_relocated": 1576934879, + "restored_files": 50, + "restoration_hashes_verified": true, + "pending_cleanup": false + }, + "local_harbor_tests_passed": 209, + "documentation_sync": { + "english": "benchmarks/harbor/reports/en/pytorch-model-recovery-prepared-20261024.md", + "chinese": "benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-prepared-20261024.md", + "status": "complete report pair" + }, + "evidence_sha256": { + "jobs/pytorch-model-recovery-prepared-20261024/completion.json": "fee46deaa58dd0250ec093e249bbfe4bb3ecc0b8befeb42644a017fd26e4ac9a", + "jobs/pytorch-model-recovery-prepared-20261024/container-retention.json": "aa56ac9c810460e14d42d57fc26812c821b278c40c3a39aa4586a59d067dd2d1", + "jobs/pytorch-model-recovery-prepared-20261024/dependency-preparation/attempts/1/test-stdout.txt": "87e806a4ad5c7e8fae0b0363a7a6fc242f90bfad72dacdcced8a920706172bbf", + "jobs/pytorch-model-recovery-prepared-20261024/dependency-preparation/retry-summary.json": "52bc20894babd6711a3cd82205b5098198d708e2c1c21a49911923b5781a7c67", + "jobs/pytorch-model-recovery-prepared-20261024/dependency-preparation/test-stdout.txt": "87e806a4ad5c7e8fae0b0363a7a6fc242f90bfad72dacdcced8a920706172bbf", + "jobs/pytorch-model-recovery-prepared-20261024/integrity.json": "f3632d7654af3effb1fd30ff08d2afebdb92c5b351b22d1b57746c0ca507dbd7", + "jobs/pytorch-model-recovery-prepared-20261024/registration.json": "154eb469b5d7b2177d082a08cddaafb729ea5a1eb00c315a72c4bab555cae894", + "jobs/pytorch-model-recovery-prepared-20261024/result.json": "e185489f5e38bd0fb99468d67075171c8bd8184913b64da525eb7a12562ac4fe", + "jobs/pytorch-model-recovery-prepared-20261024/runner-stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/pytorch-model-recovery-prepared-20261024/runner-stdout.txt": "4b618522ae681d593336b65b3473f45176b93a3681bb96603f426840dc65c6de", + "jobs/pytorch-model-recovery-prepared-20261024/source/__main__.py": "c5a12867226b94c679c162bfac2a098de1af4255c2964b4f9be9bc65355260ee", + "jobs/pytorch-model-recovery-prepared-20261024/source/_verifier_runner.py": "1733a2681bd9d13a30fcc1ab5317df1025d28d2d082b8cc9de85c0d9d95e723c", + "jobs/pytorch-model-recovery-prepared-20261024/source/bootstrap.py": "7e79712812e3554a4f3912448c095f5966f59b35f4e5dca285dc186487956a13", + "jobs/pytorch-model-recovery-prepared-20261024/source/retained.py": "2f1bc92128ade805e9fa6cbf02bb8aae3da60486423785f422152f837b57e36d", + "jobs/pytorch-model-recovery-prepared-20261024/storage-and-phase-history.jsonl": "5ab0a3b9bff6264d42801e2390a47b12d991ac2476a7658bec79ccd7e853b361", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/attempts/1/ctrf.json": "de772fb927affbed04f1d82bfa64cc89d4dda5b1901e6bdfafdfbe3b617a5e21", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/attempts/1/test-stdout.txt": "144b38a364b60e2a201c580ee50469cd510de46e58c541aa5ec4f5d3d12a67e2", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/ctrf.json": "de772fb927affbed04f1d82bfa64cc89d4dda5b1901e6bdfafdfbe3b617a5e21", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/retry-summary.json": "7d378761ee2484b9bece7e2931dff1f8f1853467024ccbb2ded6a9baefba9289", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/pytorch-model-recovery-prepared-20261024/verifier/test-stdout.txt": "144b38a364b60e2a201c580ee50469cd510de46e58c541aa5ec4f5d3d12a67e2" + }, + "scope_note": "Preparation is a changed evaluation setup. Both earlier experiments remain unchanged; do not substitute this continuation for the original baseline or report a composite score.", + "monitor_note": "The live /logs/verifier directory contains the original trial evidence until preparation succeeds and again after batch restoration. Archived per-batch logs are authoritative for new verification." +} diff --git a/benchmarks/harbor/tests/test_retained.py b/benchmarks/harbor/tests/test_retained.py index faf2e86..fab6691 100644 --- a/benchmarks/harbor/tests/test_retained.py +++ b/benchmarks/harbor/tests/test_retained.py @@ -140,7 +140,7 @@ def test_retained_timeout_does_not_publish_an_archived_reward(tmp_path): assert batch_result(tmp_path, {"status": "timeout"})["rewards"] is None -@pytest.mark.parametrize("preparation", ["sleep 0.2", "exit 2", "sleep 10"]) +@pytest.mark.parametrize("preparation", ["sleep 1", "exit 2", "sleep 10"]) def test_dependency_preparation_has_its_own_budget_and_gates_verification(tmp_path, preparation): logs = tmp_path / "logs" logs.mkdir() @@ -151,21 +151,21 @@ def test_dependency_preparation_has_its_own_budget_and_gates_verification(tmp_pa command = f"touch {shlex.quote(str(marker))}; echo 0 > {shlex.quote(str(logs / 'reward.txt'))}" result = subprocess.run([ sys.executable, "-c", _RUNNER, "prepared-batch", str(archive), preparation, - "0.5", command, "0.1", "1", "0", "0", str(logs), str(tmp_path / "control.pid"), + "2", command, "0.5", "1", "0", "0", str(logs), str(tmp_path / "control.pid"), ], capture_output=True, text=True, check=True) record = json.loads(result.stdout) - success = preparation == "sleep 0.2" + success = preparation == "sleep 1" assert marker.exists() is success assert hashes(logs) == before if success: - assert record["preparation"]["duration_seconds"] > 0.1 + assert record["preparation"]["duration_seconds"] > 0.5 assert record["verification"]["status"] == "completed" assert (archive / "verifier/reward.txt").read_text() == "0\n" else: assert record["verification"] is None assert not archive.exists() summary = json.loads((tmp_path / "batch-dependencies/retry-summary.json").read_text()) - assert summary["timeout_sec"] == 0.5 + assert summary["timeout_sec"] == 2 assert len(summary["attempts"]) == 1 From 6bec645edbac3aa7ef8f1cc0af52fd4c8ed9a5f8 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 25 Oct 2026 07:30:36 -0700 Subject: [PATCH 12/12] docs: summarize flash 1017 dev34 recovery results --- docs/changelogs/0.10.x.md | 5 ++ docs/dev_notes/en/0.10.x.md | 117 +++++++++++++++++++++++++++++++++ docs/dev_notes/zh-CN/0.10.x.md | 93 ++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) diff --git a/docs/changelogs/0.10.x.md b/docs/changelogs/0.10.x.md index 3cbd0f2..951071b 100644 --- a/docs/changelogs/0.10.x.md +++ b/docs/changelogs/0.10.x.md @@ -4,6 +4,11 @@ All notable changes in the **0.10.x** release series are documented here. ## [Unreleased] +### Added +- Add retained-workspace Harbor verification, reviewed dependency preparation + with independent deadlines, and opt-in collection-error recovery while + preserving native verifier limits and prior evidence. + ### Changed - Clarify headless task contracts, output verification, and permitted references. Allow task-supplied source, executables, and tests unless the task restricts diff --git a/docs/dev_notes/en/0.10.x.md b/docs/dev_notes/en/0.10.x.md index 21aa68c..78f48c9 100644 --- a/docs/dev_notes/en/0.10.x.md +++ b/docs/dev_notes/en/0.10.x.md @@ -4,6 +4,123 @@ ## 0.10.0 - YYYY.MM.DD +### Flash 1017 dev34: 28/34 passes after same-version recovery + +From 2026-10-19 through 10-24, run the existing 34-task development pool using +`deepseek-flash` through DeepSeek's official Anthropic-compatible endpoint, then +complete follow-ups for three unscored tasks. On October 25, summarize the same +agent version while preserving recovery and resampling provenance: **28 passes, six scored +failures, zero unscored tasks, and an 82.35% pass proportion (28/34)**. Keep 31 +valid October 19 scores and use the newly generated October 20 solutions and +subsequent verification for the other three. Preserve each result's provenance +without rewriting the original run records. + +The evaluated core agent is frozen at main +`ce28b1e826a292d796ba64f13e79199b66f65758`. The three-task supplement has +byte-identical core code and the same 385-word headless prompt. Later commits +`40fdd041c370a96dc06f86b436eacc01ce1566d7` and +`f7383dd69cec928ce61e12c49f3850c2edb34be5` change benchmark installation, recovery, +and verification. Official documentation checked before execution maps the model +alias to DeepSeek-V4.1-Flash-1017; the alias is not an immutable checkpoint pin. +Preserve 100 model replies per task, at most 65,536 output tokens per request, +the native agent timeout minus 180 seconds as the work budget, and the original +per-attempt verification limits. + +| Record scope | Passed | Scored failures | Unscored | Pass proportion | +| --- | ---: | ---: | ---: | ---: | +| Original October 19 baseline of 34 tasks | 26 | 5 | 3 | 76.47% | +| Latest same-agent summary, including recovery and three resampled tasks | 28 | 6 | 0 | 82.35% | + +This summary can track the agent version's overall development-pool results. +It includes explicitly authorized recovery and resampling rather than the first +attempt on every task. The original three containers were deleted without +archiving `/app`, so generate one new solution for each on October 20 rather +than claiming to regrade those original artifacts. Later continuations rerun +only the retained solution's verifier, with no model calls, score feedback to +the model, or solution patches. + +**Provenance and final outcomes of the three originally unscored tasks** + +| Task | Latest solution and score provenance | Final outcome | +| --- | --- | --- | +| `multi-source-data-merger` | New October 20 trial `multi-source-data-merger__jNM6pZd`; first verification completes | Passed, 3/3 tests | +| `pytorch-model-cli` | New October 20 trial `pytorch-model-cli__P8vPNwH`; one dependency timeout followed by a valid score on attempt 2, which stops recovery | Failed, five tests pass and one fails; ten of 50 images are misclassified | +| `pytorch-model-recovery` | New October 20 trial `pytorch-model-recovery__nCs7Y4H`; prepare dependencies before official verification of that same solution on October 24 | Passed, 5/5 tests; the preceding six verifier attempts never reached pytest | + +The new CLI solution additionally applies `(pixel/255 - 0.1307)/0.3081`, whereas +the official `ToTensor()` pipeline does not perform this mean/std normalization. +The original CLI trial's collection failure also involves dependency damage +after the agent uses `dpkg --force-all`. Regeneration changes that environment +history, so preserving provenance matters. + +**Independent dependency preparation and formal verification** + +The model-recovery solution's previous six verification attempts exhaust their +900-second limits downloading PyTorch/CUDA dependencies, leaving the score +`null`. Add the reusable `verify-retained --dependency-timeout-sec 3600` workflow +and the `pytorch-model-recovery-v1` profile, bound to the exact task revision and +configuration/script hashes. From `/tmp`, use the original uv 0.9.5, Python +3.13, pytest 8.4.1, torch 2.7.1, and pytest-json-ctrf 0.3.5 requirements, running +only `pytest --version`. Do not import the submitted model or execute task +assertions. A preparation failure is a separate setup failure, with no formal +verification or score. + +| Phase | Independent limit | Actual duration | Outcome | +| --- | ---: | ---: | --- | +| Dependency preparation | 3,600 s | 1,127.589 s | Succeeded, exceeding the former 900-second limit | +| Unchanged official verification script | 900 s | 49.934 s | Reward 1.0, all five tests pass | + +The pytest session itself takes 5.00 seconds. The official script's remaining +APT refresh, uv installation, and dependency checks remain inside the 49.934 +seconds and the original 900-second limit. Preparation and formal verification +each execute once; the longer preparation budget does not extend test execution. +The frozen `/app/model.pt` SHA256 remains +`098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e`. +Check solution and test hashes before and after execution, restore original +trial files byte for byte, and retain the containers and prepared dependencies. + +**The six current scored failures** + +| Task | Latest official tests passed / failed | Observed problem | +| --- | ---: | --- | +| `db-wal-recovery` | 5 / 2 | Opening the original database before backup makes SQLite discard the WAL; guessed values omit WAL updates | +| `torch-pipeline-parallelism` | 2 / 2 | The microbatch count for `model.rotary_emb.fwd` does not match | +| `dna-assembly` | 0 / 1 | Missing the required nucleotide clamp before the BsaI recognition site | +| `regex-chess` | 2 / 2 | The regex pipeline allows a king to move into a square attacked by a knight | +| `build-pov-ray` | 2 / 1 | Missing `file_id.diz` fails the source-distribution check; executable and rendering checks pass | +| `pytorch-model-cli` | 5 / 1 | Extra input normalization differs from the official data pipeline, and prediction accuracy misses the requirement | + +The selected latest results execute 155 official pytest cases: 146 pass and nine +fail. Case counts supplement rather than replace the task-level 28/34. The +different October 12 baseline also scores 28/34, but its failure set differs: +sampler, KV storage, and SQL optimization pass here, while DNA, POV-Ray, and CLI +fail. Equal totals do not imply unchanged per-task outcomes or establish the +causal effect of one code change. + +The original 34 generations have a complete token-cost estimate of USD +1.860998832; the three regenerated solutions add USD 0.071903808, totaling USD +1.932902640. Later retained-solution verification adds zero model cost. These +are estimates, not reconciled bills. Review trajectories, artifacts, and tests +only after freezing all new outputs. No task-specific reference-solution +retrieval or hidden-verifier read was observed during model work. Preserve +Scheme's official pass while recording its use of host-interpreter introspection +to reset the depth counter. All 34 tasks have development exposure. Record +cache state, live dependencies, resampling, and the mutable model alias; do not +present this as a held-out capability score. + +The original experiment branch is `benchmark/dev34-flash1017-main-20261019`; +follow-ups use `benchmark/dev34-flash1017-recovery-20261020`. Results and workflow +changes are in [PR #52](https://github.com/minixalpha/nanoPyCodeAgent/pull/52). +The [original baseline](../../../benchmarks/harbor/reports/en/dev34-flash1017-main-20261019.md), +[three-task supplement](../../../benchmarks/harbor/reports/en/dev34-flash1017-recovery-20261020.md), +[three additional verification attempts](../../../benchmarks/harbor/reports/en/pytorch-model-recovery-verifier-20261021.md), +and [passed dependency-prepared result](../../../benchmarks/harbor/reports/en/pytorch-model-recovery-prepared-20261024.md), +together with their structured results, preserve per-task provenance, every +attempt, configurations, and hashes. Earlier scores and failure records remain +unchanged. This update summarizes existing experiments only: 209 local Harbor +checks and all three CI jobs on the result commit have passed. It starts no new +benchmark. + ### Common-main baseline for 34 tasks: 28 passes and six failures On 2026-10-12, combine the previously exercised 22 tasks and the twelve additions diff --git a/docs/dev_notes/zh-CN/0.10.x.md b/docs/dev_notes/zh-CN/0.10.x.md index 93373e2..38ae58c 100644 --- a/docs/dev_notes/zh-CN/0.10.x.md +++ b/docs/dev_notes/zh-CN/0.10.x.md @@ -4,6 +4,99 @@ ## 0.10.0 - YYYY.MM.DD +### Flash 1017 dev34：同版本恢复后 28/34 通过 + +2026-10-19 至 10-24，使用 DeepSeek 官方 Anthropic 兼容端点的 `deepseek-flash` +运行现有 34 题开发池，并完成三道未评分题的补充实验。10 月 25 日按保留恢复与 +重采样来源的同一 agent 版本口径汇总，最新结果为 **28 题通过、6 题评分失败、0 题未评分， +通过比例 82.35%（28/34）**。其中 31 题沿用 10 月 19 日的有效评分，另三题使用 +10 月 20 日重新生成的解答及后续验证；每题来源分别保留，原始运行记录不改写。 + +被评估的核心 agent 固定于 main `ce28b1e826a292d796ba64f13e79199b66f65758`， +三题补充实验的核心代码和 385 词 headless 提示词与其逐字节一致。后续 +`40fdd041c370a96dc06f86b436eacc01ce1566d7` 和 +`f7383dd69cec928ce61e12c49f3850c2edb34be5` 的变化属于 benchmark 安装、重试和 +验证流程。模型别名在正式运行前的官方文档中对应 DeepSeek-V4.1-Flash-1017； +该别名不是不可变 checkpoint 固定标识。保持每题 100 次模型回复、每次最多 +65,536 输出 token、原生 agent 时限减 180 秒的工作预算及原始逐次测试时限。 + +| 记录口径 | 通过 | 评分失败 | 未评分 | 通过比例 | +| --- | ---: | ---: | ---: | ---: | +| 10 月 19 日原始 34 题 baseline | 26 | 5 | 3 | 76.47% | +| 同一 agent 版本，含恢复与三题重采样的最新汇总 | 28 | 6 | 0 | 82.35% | + +本汇总可以用于跟踪该 agent 版本在开发池上的整体结果；它包含明确授权的恢复与 +重采样，不是 34 题全部首次尝试的 pass@1。原三题容器已删除且未归档 `/app`， +因此 10 月 20 日各生成一份新解答，不声称是在原三份产物上补评分。后续只重试 +保留解答的 verifier，没有再调用模型，也没有向模型反馈评分或修补解答。 + +**原三道未评分题的来源与最终结果** + +| 题目 | 最新解答与评分来源 | 最终结果 | +| --- | --- | --- | +| `multi-source-data-merger` | 10 月 20 日新试次 `multi-source-data-merger__jNM6pZd`；首次验证完成 | 通过，3/3 测试 | +| `pytorch-model-cli` | 10 月 20 日新试次 `pytorch-model-cli__P8vPNwH`；一次依赖超时后，第二次得到有效评分并停止 | 失败，5 项通过、1 项失败；50 张图有 10 张预测错误 | +| `pytorch-model-recovery` | 10 月 20 日新试次 `pytorch-model-recovery__nCs7Y4H`；同一份解答在 10 月 24 日先准备依赖，再执行官方验证 | 通过，5/5 测试；之前六次 verifier 均未进入 pytest | + +CLI 的新解答额外应用 `(pixel/255 - 0.1307)/0.3081`，而官方 `ToTensor()` +流程没有这一均值/方差归一化。原始 CLI 试次的收集错误还包含 agent 使用 +`dpkg --force-all` 后的依赖损坏；新解答会改变这段环境历史，因此来源需要保留。 + +**独立依赖准备与正式验证** + +模型恢复题原先六次验证都在下载 PyTorch/CUDA 依赖时耗尽 900 秒，评分保持 +`null`。新增可复用的 `verify-retained --dependency-timeout-sec 3600` 流程和 +绑定精确任务版本、配置与脚本哈希的 `pytorch-model-recovery-v1` profile。 +它在 `/tmp` 使用原始 uv 0.9.5、Python 3.13、pytest 8.4.1、torch 2.7.1、 +pytest-json-ctrf 0.3.5 依赖，仅执行 `pytest --version`，不导入提交模型或执行 +题目断言。准备失败单独记为 setup 失败，不启动正式验证，也不产生评分。 + +| 阶段 | 独立上限 | 实际耗时 | 结果 | +| --- | ---: | ---: | --- | +| 依赖准备 | 3,600 秒 | 1,127.589 秒 | 成功，超过原先 900 秒限制 | +| 保持原样的官方验证脚本 | 900 秒 | 49.934 秒 | 得分 1.0，五项测试全部通过 | + +pytest 会话本身耗时 5.00 秒；官方脚本剩余的 APT 更新、uv 安装及依赖检查仍计入 +49.934 秒及原 900 秒上限。准备和正式验证各执行一次，较长准备预算没有延长 +测试时限。冻结的 `/app/model.pt` SHA256 始终为 +`098642cbf41010a7fab7a9cf32995b49300cc3ccb1d724e7d57d729330f0bb4e`；执行前后 +核对解答与测试哈希，原 trial 文件逐字节恢复，容器和已准备的依赖继续保留。 + +**当前六道评分失败题** + +| 题目 | 最新官方测试通过 / 失败 | 观察到的问题 | +| --- | ---: | --- | +| `db-wal-recovery` | 5 / 2 | 备份前打开原数据库导致 WAL 被 SQLite 丢弃，推测值遗漏 WAL 更新 | +| `torch-pipeline-parallelism` | 2 / 2 | `model.rotary_emb.fwd` 的 microbatch 数量不匹配 | +| `dna-assembly` | 0 / 1 | BsaI 识别位点前缺少要求的核苷酸 clamp | +| `regex-chess` | 2 / 2 | 正则流水线允许王移入被马攻击的格子 | +| `build-pov-ray` | 2 / 1 | 缺少 `file_id.diz`，源码发行包检查失败；可执行文件与渲染检查通过 | +| `pytorch-model-cli` | 5 / 1 | 额外的输入归一化与官方数据流程不一致，预测准确率未达到要求 | + +最新选定的 34 份结果共执行 155 个官方 pytest 用例：146 通过、9 失败；用例数 +只作辅助记录，不替代题目级的 28/34。10 月 12 日的另一轮 baseline 也为 28/34， +但失败集合不同：本轮采样器、KV 存储和 SQL 优化通过，DNA、POV-Ray 与 CLI +失败，不能将相同总分理解为逐题不变或某一代码改动的因果效果。 + +原始 34 次生成的完整 token 费用估算为 USD 1.860998832，三题重新生成为 +USD 0.071903808，合计 USD 1.932902640；后续保留解答验证的模型费用为零。 +这是估算，不是账单对账。所有新输出冻结后才审阅轨迹、产物和测试；未观察到 +模型工作期间检索任务专用参考解答或读取隐藏 verifier。Scheme 的官方通过仍 +保留，同时记录其通过宿主解释器内省重置深度计数器的实现限制。这 34 题均有 +开发暴露，缓存、实时依赖、重采样与可变模型别名都需随结果记录，不作为保留集 +能力分数。 + +原始实验分支为 `benchmark/dev34-flash1017-main-20261019`，后续恢复分支为 +`benchmark/dev34-flash1017-recovery-20261020`，实验结果与流程变更收在 +[PR #52](https://github.com/minixalpha/nanoPyCodeAgent/pull/52)。 +[原始 baseline](../../../benchmarks/harbor/reports/zh-CN/dev34-flash1017-main-20261019.md)、 +[三题补充实验](../../../benchmarks/harbor/reports/zh-CN/dev34-flash1017-recovery-20261020.md)、 +[三次追加验证](../../../benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-verifier-20261021.md)和 +[独立依赖准备后的通过结果](../../../benchmarks/harbor/reports/zh-CN/pytorch-model-recovery-prepared-20261024.md) +及其结构化结果保存逐题来源、全部尝试、配置与哈希。既有分数及失败记录均不改写。 +本次只整理已有实验；209 项本地 Harbor 检查及结果提交的三项 CI 已通过，没有 +启动新的 benchmark。 + ### 34 题统一主线 baseline：28 题通过，6 题失败 2026-10-12，将原先已运行的 22 题与新增的 12 题合为开发池，全部在当时的 main