diff --git a/harness-engineering-bench/gaia/baseline/build.shell.yaml b/harness-engineering-bench/gaia/baseline/build.shell.yaml index cfb7fcda..e11ac695 100644 --- a/harness-engineering-bench/gaia/baseline/build.shell.yaml +++ b/harness-engineering-bench/gaia/baseline/build.shell.yaml @@ -160,6 +160,16 @@ inference_gateway: request_log_attribution: true producer: allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + # Pin the gpt-5.6 optimizers to one upstream deployment. An unqualified model group + # load-balances across deployments, and Responses-API encrypted reasoning is + # decryptable only by the deployment that produced it, so every turn after the + # first fails `invalid_encrypted_content`. Measured here 2026-08-02 10:12Z: the + # gpt-5.6-sol x codex canary died on it. Same fault and same fix as + # swe-atlas-qna/baseline/build.yaml. Applied AFTER the allow-list check, so + # allowed_models still governs what the optimizer may ask for. + model_aliases: + gpt-5.6-sol: azure_ai/gpt-5.6-sol + gpt-5.6-terra: azure_ai/gpt-5.6-terra max_concurrency: 8 # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is # the spend control, so a token cap only needs to stop a runaway. diff --git a/harness-engineering-bench/officeqa/baseline/build.yaml b/harness-engineering-bench/officeqa/baseline/build.yaml index 6702a552..495a1585 100644 --- a/harness-engineering-bench/officeqa/baseline/build.yaml +++ b/harness-engineering-bench/officeqa/baseline/build.yaml @@ -154,6 +154,19 @@ inference_gateway: request_log_attribution: true producer: allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + # Pin every gpt-5.x optimizer to one upstream deployment. An unqualified model group + # load-balances across deployments, and Responses-API encrypted reasoning is + # decryptable only by the deployment that produced it, so every turn after the first + # fails invalid_encrypted_content. Measured here 2026-08-03 09:55Z: the generational + # sweep's gpt-5.5 x codex cell died on exactly that ("Received Model Group=gpt-5.5, + # Available Model Group Fallbacks=None"). Same fault and same fix as + # swe-atlas-qna/baseline/build.yaml and gaia/baseline/build.shell.yaml. Applied + # AFTER the allow-list check, so allowed_models still governs what may be asked for. + model_aliases: + gpt-5.1: azure_ai/gpt-5.1 + gpt-5.2: azure_ai/gpt-5.2 + gpt-5.4: azure_ai/gpt-5.4 + gpt-5.5: azure_ai/gpt-5.5 max_concurrency: 8 # Token caps are a runaway backstop, not the spend control: the work is already # bounded by the agent case budget above and by the fixed held-out set, so a cap diff --git a/harness-engineering-bench/scripts/rescore_candidate.py b/harness-engineering-bench/scripts/rescore_candidate.py index 7a4c9357..c922d380 100644 --- a/harness-engineering-bench/scripts/rescore_candidate.py +++ b/harness-engineering-bench/scripts/rescore_candidate.py @@ -139,6 +139,10 @@ def harbor_command( attempts: int, concurrency: int, model: str, + agent: str | None = None, + setup_timeout_multiplier: float | None = None, + agent_env: list[str] | None = None, + extra_requirements: list[str] | None = None, ) -> list[str]: """Mirror vero/src/vero/harbor/backend.py::_command and the baseline runs. @@ -159,23 +163,83 @@ def harbor_command( "--no-config", "--no-env-file", "--project", str(workspace), "--with", build.get("harbor_requirement", "harbor[modal]==0.20.0"), + # Harbor-native harnesses import their framework in the ORCHESTRATOR, not the + # task container, and harbor does not declare those imports: dspy-rlm dies on + # ModuleNotFoundError: No module named 'dspy' before the agent ever starts. + *[arg for req in (extra_requirements or []) for arg in ("--with", req)], "harbor", "run", "--yes", *source_args, - "--agent-import-path", build["agent_import_path"], + # A reference run swaps the seed program for an installed harness + # (claude-code, opencode, ...) and changes NOTHING else -- same dataset + # ref, same partition, same rounds, same aggregation -- so the number + # stays comparable to baseline_reward and to every contestant. The target + # model stays pinned, so the harness is the only variable. + *(["--agent", agent] if agent + else ["--agent-import-path", build["agent_import_path"]]), "-e", resolve_param(build.get("environment_name", "modal")), "-m", str(model), "-n", str(concurrency), "--n-attempts", str(attempts), "--jobs-dir", str(jobs_dir), ] + # opencode's install (nvm -> node 22 -> npm -g) is far heavier than the seed's + # zero setup and can exceed harbor's default. swe-atlas already gives its + # OPTIMIZER agent a 4x multiplier for the same reason. + if setup_timeout_multiplier: + command += ["--agent-setup-timeout-multiplier", str(setup_timeout_multiplier)] + for pair in agent_env or []: + command += ["--ae", pair] for task in tasks: command.extend(["-i", task]) command.extend(str(a) for a in build.get("extra_harbor_args", [])) return command +# Convention (2026-07-31, adopted on Greptile's PR #75 catch) -- IDENTICAL to +# runs/recompute.py, which produced the pinned baselines. A trial the HARNESS killed +# scores 0: the harness owns its own install, its context management and its step +# budget, so a reference harness that cannot install or cannot finish must pay for it +# exactly as a candidate does at finalization, which zero-fills dead attempts. A trial +# the PLATFORM killed is dropped: a retry cannot score a trial that never ran, and +# zero-filling infra bakes outage luck into the number. +# +# Dropping them instead -- which this script did until 2026-08-02 -- silently inflates +# every reference score by scoring only the trials that survived. It cost 73 zeroes +# across the reference grid, worst on swe-atlas x mini-swe-agent (n=122 of 150). +HARNESS_EXCEPTIONS = { + "RuntimeError", # swe-atlas seed: empty-completion fail-fast + "UnicodeDecodeError", # terminal-bench seed: undecodable command output + "AgentTimeoutError", # wall-clock exhaustion: the step budget is harness-owned + "BadRequestError", # gpt-oss 128k overflow: context management is harness-owned + "NonZeroAgentExitCodeError", # the harness crashed or never installed its binary + "AgentSetupTimeoutError", # the harness owns how long its own install takes + "AgentAuthenticationError", # subclasses NonZeroAgentExitCodeError: the CLI reports + # no login, i.e. the harness did not read a credential + # surface we set (goose reads OPENAI_HOST, not _BASE_URL) + "AdapterParseError", # dspy's own output parser gave up: harness-owned +} +INFRA_EXCEPTIONS = { + "RateLimitError", + "ApiRateLimitError", + "NetworkConnectionError", + "VerifierTimeoutError", + "EnvironmentStartTimeoutError", + "SandboxFilesystemNotFoundError", + "AddTestsDirError", # harbor could not stage the tests: platform, not agent + "ConnectionError", + "UnknownApiError", # harbor's ApiError subclass for a provider error it + # could not classify: upstream, like the two RateLimits + "RewardFileNotFoundError", # the verifier ran and produced no reward file + "CancelledError", # harbor cancelled the trial (job.py CANCELLED_ERROR_TYPE) +} +# ValueError is deliberately in NEITHER set. Both instances on disk are harbor's +# "ContextVar ... was created in a different Context" orchestration bug, which is +# platform-side, but the name is generic enough that an agent-side ValueError would +# land here too. A rescore that meets one should stop and be looked at, not guess. + + def trial_rewards(round_dir: Path) -> list[float]: - """Same extraction as runs/recompute.py, so numbers are comparable.""" + """Same extraction AND the same failure convention as runs/recompute.py.""" rewards = [] for path in glob.glob(f"{round_dir}/**/result.json", recursive=True): if "/verifier/" in path: @@ -189,8 +253,18 @@ def trial_rewards(round_dir: Path) -> list[float]: verifier = data.get("verifier_result") or {} block = verifier.get("rewards") reward = block.get("reward") if isinstance(block, dict) else verifier.get("reward") + exception = (data.get("exception_info") or {}).get("exception_type") if reward is not None: rewards.append(float(reward)) + elif exception in HARNESS_EXCEPTIONS: + rewards.append(0.0) # the harness killed it: price the defect + elif exception in INFRA_EXCEPTIONS: + continue # the platform killed it: drop, do not bake in outage luck + elif exception: + message = (data.get("exception_info") or {}).get("exception_message") or "" + sys.exit(f"unclassified exception {exception!r} in {round_dir}: add it to " + "HARNESS_EXCEPTIONS or INFRA_EXCEPTIONS before quoting a number" + f"\n {message.splitlines()[0][:200] if message else '(no message)'}") return rewards @@ -222,12 +296,36 @@ def main() -> int: help="independent rounds, pooled (default 3, as the baselines)") parser.add_argument("--attempts", type=int, default=1, help="attempts per case within a round (default 1)") + parser.add_argument("--agent", + help="run an INSTALLED harbor harness (claude-code, opencode, " + "...) instead of the seed program, at the benchmark's " + "pinned model. Measures a SOTA reference, not a bound.") + parser.add_argument("--harbor-requirement", + help="override build.yaml's harbor_requirement for this run " + "only. Also relaxes the copied workspace's own harbor pin, " + "which build.yaml and target/pyproject.toml must otherwise " + "keep in lockstep (see a70c572) or uv cannot resolve.") + parser.add_argument("--setup-timeout-multiplier", type=float, + help="scale harbor's agent-setup timeout (installed harnesses " + "with heavy toolchains need this)") + parser.add_argument("--agent-env", action="append", metavar="KEY=VALUE", + default=[], + help="extra env var for the agent (repeatable). goose reads " + "OPENAI_HOST/OPENAI_BASE_PATH rather than OPENAI_BASE_URL, " + "so it needs the proxy pointed at explicitly.") + parser.add_argument("--with-requirement", action="append", metavar="SPEC", + default=[], dest="extra_requirements", + help="extra package for the orchestrator uv env (repeatable). " + "Harbor-native harnesses import their framework here and " + "harbor does not declare it -- dspy-rlm needs 'dspy'.") parser.add_argument("--concurrency", type=int, default=24) parser.add_argument("--output", help="output dir (default: a temp dir)") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() build, build_path = load_build(args.benchmark) + if args.harbor_requirement: + build["harbor_requirement"] = args.harbor_requirement outdir = Path(args.output).resolve() if args.output else Path( tempfile.mkdtemp(prefix=f"rescore-{args.benchmark}-")) outdir.mkdir(parents=True, exist_ok=True) @@ -245,6 +343,15 @@ def main() -> int: "__pycache__", "*.pyc", ".venv", ".git")) version = "seed" log(f"seed harness from {origin}") + if args.harbor_requirement: + pyproject = workspace / "pyproject.toml" + if pyproject.is_file(): + import re as _re + text = pyproject.read_text() + relaxed = _re.sub(r'"harbor==[^"]+"', '"harbor"', text) + if relaxed != text: + pyproject.write_text(relaxed) + log("relaxed the copied workspace's harbor pin so the override resolves") else: session_dir = open_session(args.session, outdir) version = shipped_version( @@ -284,7 +391,10 @@ def main() -> int: command = harbor_command( build=build, build_path=build_path, workspace=workspace, tasks=tasks, jobs_dir=jobs_dir, attempts=args.attempts, concurrency=args.concurrency, - model=args.model or build["model"], + model=args.model or build["model"], agent=args.agent, + setup_timeout_multiplier=args.setup_timeout_multiplier, + agent_env=args.agent_env, + extra_requirements=args.extra_requirements, ) if args.dry_run: print(" ".join(command)) diff --git a/harness-engineering-bench/swe-atlas-qna/baseline/build.yaml b/harness-engineering-bench/swe-atlas-qna/baseline/build.yaml index 52e66d45..a9174a23 100644 --- a/harness-engineering-bench/swe-atlas-qna/baseline/build.yaml +++ b/harness-engineering-bench/swe-atlas-qna/baseline/build.yaml @@ -165,6 +165,22 @@ inference_gateway: - "${optimizer_model:-openai/gpt-5.4}" - "${optimizer_model_bare:-gpt-5.4}" - "${optimizer_aux_model:-gpt-5.4-nano}" + # Pin the gpt-5.6 optimizers to one upstream deployment. An unqualified model + # group is load-balanced across deployments, and Responses-API encrypted + # reasoning content is decryptable only by the deployment that produced it, so + # replaying it against a sibling fails `invalid_encrypted_content` on every turn + # after the first. Measured 2026-08-01 on the bare group: 8 of 8 replays failed, + # against 0 of 5 on azure_ai. It killed both gpt-5.6-sol cells here at 2m58s -- + # opencode AND codex, so this is not a codex-only fault as terminal-bench's + # instance suggested. + # + # Applied AFTER the allow-list check, so allowed_models still governs what the + # optimizer may ask for and a cell's label still means what it says. Keys outside + # the allow-list are permitted and inert (specs.py validate_aliases), which is why + # both models can sit here in a config shared by every cell of the grid. + model_aliases: + gpt-5.6-sol: azure_ai/gpt-5.6-sol + gpt-5.6-terra: azure_ai/gpt-5.6-terra max_concurrency: 8 # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is # the spend control, so a token cap only needs to stop a runaway. diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 90562c56..d93866d6 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -528,6 +528,20 @@ def _compiled_run_environment( environment["ANTHROPIC_BASE_URL"] = producer_base_url[: -len("/v1")] if ( producer_base_url.endswith("/v1") ) else producer_base_url + # openhands-sdk reads its own LLM_* surface and ignores OPENAI_*/ANTHROPIC_* + # entirely (harbor/agents/installed/openhands_sdk.py). The producer token is + # minted per run, so it cannot be supplied through build.yaml agent_env -- + # it has to be injected here alongside the other two surfaces. Same + # unconditional rule: a harness that does not read these ignores them. + environment["LLM_API_KEY"] = producer_api_key + environment["LLM_BASE_URL"] = producer_base_url + # goose reads OPENAI_HOST + OPENAI_BASE_PATH and never OPENAI_BASE_URL, and it + # JOINS them, so the host must not already carry /v1 or the request goes to + # .../v1/v1/chat/completions and the proxy 403s. + environment["OPENAI_HOST"] = producer_base_url[: -len("/v1")] if ( + producer_base_url.endswith("/v1") + ) else producer_base_url + environment["OPENAI_BASE_PATH"] = "v1/chat/completions" return environment