diff --git a/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py b/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py index 4679e15b..e9eb30cd 100644 --- a/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py +++ b/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py @@ -47,6 +47,12 @@ def _is_reasoning_model(model: str) -> bool: MAX_TOOL_OUTPUT_CHARS = 20_000 MAX_FILE_READ_CHARS = 60_000 +# ``run`` resends the whole conversation every turn instead of relying on the +# provider to remember it, so the transcript has to be bounded: 50 turns of +# 20k-character tool output would overflow any context window. Oldest whole +# turns are dropped first and the task statement is never dropped. +MAX_HISTORY_CHARS = 300_000 + # Repository checkout location inside the task environment. The pinned # swebenchpro task images set `WORKDIR /app` and reset the project's git # checkout in place there, and the task instruction says so verbatim ("I've @@ -285,6 +291,26 @@ def _truncate(value: str) -> str: omitted = len(value) - (2 * half) return f"{value[:half]}\n...[{omitted} characters omitted]...\n{value[-half:]}" + @staticmethod + def _history_size(blocks: list[list[dict[str, Any]]]) -> int: + return sum( + len(json.dumps(item, ensure_ascii=False)) + for block in blocks + for item in block + ) + + def _trim_history(self, blocks: list[list[dict[str, Any]]]) -> int: + """Drop whole oldest turns until the transcript fits the budget. + + Whole turns, never individual items: a ``function_call`` sent without its + matching ``function_call_output`` is a hard 400 from the Responses API. + """ + dropped = 0 + while len(blocks) > 1 and self._history_size(blocks) > MAX_HISTORY_CHARS: + blocks.pop(0) + dropped += 1 + return dropped + def _trace(self, event: dict[str, Any]) -> None: self.logs_dir.mkdir(parents=True, exist_ok=True) trace_path = self.logs_dir / "swe-bench-pro-trace.jsonl" @@ -447,17 +473,30 @@ async def run( context: AgentContext, ) -> None: self._patch_index = 0 - next_input: Any = instruction - previous_response_id: str | None = None input_tokens = 0 output_tokens = 0 cached_tokens = 0 + # The conversation is held HERE, not on the provider. An earlier version + # sent only the newest tool result and passed ``previous_response_id`` to + # let the server reconstruct the rest. That silently loses everything on + # any OpenAI-compatible gateway that accepts the field without backing it + # with a response store: from turn 2 the model saw a bare tool result with + # no task attached, so it re-explored the repository until the turn budget + # ran out. Measured on the 66-case sample with deepseek-v4-flash: 66/66 + # exhausted all 50 turns, 3163 of 3300 tool calls were ls/find/git/cat, and + # write_file, apply_patch and submit were called zero times, for a reward + # of 0.0000 on every case. + task_item: dict[str, Any] = {"role": "user", "content": instruction} + blocks: list[list[dict[str, Any]]] = [] for turn in range(1, MAX_TURNS + 1): + conversation: list[Any] = [task_item] + for block in blocks: + conversation.extend(block) request: dict[str, Any] = { "model": self._api_model, "instructions": INSTRUCTIONS, - "input": next_input, + "input": conversation, "tools": TOOLS, "max_output_tokens": 12_000, "parallel_tool_calls": False, @@ -466,8 +505,6 @@ async def run( # gpt-4.1 is a hard 400 on the very first turn. if _is_reasoning_model(self._api_model): request["reasoning"] = {"effort": "high"} - if previous_response_id is not None: - request["previous_response_id"] = previous_response_id response = await self._responses_create(**request) usage = response.usage input_tokens += self._usage_value(usage, "input_tokens") @@ -494,9 +531,23 @@ async def run( context.metadata = {"turns": turn, "trace": "swe-bench-pro-trace.jsonl"} break - next_input = [] + turn_items: list[dict[str, Any]] = [] + if response.output_text: + turn_items.append( + {"role": "assistant", "content": response.output_text} + ) submitted = False for call in calls: + # Echo the call itself before its output: the API matches the two + # by call_id, and an orphaned output is rejected. + turn_items.append( + { + "type": "function_call", + "call_id": call.call_id, + "name": call.name, + "arguments": call.arguments, + } + ) try: arguments = json.loads(call.arguments or "{}") except json.JSONDecodeError as error: @@ -531,7 +582,7 @@ async def run( else: result = {"error": f"unknown tool: {call.name}"} self._trace({"turn": turn, "tool": call.name, "result": result}) - next_input.append( + turn_items.append( { "type": "function_call_output", "call_id": call.call_id, @@ -540,10 +591,13 @@ async def run( ) if submitted: break + blocks.append(turn_items) + dropped = self._trim_history(blocks) + if dropped: + self._trace({"turn": turn, "event": "history_trimmed", "turns_dropped": dropped}) if submitted: context.metadata = {"turns": turn, "trace": "swe-bench-pro-trace.jsonl"} break - previous_response_id = response.id else: # Exhausting the turn budget is NOT a trial failure. The reward comes # from the task's hidden suite run against whatever the agent left in diff --git a/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py b/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py index 30701a20..05a92782 100644 --- a/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py +++ b/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py @@ -129,6 +129,96 @@ async def test_responses_create_retries_transient_errors(tmp_path, monkeypatch): } +class TwoTurnResponses: + """Runs one shell command, then submits. Records every request it received.""" + + def __init__(self): + self.calls = 0 + self.requests: list[dict] = [] + + async def create(self, **kwargs): + self.calls += 1 + self.requests.append(kwargs) + name = "run_shell" if self.calls == 1 else "submit" + arguments = '{"command": "git status --short"}' if self.calls == 1 else "{}" + return SimpleNamespace( + id=f"response-{self.calls}", + output=[ + SimpleNamespace( + type="function_call", + name=name, + arguments=arguments, + call_id=f"call-{self.calls}", + ) + ], + output_text="", + usage=SimpleNamespace( + input_tokens=10, + output_tokens=1, + input_tokens_details=SimpleNamespace(cached_tokens=0), + ), + ) + + +@pytest.mark.asyncio +async def test_conversation_is_resent_and_never_relies_on_the_provider( + tmp_path, monkeypatch +): + """The task and prior turns must be in the request, not on the server. + + Regression test for the bug that scored 0.0000 on all 66 sampled cases: + the agent sent only the newest tool result plus ``previous_response_id``, + so a gateway without a response store dropped the task entirely. + """ + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = SweBenchProAgent(logs_dir=tmp_path / "logs", model_name="openai/gpt-4o") + responses = TwoTurnResponses() + agent._client = SimpleNamespace(responses=responses) + context = SimpleNamespace( + metadata=None, + n_input_tokens=None, + n_output_tokens=None, + n_cache_tokens=None, + ) + + task = "Preserve this exact objective after every tool call." + await agent.run(task, FakeEnvironment(), context) + + assert len(responses.requests) == 2 + # Never delegate memory to the provider. + assert all("previous_response_id" not in r for r in responses.requests) + # The task survives into the second turn, carried by us. + second = responses.requests[1]["input"] + assert second[0] == {"role": "user", "content": task} + # ... and so does the first turn's call together with its output. + kinds = [item.get("type") for item in second[1:]] + assert "function_call" in kinds and "function_call_output" in kinds + call = next(i for i in second[1:] if i.get("type") == "function_call") + output = next(i for i in second[1:] if i.get("type") == "function_call_output") + assert call["call_id"] == output["call_id"], "orphaned output is a 400" + assert "git status --short" in call["arguments"] + + +def test_trim_history_drops_whole_turns_and_keeps_pairs_matched(tmp_path, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = SweBenchProAgent(logs_dir=tmp_path / "logs", model_name="openai/gpt-4o") + blocks = [ + [ + {"type": "function_call", "call_id": f"c{i}", "name": "run_shell", + "arguments": "{}"}, + {"type": "function_call_output", "call_id": f"c{i}", "output": "x" * 80_000}, + ] + for i in range(10) + ] + dropped = agent._trim_history(blocks) + + assert dropped > 0, "an oversized transcript must be trimmed" + assert agent._history_size(blocks) <= 300_000 + for block in blocks: + ids = [i["call_id"] for i in block] + assert len(set(ids)) == 1, "a turn must keep its call and output together" + + def test_agent_requires_model(tmp_path, monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "test-key") with pytest.raises(ValueError, match="requires a Harbor model"):