From 69f031af1b008d6b0c271a2f3e86e4d77eccd4da Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Wed, 29 Jul 2026 09:23:53 +0300 Subject: [PATCH] fix: keep the swe-bench-pro seed agent's task context across tool turns The seed delegated conversation state to the Responses API field `previous_response_id`. The LiteLLM gateway accepts that field without backing it with a response store, so from turn two the model received a bare tool result with no task attached. Measured over the 66-case sampled test split: write_file, apply_patch and submit were called ZERO times, 3163 of 3300 tool calls were ls/find/git/cat, all 66 exhausted the 50-turn budget in under seven minutes each, and the split scored 0.0000. With the transcript held locally and resent, the same 66 cases score 0.2000 and 10 development cases score 0.3000. The transcript is bounded at MAX_HISTORY_CHARS and trimmed by whole turns, never by individual items: a function_call sent without its matching function_call_output is a hard 400 from the Responses API. Co-Authored-By: Claude Opus 5 (1M context) --- .../target/src/swebench_pro_agent/agent.py | 70 +++++++++++++-- .../baseline/target/tests/test_agent.py | 90 +++++++++++++++++++ 2 files changed, 152 insertions(+), 8 deletions(-) diff --git a/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py b/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py index 4679e15b..e9eb30cd 100644 --- a/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py +++ b/harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py @@ -47,6 +47,12 @@ def _is_reasoning_model(model: str) -> bool: MAX_TOOL_OUTPUT_CHARS = 20_000 MAX_FILE_READ_CHARS = 60_000 +# ``run`` resends the whole conversation every turn instead of relying on the +# provider to remember it, so the transcript has to be bounded: 50 turns of +# 20k-character tool output would overflow any context window. Oldest whole +# turns are dropped first and the task statement is never dropped. +MAX_HISTORY_CHARS = 300_000 + # Repository checkout location inside the task environment. The pinned # swebenchpro task images set `WORKDIR /app` and reset the project's git # checkout in place there, and the task instruction says so verbatim ("I've @@ -285,6 +291,26 @@ def _truncate(value: str) -> str: omitted = len(value) - (2 * half) return f"{value[:half]}\n...[{omitted} characters omitted]...\n{value[-half:]}" + @staticmethod + def _history_size(blocks: list[list[dict[str, Any]]]) -> int: + return sum( + len(json.dumps(item, ensure_ascii=False)) + for block in blocks + for item in block + ) + + def _trim_history(self, blocks: list[list[dict[str, Any]]]) -> int: + """Drop whole oldest turns until the transcript fits the budget. + + Whole turns, never individual items: a ``function_call`` sent without its + matching ``function_call_output`` is a hard 400 from the Responses API. + """ + dropped = 0 + while len(blocks) > 1 and self._history_size(blocks) > MAX_HISTORY_CHARS: + blocks.pop(0) + dropped += 1 + return dropped + def _trace(self, event: dict[str, Any]) -> None: self.logs_dir.mkdir(parents=True, exist_ok=True) trace_path = self.logs_dir / "swe-bench-pro-trace.jsonl" @@ -447,17 +473,30 @@ async def run( context: AgentContext, ) -> None: self._patch_index = 0 - next_input: Any = instruction - previous_response_id: str | None = None input_tokens = 0 output_tokens = 0 cached_tokens = 0 + # The conversation is held HERE, not on the provider. An earlier version + # sent only the newest tool result and passed ``previous_response_id`` to + # let the server reconstruct the rest. That silently loses everything on + # any OpenAI-compatible gateway that accepts the field without backing it + # with a response store: from turn 2 the model saw a bare tool result with + # no task attached, so it re-explored the repository until the turn budget + # ran out. Measured on the 66-case sample with deepseek-v4-flash: 66/66 + # exhausted all 50 turns, 3163 of 3300 tool calls were ls/find/git/cat, and + # write_file, apply_patch and submit were called zero times, for a reward + # of 0.0000 on every case. + task_item: dict[str, Any] = {"role": "user", "content": instruction} + blocks: list[list[dict[str, Any]]] = [] for turn in range(1, MAX_TURNS + 1): + conversation: list[Any] = [task_item] + for block in blocks: + conversation.extend(block) request: dict[str, Any] = { "model": self._api_model, "instructions": INSTRUCTIONS, - "input": next_input, + "input": conversation, "tools": TOOLS, "max_output_tokens": 12_000, "parallel_tool_calls": False, @@ -466,8 +505,6 @@ async def run( # gpt-4.1 is a hard 400 on the very first turn. if _is_reasoning_model(self._api_model): request["reasoning"] = {"effort": "high"} - if previous_response_id is not None: - request["previous_response_id"] = previous_response_id response = await self._responses_create(**request) usage = response.usage input_tokens += self._usage_value(usage, "input_tokens") @@ -494,9 +531,23 @@ async def run( context.metadata = {"turns": turn, "trace": "swe-bench-pro-trace.jsonl"} break - next_input = [] + turn_items: list[dict[str, Any]] = [] + if response.output_text: + turn_items.append( + {"role": "assistant", "content": response.output_text} + ) submitted = False for call in calls: + # Echo the call itself before its output: the API matches the two + # by call_id, and an orphaned output is rejected. + turn_items.append( + { + "type": "function_call", + "call_id": call.call_id, + "name": call.name, + "arguments": call.arguments, + } + ) try: arguments = json.loads(call.arguments or "{}") except json.JSONDecodeError as error: @@ -531,7 +582,7 @@ async def run( else: result = {"error": f"unknown tool: {call.name}"} self._trace({"turn": turn, "tool": call.name, "result": result}) - next_input.append( + turn_items.append( { "type": "function_call_output", "call_id": call.call_id, @@ -540,10 +591,13 @@ async def run( ) if submitted: break + blocks.append(turn_items) + dropped = self._trim_history(blocks) + if dropped: + self._trace({"turn": turn, "event": "history_trimmed", "turns_dropped": dropped}) if submitted: context.metadata = {"turns": turn, "trace": "swe-bench-pro-trace.jsonl"} break - previous_response_id = response.id else: # Exhausting the turn budget is NOT a trial failure. The reward comes # from the task's hidden suite run against whatever the agent left in diff --git a/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py b/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py index 30701a20..05a92782 100644 --- a/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py +++ b/harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py @@ -129,6 +129,96 @@ async def test_responses_create_retries_transient_errors(tmp_path, monkeypatch): } +class TwoTurnResponses: + """Runs one shell command, then submits. Records every request it received.""" + + def __init__(self): + self.calls = 0 + self.requests: list[dict] = [] + + async def create(self, **kwargs): + self.calls += 1 + self.requests.append(kwargs) + name = "run_shell" if self.calls == 1 else "submit" + arguments = '{"command": "git status --short"}' if self.calls == 1 else "{}" + return SimpleNamespace( + id=f"response-{self.calls}", + output=[ + SimpleNamespace( + type="function_call", + name=name, + arguments=arguments, + call_id=f"call-{self.calls}", + ) + ], + output_text="", + usage=SimpleNamespace( + input_tokens=10, + output_tokens=1, + input_tokens_details=SimpleNamespace(cached_tokens=0), + ), + ) + + +@pytest.mark.asyncio +async def test_conversation_is_resent_and_never_relies_on_the_provider( + tmp_path, monkeypatch +): + """The task and prior turns must be in the request, not on the server. + + Regression test for the bug that scored 0.0000 on all 66 sampled cases: + the agent sent only the newest tool result plus ``previous_response_id``, + so a gateway without a response store dropped the task entirely. + """ + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = SweBenchProAgent(logs_dir=tmp_path / "logs", model_name="openai/gpt-4o") + responses = TwoTurnResponses() + agent._client = SimpleNamespace(responses=responses) + context = SimpleNamespace( + metadata=None, + n_input_tokens=None, + n_output_tokens=None, + n_cache_tokens=None, + ) + + task = "Preserve this exact objective after every tool call." + await agent.run(task, FakeEnvironment(), context) + + assert len(responses.requests) == 2 + # Never delegate memory to the provider. + assert all("previous_response_id" not in r for r in responses.requests) + # The task survives into the second turn, carried by us. + second = responses.requests[1]["input"] + assert second[0] == {"role": "user", "content": task} + # ... and so does the first turn's call together with its output. + kinds = [item.get("type") for item in second[1:]] + assert "function_call" in kinds and "function_call_output" in kinds + call = next(i for i in second[1:] if i.get("type") == "function_call") + output = next(i for i in second[1:] if i.get("type") == "function_call_output") + assert call["call_id"] == output["call_id"], "orphaned output is a 400" + assert "git status --short" in call["arguments"] + + +def test_trim_history_drops_whole_turns_and_keeps_pairs_matched(tmp_path, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = SweBenchProAgent(logs_dir=tmp_path / "logs", model_name="openai/gpt-4o") + blocks = [ + [ + {"type": "function_call", "call_id": f"c{i}", "name": "run_shell", + "arguments": "{}"}, + {"type": "function_call_output", "call_id": f"c{i}", "output": "x" * 80_000}, + ] + for i in range(10) + ] + dropped = agent._trim_history(blocks) + + assert dropped > 0, "an oversized transcript must be trimmed" + assert agent._history_size(blocks) <= 300_000 + for block in blocks: + ids = [i["call_id"] for i in block] + assert len(set(ids)) == 1, "a turn must keep its call and output together" + + def test_agent_requires_model(tmp_path, monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "test-key") with pytest.raises(ValueError, match="requires a Harbor model"):