fix: keep the swe-bench-pro seed agent's task context across tool turns

harness-engineering-bench/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py

@@ -47,6 +47,12 @@ def _is_reasoning_model(model: str) -> bool:
 MAX_TOOL_OUTPUT_CHARS = 20_000
 MAX_FILE_READ_CHARS = 60_000
 
+# ``run`` resends the whole conversation every turn instead of relying on the
+# provider to remember it, so the transcript has to be bounded: 50 turns of
+# 20k-character tool output would overflow any context window. Oldest whole
+# turns are dropped first and the task statement is never dropped.
+MAX_HISTORY_CHARS = 300_000
+
 # Repository checkout location inside the task environment. The pinned
 # swebenchpro task images set `WORKDIR /app` and reset the project's git
 # checkout in place there, and the task instruction says so verbatim ("I've
@@ -285,6 +291,26 @@ def _truncate(value: str) -> str:
         omitted = len(value) - (2 * half)
         return f"{value[:half]}\n...[{omitted} characters omitted]...\n{value[-half:]}"
 
+    @staticmethod
+    def _history_size(blocks: list[list[dict[str, Any]]]) -> int:
+        return sum(
+            len(json.dumps(item, ensure_ascii=False))
+            for block in blocks
+            for item in block
+        )
+
+    def _trim_history(self, blocks: list[list[dict[str, Any]]]) -> int:
+        """Drop whole oldest turns until the transcript fits the budget.
+
+        Whole turns, never individual items: a ``function_call`` sent without its
+        matching ``function_call_output`` is a hard 400 from the Responses API.
+        """
+        dropped = 0
+        while len(blocks) > 1 and self._history_size(blocks) > MAX_HISTORY_CHARS:
+            blocks.pop(0)
+            dropped += 1
+        return dropped
+
     def _trace(self, event: dict[str, Any]) -> None:
         self.logs_dir.mkdir(parents=True, exist_ok=True)
         trace_path = self.logs_dir / "swe-bench-pro-trace.jsonl"
@@ -447,17 +473,30 @@ async def run(
         context: AgentContext,
     ) -> None:
         self._patch_index = 0
-        next_input: Any = instruction
-        previous_response_id: str | None = None
         input_tokens = 0
         output_tokens = 0
         cached_tokens = 0
+        # The conversation is held HERE, not on the provider. An earlier version
+        # sent only the newest tool result and passed ``previous_response_id`` to
+        # let the server reconstruct the rest. That silently loses everything on
+        # any OpenAI-compatible gateway that accepts the field without backing it
+        # with a response store: from turn 2 the model saw a bare tool result with
+        # no task attached, so it re-explored the repository until the turn budget
+        # ran out. Measured on the 66-case sample with deepseek-v4-flash: 66/66
+        # exhausted all 50 turns, 3163 of 3300 tool calls were ls/find/git/cat, and
+        # write_file, apply_patch and submit were called zero times, for a reward
+        # of 0.0000 on every case.
+        task_item: dict[str, Any] = {"role": "user", "content": instruction}
+        blocks: list[list[dict[str, Any]]] = []
 
         for turn in range(1, MAX_TURNS + 1):
+            conversation: list[Any] = [task_item]
+            for block in blocks:
+                conversation.extend(block)
             request: dict[str, Any] = {
                 "model": self._api_model,
                 "instructions": INSTRUCTIONS,
-                "input": next_input,
+                "input": conversation,
                 "tools": TOOLS,
                 "max_output_tokens": 12_000,
                 "parallel_tool_calls": False,
@@ -466,8 +505,6 @@ async def run(
             # gpt-4.1 is a hard 400 on the very first turn.
             if _is_reasoning_model(self._api_model):
                 request["reasoning"] = {"effort": "high"}
-            if previous_response_id is not None:
-                request["previous_response_id"] = previous_response_id
             response = await self._responses_create(**request)
             usage = response.usage
             input_tokens += self._usage_value(usage, "input_tokens")
@@ -494,9 +531,23 @@ async def run(
                 context.metadata = {"turns": turn, "trace": "swe-bench-pro-trace.jsonl"}
                 break
 
-            next_input = []
+            turn_items: list[dict[str, Any]] = []
+            if response.output_text:
+                turn_items.append(
+                    {"role": "assistant", "content": response.output_text}
+                )
             submitted = False
             for call in calls:
+                # Echo the call itself before its output: the API matches the two
+                # by call_id, and an orphaned output is rejected.
+                turn_items.append(
+                    {
+                        "type": "function_call",
+                        "call_id": call.call_id,
+                        "name": call.name,
+                        "arguments": call.arguments,
+                    }
+                )
                 try:
                     arguments = json.loads(call.arguments or "{}")
                 except json.JSONDecodeError as error:
@@ -531,7 +582,7 @@ async def run(
                     else:
                         result = {"error": f"unknown tool: {call.name}"}
                 self._trace({"turn": turn, "tool": call.name, "result": result})
-                next_input.append(
+                turn_items.append(
                     {
                         "type": "function_call_output",
                         "call_id": call.call_id,
@@ -540,10 +591,13 @@ async def run(
                 )
                 if submitted:
                     break
+            blocks.append(turn_items)
+            dropped = self._trim_history(blocks)
+            if dropped:
+                self._trace({"turn": turn, "event": "history_trimmed", "turns_dropped": dropped})
             if submitted:
                 context.metadata = {"turns": turn, "trace": "swe-bench-pro-trace.jsonl"}
                 break
-            previous_response_id = response.id
         else:
             # Exhausting the turn budget is NOT a trial failure. The reward comes
             # from the task's hidden suite run against whatever the agent left in

harness-engineering-bench/swe-bench-pro/baseline/target/tests/test_agent.py

@@ -129,6 +129,96 @@ async def test_responses_create_retries_transient_errors(tmp_path, monkeypatch):
     }
 
 
+class TwoTurnResponses:
+    """Runs one shell command, then submits. Records every request it received."""
+
+    def __init__(self):
+        self.calls = 0
+        self.requests: list[dict] = []
+
+    async def create(self, **kwargs):
+        self.calls += 1
+        self.requests.append(kwargs)
+        name = "run_shell" if self.calls == 1 else "submit"
+        arguments = '{"command": "git status --short"}' if self.calls == 1 else "{}"
+        return SimpleNamespace(
+            id=f"response-{self.calls}",
+            output=[
+                SimpleNamespace(
+                    type="function_call",
+                    name=name,
+                    arguments=arguments,
+                    call_id=f"call-{self.calls}",
+                )
+            ],
+            output_text="",
+            usage=SimpleNamespace(
+                input_tokens=10,
+                output_tokens=1,
+                input_tokens_details=SimpleNamespace(cached_tokens=0),
+            ),
+        )
+
+
+@pytest.mark.asyncio
+async def test_conversation_is_resent_and_never_relies_on_the_provider(
+    tmp_path, monkeypatch
+):
+    """The task and prior turns must be in the request, not on the server.
+
+    Regression test for the bug that scored 0.0000 on all 66 sampled cases:
+    the agent sent only the newest tool result plus ``previous_response_id``,
+    so a gateway without a response store dropped the task entirely.
+    """
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+    agent = SweBenchProAgent(logs_dir=tmp_path / "logs", model_name="openai/gpt-4o")
+    responses = TwoTurnResponses()
+    agent._client = SimpleNamespace(responses=responses)
+    context = SimpleNamespace(
+        metadata=None,
+        n_input_tokens=None,
+        n_output_tokens=None,
+        n_cache_tokens=None,
+    )
+
+    task = "Preserve this exact objective after every tool call."
+    await agent.run(task, FakeEnvironment(), context)
+
+    assert len(responses.requests) == 2
+    # Never delegate memory to the provider.
+    assert all("previous_response_id" not in r for r in responses.requests)
+    # The task survives into the second turn, carried by us.
+    second = responses.requests[1]["input"]
+    assert second[0] == {"role": "user", "content": task}
+    # ... and so does the first turn's call together with its output.
+    kinds = [item.get("type") for item in second[1:]]
+    assert "function_call" in kinds and "function_call_output" in kinds
+    call = next(i for i in second[1:] if i.get("type") == "function_call")
+    output = next(i for i in second[1:] if i.get("type") == "function_call_output")
+    assert call["call_id"] == output["call_id"], "orphaned output is a 400"
+    assert "git status --short" in call["arguments"]
+
+
+def test_trim_history_drops_whole_turns_and_keeps_pairs_matched(tmp_path, monkeypatch):
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+    agent = SweBenchProAgent(logs_dir=tmp_path / "logs", model_name="openai/gpt-4o")
+    blocks = [
+        [
+            {"type": "function_call", "call_id": f"c{i}", "name": "run_shell",
+             "arguments": "{}"},
+            {"type": "function_call_output", "call_id": f"c{i}", "output": "x" * 80_000},
+        ]
+        for i in range(10)
+    ]
+    dropped = agent._trim_history(blocks)
+
+    assert dropped > 0, "an oversized transcript must be trimmed"
+    assert agent._history_size(blocks) <= 300_000
+    for block in blocks:
+        ids = [i["call_id"] for i in block]
+        assert len(set(ids)) == 1, "a turn must keep its call and output together"
+
+
 def test_agent_requires_model(tmp_path, monkeypatch):
     monkeypatch.setenv("OPENAI_API_KEY", "test-key")
     with pytest.raises(ValueError, match="requires a Harbor model"):