From 40d2a19f645f2f007f99538470289f459dd1627c Mon Sep 17 00:00:00 2001 From: glayyiyi Date: Mon, 26 Oct 2026 07:59:05 +0800 Subject: [PATCH] fix(flash): stop after consecutive tool-less turns instead of looping An endpoint that keeps answering with nothing (empty content, stop_reason refusal) made an unbounded Flash run call the model about once a second for as long as the caller waited. flash.max_silent_turns (default 5) ends the run as failed with the last stop reason; 0 disables the guard. --- artemis/agents/flash/runner.py | 23 +++++++++ artemis/config/agent.py | 10 ++++ tests/unit/agents/test_flash_runner_ledger.py | 47 +++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 3ea3f6865..847ab8a9b 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -145,12 +145,14 @@ def __init__(self, ctx: ArtemisContext, goal: str, max_turns: int | None = None) try: cfg = load_agent_config() self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns + self.max_silent_turns = cfg.flash.max_silent_turns self.step_summarizer_cfg = cfg.flash.step_summarizer self.memory_runtime_cfg = cfg.memory.runtime self.transcript_cfg = cfg.memory.transcript self.chunking_cfg = cfg.memory.chunking except Exception: self.max_turns = max_turns if max_turns is not None else 0 + self.max_silent_turns = 5 self.step_summarizer_cfg = StepSummarizerConfig() self.memory_runtime_cfg = MemoryRuntimeConfig() self.transcript_cfg = MemoryTranscriptConfig() @@ -1056,6 +1058,7 @@ async def run(self, state: State) -> dict: current_xml_list = xml_list previous_turn: _TurnRecord | None = None pending_notices: list[str] = [] + silent_turns = 0 while limit is None or turns < limit: turns += 1 @@ -1131,11 +1134,31 @@ async def run(self, state: State) -> dict: if is_final: final_report = {"status": "failed", "explanation": raw_text} break + silent_turns += 1 + max_silent = getattr(self, "max_silent_turns", 5) or 0 + if max_silent and silent_turns >= max_silent: + stop_reason = (getattr(response, "response_metadata", None) or {}).get( + "stop_reason" + ) + logger.warning( + f"FlashRunner stopping: {silent_turns} consecutive turns without a tool call" + f" (last stop_reason={stop_reason!r})." + ) + final_report = { + "status": "failed", + "explanation": ( + f"The model called no tool for {silent_turns} consecutive turns" + f" (last stop_reason={stop_reason!r}, last reply {raw_text[:200]!r});" + " stopped instead of looping." + ), + } + break pending_notices.append(_NO_TOOL_CALL_NOTICE) ledger.stage_turn(messages[turn_base:]) previous_turn = _TurnRecord() continue + silent_turns = 0 # Process tool calls turn = _TurnRecord() ( diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 6fb00d398..35cefb0cf 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -769,6 +769,16 @@ class FlashProfileConfig(BaseModel): " task simply runs until it reports its status." ), ) + max_silent_turns: int = Field( + default=5, + ge=0, + description=( + "Stop the Flash loop as failed after this many consecutive turns in" + " which the model called no tool (e.g. empty replies, refusals); 0" + " disables the guard. Without it an endpoint that keeps answering" + " with nothing loops for as long as max_turns allows." + ), + ) explorer_mode: ExplorerVersion = Field( default="flash", description="Perception mode for Flash runner ('flash' for 1-shot detection).", diff --git a/tests/unit/agents/test_flash_runner_ledger.py b/tests/unit/agents/test_flash_runner_ledger.py index 57ff1ef87..fbc59d39f 100644 --- a/tests/unit/agents/test_flash_runner_ledger.py +++ b/tests/unit/agents/test_flash_runner_ledger.py @@ -684,3 +684,50 @@ def test_turn_record_reports_the_first_failure_of_a_multi_action_turn(): ] ) assert record.result()["error"] == "click rejected: coordinates out of bounds" + + +@pytest.mark.asyncio +async def test_run_stops_after_consecutive_silent_turns(mock_context): + """An endpoint that keeps answering with nothing (empty content, refusal) + must not loop an unbounded run: after ``max_silent_turns`` consecutive + tool-less turns the run fails with the stop reason, and no further call.""" + empty = AIMessage(content="", tool_calls=[], response_metadata={"stop_reason": "refusal"}) + responses = [empty] * 3 + [_report()] + with ( + patch("artemis.controllers.unified_controller.get_driver"), + patch( + "artemis.agents.flash.runner.capture_screenshot_and_parse_ui", + AsyncMock(return_value=_INITIAL_OBSERVATION), + ), + ): + runner = _make_runner(mock_context, responses, max_turns=0) + runner.max_silent_turns = 3 + result = await runner.run(State(initial_goal="Open Settings")) + + assert runner._invoke_model.await_count == 3 + assert result["status"] == "failed" + assert "3 consecutive turns" in result["explanation"] + assert "'refusal'" in result["explanation"] + + +@pytest.mark.asyncio +async def test_run_silent_turn_counter_resets_on_a_tool_call(mock_context): + """Only consecutive silent turns count: an action in between resets it.""" + silent = AIMessage(content="hmm", tool_calls=[]) + act = AIMessage( + content="tap", tool_calls=[{"name": "click", "args": {"target": 1}, "id": "tc1"}] + ) + responses = [silent, silent, act, silent, silent, _report()] + with ( + patch("artemis.controllers.unified_controller.get_driver"), + patch( + "artemis.agents.flash.runner.capture_screenshot_and_parse_ui", + AsyncMock(return_value=_INITIAL_OBSERVATION), + ), + ): + runner = _make_runner(mock_context, responses, max_turns=0) + runner.max_silent_turns = 3 + result = await runner.run(State(initial_goal="Open Settings")) + + assert result == {"status": "completed"} + assert runner._invoke_model.await_count == 6