Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions artemis/agents/flash/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,12 +145,14 @@ def __init__(self, ctx: ArtemisContext, goal: str, max_turns: int | None = None)
try:
cfg = load_agent_config()
self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns
self.max_silent_turns = cfg.flash.max_silent_turns
self.step_summarizer_cfg = cfg.flash.step_summarizer
self.memory_runtime_cfg = cfg.memory.runtime
self.transcript_cfg = cfg.memory.transcript
self.chunking_cfg = cfg.memory.chunking
except Exception:
self.max_turns = max_turns if max_turns is not None else 0
self.max_silent_turns = 5
self.step_summarizer_cfg = StepSummarizerConfig()
self.memory_runtime_cfg = MemoryRuntimeConfig()
self.transcript_cfg = MemoryTranscriptConfig()
Expand Down Expand Up @@ -1056,6 +1058,7 @@ async def run(self, state: State) -> dict:
current_xml_list = xml_list
previous_turn: _TurnRecord | None = None
pending_notices: list[str] = []
silent_turns = 0

while limit is None or turns < limit:
turns += 1
Expand Down Expand Up @@ -1131,11 +1134,31 @@ async def run(self, state: State) -> dict:
if is_final:
final_report = {"status": "failed", "explanation": raw_text}
break
silent_turns += 1
max_silent = getattr(self, "max_silent_turns", 5) or 0
if max_silent and silent_turns >= max_silent:
stop_reason = (getattr(response, "response_metadata", None) or {}).get(
"stop_reason"
)
logger.warning(
f"FlashRunner stopping: {silent_turns} consecutive turns without a tool call"
f" (last stop_reason={stop_reason!r})."
)
final_report = {
"status": "failed",
"explanation": (
f"The model called no tool for {silent_turns} consecutive turns"
f" (last stop_reason={stop_reason!r}, last reply {raw_text[:200]!r});"
" stopped instead of looping."
),
}
break
pending_notices.append(_NO_TOOL_CALL_NOTICE)
ledger.stage_turn(messages[turn_base:])
previous_turn = _TurnRecord()
continue

silent_turns = 0
# Process tool calls
turn = _TurnRecord()
(
Expand Down
10 changes: 10 additions & 0 deletions artemis/config/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,16 @@ class FlashProfileConfig(BaseModel):
" task simply runs until it reports its status."
),
)
max_silent_turns: int = Field(
default=5,
ge=0,
description=(
"Stop the Flash loop as failed after this many consecutive turns in"
" which the model called no tool (e.g. empty replies, refusals); 0"
" disables the guard. Without it an endpoint that keeps answering"
" with nothing loops for as long as max_turns allows."
),
)
explorer_mode: ExplorerVersion = Field(
default="flash",
description="Perception mode for Flash runner ('flash' for 1-shot detection).",
Expand Down
47 changes: 47 additions & 0 deletions tests/unit/agents/test_flash_runner_ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -684,3 +684,50 @@ def test_turn_record_reports_the_first_failure_of_a_multi_action_turn():
]
)
assert record.result()["error"] == "click rejected: coordinates out of bounds"


@pytest.mark.asyncio
async def test_run_stops_after_consecutive_silent_turns(mock_context):
"""An endpoint that keeps answering with nothing (empty content, refusal)
must not loop an unbounded run: after ``max_silent_turns`` consecutive
tool-less turns the run fails with the stop reason, and no further call."""
empty = AIMessage(content="", tool_calls=[], response_metadata={"stop_reason": "refusal"})
responses = [empty] * 3 + [_report()]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
runner.max_silent_turns = 3
result = await runner.run(State(initial_goal="Open Settings"))

assert runner._invoke_model.await_count == 3
assert result["status"] == "failed"
assert "3 consecutive turns" in result["explanation"]
assert "'refusal'" in result["explanation"]


@pytest.mark.asyncio
async def test_run_silent_turn_counter_resets_on_a_tool_call(mock_context):
"""Only consecutive silent turns count: an action in between resets it."""
silent = AIMessage(content="hmm", tool_calls=[])
act = AIMessage(
content="tap", tool_calls=[{"name": "click", "args": {"target": 1}, "id": "tc1"}]
)
responses = [silent, silent, act, silent, silent, _report()]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
runner.max_silent_turns = 3
result = await runner.run(State(initial_goal="Open Settings"))

assert result == {"status": "completed"}
assert runner._invoke_model.await_count == 6
Loading