From 5e721f178ef7bb95287509963e3d9480be448146 Mon Sep 17 00:00:00 2001 From: Michael Solomon Date: Sat, 31 Oct 2026 01:38:17 +0000 Subject: [PATCH] Don't await tasks when a scope is closed by GeneratorExit If an async generator holds a scope open across a yield, e.g. async def generate(): async with duet.new_scope() as scope: async for x in duet.pmap_aiter(scope, func, items): yield x and the caller stops iterating early (break, or an exception) without calling aclose(), the generator is closed synchronously when it is garbage collected, throwing GeneratorExit into new_scope. new_scope then interrupted the remaining tasks and awaited them, but a generator cannot yield while handling GeneratorExit, so this failed with an unraisable "RuntimeError: async generator ignored GeneratorExit". Worse, the abandoned new_scope frame had already set main_task.interruptible = False and never restored it, so the caller could no longer be interrupted, e.g. by a timeout: a following duet.sleep() hung forever. And because the interrupted tasks were still attached to the caller's task, their eventual failure (ScopeExit) would interrupt the caller, which had moved on. On GeneratorExit, interrupt the remaining tasks with ScopeExit without awaiting them, and detach them from main_task with a new Task.detach() method, so that their failure is dropped instead of reported to main_task. Behaviour change: when the generator is closed with `await agen.aclose()`, the tasks are now interrupted but finish in the background rather than before aclose() returns. Detached tasks are exactly the kind of tasks that no scope waits for, so this is robust only together with the fix that re-registers ready tasks when a root task raises in Scheduler.tick (google/duet#103). _DETACHED refuses interrupts, like a task that can't be interrupted, so that once the scheduler re-posts a KeyboardInterrupt its main task can't take (google/duet#104), a Ctrl-C that reaches a detached task is not lost. --- duet/api.py | 7 +++++ duet/api_test.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++- duet/impl.py | 25 +++++++++++++++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/duet/api.py b/duet/api.py index 900a9cc..072054f 100644 --- a/duet/api.py +++ b/duet/api.py @@ -353,6 +353,13 @@ async def finish_tasks(): yield scope if scope._spawned: await finish_tasks() + except GeneratorExit: + # An async generator holding this scope open is being closed, maybe + # synchronously (e.g. garbage collected), so we can't await our tasks. + # Interrupt them and detach them from main_task, which has moved on. + for task in tasks: + task.detach(ScopeExit()) + raise except BaseException: # Interrupt remaining tasks. if scope._spawned: diff --git a/duet/api_test.py b/duet/api_test.py index 84d3f03..61f14f7 100644 --- a/duet/api_test.py +++ b/duet/api_test.py @@ -22,11 +22,12 @@ import time import weakref from collections.abc import Iterator -from typing import List, Tuple +from typing import AsyncGenerator, List, Tuple import pytest import duet +import duet.impl as impl async def mul(a, b): @@ -682,6 +683,70 @@ async def cancel_task(): assert task_future.cancelled() + # Without the fix, closing the generator fails with an unraisable + # "RuntimeError: async generator ignored GeneratorExit", which pytest only + # reports as a warning by default. + @pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") + def test_async_generator_with_scope_closed_when_caller_fails(self) -> None: + """A generator with a scope can be closed after the caller fails mid-iteration.""" + + async def generate() -> AsyncGenerator[int, None]: + async def work(x: int) -> int: + await duet.completed_future(None) + return x * 2 + + async with duet.new_scope() as scope: + async for value in duet.pmap_aiter(scope, work, [1, 2, 3, 4]): + yield value + + async def main() -> None: + async for _ in generate(): + raise Fail() + + with pytest.raises(Fail): + duet.run(main) + + @pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") + @pytest.mark.parametrize("close", ["del", "aclose"]) + def test_async_generator_with_scope_closed_early(self, close: str) -> None: + """Tasks of a scope in a closed generator must not interrupt the caller.""" + log: List[str] = [] + child_started: duet.AwaitableFuture[None] = duet.AwaitableFuture() + + async def child() -> None: + child_started.set_result(None) + try: + await duet.AwaitableFuture() # Never completes. + except BaseException: + log.append("child interrupted") + raise + + async def generate() -> AsyncGenerator[int, None]: + async with duet.new_scope() as scope: + scope.spawn(child) + await child_started + yield 1 + yield 2 + + async def main() -> None: + gen = generate() + async for _ in gen: + break + if close == "del": + del gen # Closes the generator synchronously. + else: + await gen.aclose() + # Otherwise the sleep below would hang because its timeout cannot + # interrupt this task. + assert impl.current_task().interruptible + # The interrupted child fails, maybe on a later tick. That must not + # interrupt this task. + await duet.sleep(0.05) + log.append("main done") + + duet.run(main) + assert log == ["child interrupted", "main done"] + @pytest.mark.skipif( sys.version_info >= (3, 8), reason="inapplicable for python 3.8+ (can be removed)" diff --git a/duet/impl.py b/duet/impl.py index 6a0f3ef..ed18e89 100644 --- a/duet/impl.py +++ b/duet/impl.py @@ -35,6 +35,7 @@ Optional, Set, TypeVar, + Union, ) import duet.futuretools as futuretools @@ -60,12 +61,24 @@ def __init__(self, state: TaskState, expected_state: TaskState) -> None: LOCALS_TASK_SCHEDULER = "__duet_task_scheduler__" +class _Detached: + """Main task of detached tasks; drops their failures (see Task.detach).""" + + def interrupt(self, error: BaseException) -> bool: + # Refuse, like a task that can't be interrupted, so that a scheduler that + # re-posts a refused KeyboardInterrupt still delivers Ctrl-C to another task. + return False + + +_DETACHED = _Detached() + + class Task(Generic[T]): def __init__( self, awaitable: Awaitable[T], scheduler: "Scheduler", main_task: Optional["Task"] ) -> None: self.scheduler = scheduler - self.main_task = main_task + self.main_task: Optional[Union[Task, _Detached]] = main_task self._state = TaskState.WAITING self._future: Optional[Future] = None self._ready_future: futuretools.AwaitableFuture[None] = futuretools.AwaitableFuture() @@ -170,6 +183,16 @@ def interrupt(self, error: BaseException) -> None: if self._future: self._future.cancel() + def detach(self, error: BaseException) -> None: + """Detaches this task from its main task and interrupts it with error. + + The task keeps running until it handles the interrupt, but its failure + will no longer be reported to its former main task. This is used when + the main task cannot wait for this task to finish. + """ + self.main_task = _DETACHED + self.interrupt(error) + def close(self): self._generator.close() self.scheduler = None