diff --git a/duet/api.py b/duet/api.py index 900a9cc..072054f 100644 --- a/duet/api.py +++ b/duet/api.py @@ -353,6 +353,13 @@ async def finish_tasks(): yield scope if scope._spawned: await finish_tasks() + except GeneratorExit: + # An async generator holding this scope open is being closed, maybe + # synchronously (e.g. garbage collected), so we can't await our tasks. + # Interrupt them and detach them from main_task, which has moved on. + for task in tasks: + task.detach(ScopeExit()) + raise except BaseException: # Interrupt remaining tasks. if scope._spawned: diff --git a/duet/api_test.py b/duet/api_test.py index 84d3f03..61f14f7 100644 --- a/duet/api_test.py +++ b/duet/api_test.py @@ -22,11 +22,12 @@ import time import weakref from collections.abc import Iterator -from typing import List, Tuple +from typing import AsyncGenerator, List, Tuple import pytest import duet +import duet.impl as impl async def mul(a, b): @@ -682,6 +683,70 @@ async def cancel_task(): assert task_future.cancelled() + # Without the fix, closing the generator fails with an unraisable + # "RuntimeError: async generator ignored GeneratorExit", which pytest only + # reports as a warning by default. + @pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") + def test_async_generator_with_scope_closed_when_caller_fails(self) -> None: + """A generator with a scope can be closed after the caller fails mid-iteration.""" + + async def generate() -> AsyncGenerator[int, None]: + async def work(x: int) -> int: + await duet.completed_future(None) + return x * 2 + + async with duet.new_scope() as scope: + async for value in duet.pmap_aiter(scope, work, [1, 2, 3, 4]): + yield value + + async def main() -> None: + async for _ in generate(): + raise Fail() + + with pytest.raises(Fail): + duet.run(main) + + @pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") + @pytest.mark.parametrize("close", ["del", "aclose"]) + def test_async_generator_with_scope_closed_early(self, close: str) -> None: + """Tasks of a scope in a closed generator must not interrupt the caller.""" + log: List[str] = [] + child_started: duet.AwaitableFuture[None] = duet.AwaitableFuture() + + async def child() -> None: + child_started.set_result(None) + try: + await duet.AwaitableFuture() # Never completes. + except BaseException: + log.append("child interrupted") + raise + + async def generate() -> AsyncGenerator[int, None]: + async with duet.new_scope() as scope: + scope.spawn(child) + await child_started + yield 1 + yield 2 + + async def main() -> None: + gen = generate() + async for _ in gen: + break + if close == "del": + del gen # Closes the generator synchronously. + else: + await gen.aclose() + # Otherwise the sleep below would hang because its timeout cannot + # interrupt this task. + assert impl.current_task().interruptible + # The interrupted child fails, maybe on a later tick. That must not + # interrupt this task. + await duet.sleep(0.05) + log.append("main done") + + duet.run(main) + assert log == ["child interrupted", "main done"] + @pytest.mark.skipif( sys.version_info >= (3, 8), reason="inapplicable for python 3.8+ (can be removed)" diff --git a/duet/impl.py b/duet/impl.py index 6a0f3ef..ed18e89 100644 --- a/duet/impl.py +++ b/duet/impl.py @@ -35,6 +35,7 @@ Optional, Set, TypeVar, + Union, ) import duet.futuretools as futuretools @@ -60,12 +61,24 @@ def __init__(self, state: TaskState, expected_state: TaskState) -> None: LOCALS_TASK_SCHEDULER = "__duet_task_scheduler__" +class _Detached: + """Main task of detached tasks; drops their failures (see Task.detach).""" + + def interrupt(self, error: BaseException) -> bool: + # Refuse, like a task that can't be interrupted, so that a scheduler that + # re-posts a refused KeyboardInterrupt still delivers Ctrl-C to another task. + return False + + +_DETACHED = _Detached() + + class Task(Generic[T]): def __init__( self, awaitable: Awaitable[T], scheduler: "Scheduler", main_task: Optional["Task"] ) -> None: self.scheduler = scheduler - self.main_task = main_task + self.main_task: Optional[Union[Task, _Detached]] = main_task self._state = TaskState.WAITING self._future: Optional[Future] = None self._ready_future: futuretools.AwaitableFuture[None] = futuretools.AwaitableFuture() @@ -170,6 +183,16 @@ def interrupt(self, error: BaseException) -> None: if self._future: self._future.cancel() + def detach(self, error: BaseException) -> None: + """Detaches this task from its main task and interrupts it with error. + + The task keeps running until it handles the interrupt, but its failure + will no longer be reported to its former main task. This is used when + the main task cannot wait for this task to finish. + """ + self.main_task = _DETACHED + self.interrupt(error) + def close(self): self._generator.close() self.scheduler = None