Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions duet/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,13 @@ async def finish_tasks():
yield scope
if scope._spawned:
await finish_tasks()
except GeneratorExit:
# An async generator holding this scope open is being closed, maybe
# synchronously (e.g. garbage collected), so we can't await our tasks.
# Interrupt them and detach them from main_task, which has moved on.
for task in tasks:
task.detach(ScopeExit())
raise
except BaseException:
# Interrupt remaining tasks.
if scope._spawned:
Expand Down
67 changes: 66 additions & 1 deletion duet/api_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@
import time
import weakref
from collections.abc import Iterator
from typing import List, Tuple
from typing import AsyncGenerator, List, Tuple

import pytest

import duet
import duet.impl as impl


async def mul(a, b):
Expand Down Expand Up @@ -682,6 +683,70 @@ async def cancel_task():

assert task_future.cancelled()

# Without the fix, closing the generator fails with an unraisable
# "RuntimeError: async generator ignored GeneratorExit", which pytest only
# reports as a warning by default.
@pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning")
def test_async_generator_with_scope_closed_when_caller_fails(self) -> None:
"""A generator with a scope can be closed after the caller fails mid-iteration."""

async def generate() -> AsyncGenerator[int, None]:
async def work(x: int) -> int:
await duet.completed_future(None)
return x * 2

async with duet.new_scope() as scope:
async for value in duet.pmap_aiter(scope, work, [1, 2, 3, 4]):
yield value

async def main() -> None:
async for _ in generate():
raise Fail()

with pytest.raises(Fail):
duet.run(main)

@pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning")
@pytest.mark.parametrize("close", ["del", "aclose"])
def test_async_generator_with_scope_closed_early(self, close: str) -> None:
"""Tasks of a scope in a closed generator must not interrupt the caller."""
log: List[str] = []
child_started: duet.AwaitableFuture[None] = duet.AwaitableFuture()

async def child() -> None:
child_started.set_result(None)
try:
await duet.AwaitableFuture() # Never completes.
except BaseException:
log.append("child interrupted")
raise

async def generate() -> AsyncGenerator[int, None]:
async with duet.new_scope() as scope:
scope.spawn(child)
await child_started
yield 1
yield 2

async def main() -> None:
gen = generate()
async for _ in gen:
break
if close == "del":
del gen # Closes the generator synchronously.
else:
await gen.aclose()
# Otherwise the sleep below would hang because its timeout cannot
# interrupt this task.
assert impl.current_task().interruptible
# The interrupted child fails, maybe on a later tick. That must not
# interrupt this task.
await duet.sleep(0.05)
log.append("main done")

duet.run(main)
assert log == ["child interrupted", "main done"]


@pytest.mark.skipif(
sys.version_info >= (3, 8), reason="inapplicable for python 3.8+ (can be removed)"
Expand Down
25 changes: 24 additions & 1 deletion duet/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
Optional,
Set,
TypeVar,
Union,
)

import duet.futuretools as futuretools
Expand All @@ -60,12 +61,24 @@ def __init__(self, state: TaskState, expected_state: TaskState) -> None:
LOCALS_TASK_SCHEDULER = "__duet_task_scheduler__"


class _Detached:
"""Main task of detached tasks; drops their failures (see Task.detach)."""

def interrupt(self, error: BaseException) -> bool:
# Refuse, like a task that can't be interrupted, so that a scheduler that
# re-posts a refused KeyboardInterrupt still delivers Ctrl-C to another task.
return False


_DETACHED = _Detached()


class Task(Generic[T]):
def __init__(
self, awaitable: Awaitable[T], scheduler: "Scheduler", main_task: Optional["Task"]
) -> None:
self.scheduler = scheduler
self.main_task = main_task
self.main_task: Optional[Union[Task, _Detached]] = main_task
self._state = TaskState.WAITING
self._future: Optional[Future] = None
self._ready_future: futuretools.AwaitableFuture[None] = futuretools.AwaitableFuture()
Expand Down Expand Up @@ -170,6 +183,16 @@ def interrupt(self, error: BaseException) -> None:
if self._future:
self._future.cancel()

def detach(self, error: BaseException) -> None:
"""Detaches this task from its main task and interrupts it with error.

The task keeps running until it handles the interrupt, but its failure
will no longer be reported to its former main task. This is used when
the main task cannot wait for this task to finish.
"""
self.main_task = _DETACHED
self.interrupt(error)

def close(self):
self._generator.close()
self.scheduler = None
Expand Down
Loading