From c7973b844ac1ebefb3bac1e5818f99ae9b224fa0 Mon Sep 17 00:00:00 2001 From: Matthew Neeley Date: Thu, 19 Nov 2026 21:00:23 -0800 Subject: [PATCH] Fix deadline cleanup and handling of multiple expired deadlines Previously, child tasks spawned inside a deadline scope pushed an inherited DeadlineEntry in Task.__init__ that was never invalidated in Task.close(), leaving valid entries for finished tasks in the scheduler's deadline queue. Additionally, when a deadline expired, Scheduler.tick() only popped entries up to that single oldest timestamp and then blocked unconditionally in get_all(None), even if interrupting those entries produced no ready tasks (e.g. for completed or uninterruptible tasks), which stalled later deadlines until external wakeups drained the queue. This addresses those issues along with related deadline edge cases: - Pop and invalidate all remaining deadlines in Task.close(), and move awaitable.__await__() before push_deadline in Task.__init__. - Clear the task reference in DeadlineEntry.invalidate() (deriving valid from _task is not None) so invalidated entries do not pin finished tasks, and lazily compact the scheduler heap when invalid entries exceed half its size. - Loop in Scheduler._get_ready_tasks() to handle all deadlines <= now via handle_expired_deadlines(), continue waiting for the next deadline if no tasks become ready, avoid starving expired deadlines behind already- ready tasks, and handle interrupt signals without prematurely firing future deadlines. --- duet/api_test.py | 119 ++++++++++++++++++++++++++++ duet/impl.py | 138 +++++++++++++++++++-------------- duet/impl_test.py | 192 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+), 59 deletions(-) diff --git a/duet/api_test.py b/duet/api_test.py index 84d3f03..98c1080 100644 --- a/duet/api_test.py +++ b/duet/api_test.py @@ -19,6 +19,7 @@ import gc import inspect import sys +import threading import time import weakref from collections.abc import Iterator @@ -682,6 +683,124 @@ async def cancel_task(): assert task_future.cancelled() + @duet.sync + async def test_completed_deadline_scopes_with_subtasks_do_not_block_later_deadlines( + self, + ) -> None: + """Completed child tasks must not leave stale deadlines that stall later sleeps.""" + + async def fast_subtask() -> None: + await duet.completed_future(None) + + # Create several sequential deadline scopes with distinct timestamps that + # all finish well before their deadlines expire. + for _ in range(10): + async with duet.new_scope(timeout=0.05) as scope: + scope.spawn(fast_subtask) + scope.spawn(fast_subtask) + await duet.sleep(0.005) + + # Sleep past all previous deadlines. Use a background thread watchdog so a + # regression fails fast instead of hanging the test suite. + watchdog_fired = False + stop_watchdog = threading.Event() + done: duet.AwaitableFuture[None] = duet.AwaitableFuture() + + async def sleeper() -> None: + await duet.sleep(0.15) + done.try_set_result(None) + + def watchdog() -> None: + nonlocal watchdog_fired + if not stop_watchdog.wait(2.0) and done.try_set_result(None): + watchdog_fired = True + + thread = threading.Thread(target=watchdog) + thread.start() + try: + try: + async with duet.new_scope() as scope: + scope.spawn(sleeper) + await done + scope.cancel() + except duet.CancelledError: + pass + finally: + stop_watchdog.set() + thread.join() + assert not watchdog_fired + + @duet.sync + async def test_uninterruptible_scope_cleanup_does_not_block_later_deadlines(self) -> None: + """An expired deadline on an uninterruptible task must not stall other deadlines.""" + release: duet.AwaitableFuture[None] = duet.AwaitableFuture() + watchdog_fired = False + stop_watchdog = threading.Event() + + async def bg() -> None: + try: + await duet.AwaitableFuture() + finally: + # Absorb the inner scope's 0.05s timeout, then wait for sibling's sleep. + try: + await release + except TimeoutError: + await release + + async def sibling() -> None: + await duet.sleep(0.15) + release.try_set_result(None) + + def watchdog() -> None: + nonlocal watchdog_fired + if not stop_watchdog.wait(2.0) and release.try_set_result(None): + watchdog_fired = True + + thread = threading.Thread(target=watchdog) + thread.start() + try: + async with duet.new_scope() as outer: + outer.spawn(sibling) + with pytest.raises(Fail): + # Outer 0.08s deadline expires while main_task is uninterruptible + # waiting for bg() to finish, after bg()'s 0.05s deadline already fired. + async with duet.timeout_scope(0.08): + async with duet.new_scope(timeout=0.05) as inner: + inner.spawn(bg) + await duet.completed_future(None) + raise Fail() + finally: + stop_watchdog.set() + thread.join() + assert not watchdog_fired + + @duet.sync + async def test_timeout_not_starved_by_ready_tasks(self) -> None: + start = time.time() + with pytest.raises(TimeoutError): + async with duet.timeout_scope(0.05): + while True: + await duet.completed_future(None) + assert time.time() - start < 1.0 + + @duet.sync + async def test_completed_subtask_not_retained_by_deadline_queue(self) -> None: + payload_refs = [] + + async def child() -> None: + payload = _Payload() + payload_refs.append(weakref.ref(payload)) + await duet.completed_future(payload) + + async with duet.new_scope(timeout=60) as scope: + scope.spawn(child) + # Keep a shorter deadline on the heap so the invalidated entry cannot + # simply be popped from the top of the heap before this check. + await duet.sleep(0.05) + gc.collect() + assert len(payload_refs) == 1 + assert payload_refs[0]() is None + @pytest.mark.skipif( sys.version_info >= (3, 8), reason="inapplicable for python 3.8+ (can be removed)" diff --git a/duet/impl.py b/duet/impl.py index 6a0f3ef..dab9d55 100644 --- a/duet/impl.py +++ b/duet/impl.py @@ -23,19 +23,7 @@ import threading import time from concurrent.futures import Future -from typing import ( - Any, - Awaitable, - Callable, - cast, - Coroutine, - Generic, - Iterator, - List, - Optional, - Set, - TypeVar, -) +from typing import Any, Awaitable, Callable, cast, Coroutine, Generic, List, Optional, Set, TypeVar import duet.futuretools as futuretools @@ -77,12 +65,12 @@ def __init__( self._result: Optional[T] = None self._error: Optional[Exception] = None self._deadlines: List[DeadlineEntry] = [] - if main_task and main_task.deadline_entry is not None: - entry = main_task.deadline_entry - self.push_deadline(deadline=entry.deadline, timeout_error=entry.timeout_error) self._generator = awaitable.__await__() # Returns coroutine generator. if isinstance(awaitable, Coroutine): awaitable.cr_frame.f_locals.setdefault(LOCALS_TASK_SCHEDULER, scheduler) + if main_task and main_task.deadline_entry is not None: + entry = main_task.deadline_entry + self.push_deadline(deadline=entry.deadline, timeout_error=entry.timeout_error) def _check_state(self, expected_state: TaskState) -> None: if self._state != expected_state: @@ -156,7 +144,7 @@ def push_deadline(self, deadline: float, timeout_error: TimeoutError) -> None: def pop_deadline(self) -> None: entry = self._deadlines.pop(-1) - entry.valid = False + self.scheduler.remove_deadline(entry) @property def deadline_entry(self) -> Optional["DeadlineEntry"]: @@ -171,9 +159,13 @@ def interrupt(self, error: BaseException) -> None: self._future.cancel() def close(self): - self._generator.close() - self.scheduler = None - self.main_task = None + try: + self._generator.close() + finally: + while self._deadlines: + self.pop_deadline() + self.scheduler = None + self.main_task = None _current_task: contextvars.ContextVar[Task] = contextvars.ContextVar("current_task") @@ -270,32 +262,40 @@ def interrupt(self) -> None: @functools.total_ordering class DeadlineEntry: - """A entry for one Deadline in the Scheduler's priority queue. + """An entry for one Deadline in the Scheduler's priority queue. This follows the implementation notes in the stdlib heapq docs: https://docs.python.org/3/library/heapq.html#priority-queue-implementation-notes + If a task exits its scope before its deadline elapses, we invalidate the + entry (clearing its task reference) rather than removing it directly from + the priority queue, which would require an O(n) scan. The scheduler skips + invalid entries when they reach the head of the queue. + Attributes: - task: The task associated with this deadline. deadline: Absolute time when the deadline will elapse. + timeout_error: Error used to interrupt the task if the deadline elapses. count: Monotonically-increasing counter to preserve creation order when comparing entries with the same deadline. - valid: Flag indicating whether the deadline is still valid. If the task - exits its scope before the deadline elapses, we mark the deadline as - invalid but leave it in the scheduler's priority queue since removal - would require an O(n) scan. The scheduler ignores invalid deadlines - when they elapse. """ _counter = itertools.count() def __init__(self, task: Task, deadline: float, timeout_error: TimeoutError) -> None: - self.task = task + self._task: Optional[Task] = task self.deadline = deadline self.timeout_error = timeout_error self.count = next(self._counter) self._cmp_val = (deadline, self.count) - self.valid = True + + @property + def valid(self) -> bool: + return self._task is not None + + def invalidate(self) -> Optional[Task]: + task = self._task + self._task = None + return task def __eq__(self, other: Any) -> bool: if not isinstance(other, DeadlineEntry): @@ -308,7 +308,7 @@ def __lt__(self, other: Any) -> bool: return self._cmp_val < other._cmp_val def __repr__(self) -> str: - return f"DeadlineEntry({self.task}, {self.deadline}, {self.count})" + return f"DeadlineEntry({self._task}, {self.deadline}, {self.count})" class Scheduler: @@ -318,6 +318,7 @@ def __init__(self) -> None: self._prev_signal: Optional[Callable] = None self._interrupted = False self._deadlines: List[DeadlineEntry] = [] + self._valid_deadlines = 0 def spawn(self, awaitable: Awaitable[Any], main_task: Optional[Task] = None) -> Task: """Spawns a new Task to run an awaitable in this Scheduler. @@ -344,6 +345,20 @@ def time(self) -> float: def add_deadline(self, entry: DeadlineEntry) -> None: heapq.heappush(self._deadlines, entry) + self._valid_deadlines += 1 + + def remove_deadline(self, entry: DeadlineEntry) -> None: + if entry.invalidate() is not None: + self._valid_deadlines -= 1 + # Invalidated entries are normally popped when they reach the head + # of the heap, but can accumulate behind an earlier active deadline. + # Clear the heap once empty, or rebuild it when over half invalid to + # keep space O(valid) with amortized O(1) removal. + if self._valid_deadlines == 0: + self._deadlines.clear() + elif len(self._deadlines) > 64 and len(self._deadlines) > 2 * self._valid_deadlines: + self._deadlines = [e for e in self._deadlines if e.valid] + heapq.heapify(self._deadlines) def get_next_deadline(self) -> Optional[float]: while self._deadlines: @@ -353,11 +368,40 @@ def get_next_deadline(self) -> Optional[float]: return self._deadlines[0].deadline return None - def get_deadline_entries(self, deadline: float) -> Iterator[DeadlineEntry]: - while self._deadlines and self._deadlines[0].deadline <= deadline: + def handle_expired_deadlines(self, now: float) -> None: + while self._deadlines and self._deadlines[0].deadline <= now: entry = heapq.heappop(self._deadlines) - if entry.valid: - yield entry + task = entry.invalidate() + if task is not None: + self._valid_deadlines -= 1 + task.interrupt(entry.timeout_error) + + def _get_ready_tasks(self) -> List[Task]: + while True: + if self._interrupted: + task = next(iter(self.active_tasks)) + task.interrupt(KeyboardInterrupt()) + self._interrupted = False + + deadline = self.get_next_deadline() + if deadline is None: + ready_tasks = self._ready_tasks.get_all(None) + if ready_tasks: + return ready_tasks + continue + + now = self.time() + if deadline <= now: + self.handle_expired_deadlines(now) + continue + + timeout = min(deadline - now, threading.TIMEOUT_MAX) + try: + ready_tasks = self._ready_tasks.get_all(timeout) + except TimeoutError: + continue + if ready_tasks: + return ready_tasks def tick(self) -> None: """Runs the scheduler ahead by one tick. @@ -370,31 +414,7 @@ def tick(self) -> None: if not self.active_tasks: raise RuntimeError("tick called with no active tasks") - if self._interrupted: - task = next(iter(self.active_tasks)) - task.interrupt(KeyboardInterrupt()) - self._interrupted = False - - deadline = self.get_next_deadline() - if deadline is None: - ready_tasks = self._ready_tasks.get_all(None) - else: - ready_tasks = [] - for i in itertools.count(): - timeout = deadline - self.time() - if i and timeout < 0: - break - try: - ready_tasks = self._ready_tasks.get_all( - max(0, min(timeout, threading.TIMEOUT_MAX)) - ) - break - except TimeoutError: - pass - if not ready_tasks: - for entry in self.get_deadline_entries(deadline): - entry.task.interrupt(entry.timeout_error) - ready_tasks = self._ready_tasks.get_all(None) + ready_tasks = self._get_ready_tasks() for task in ready_tasks: try: task.advance() diff --git a/duet/impl_test.py b/duet/impl_test.py index 1795231..f19b1c9 100644 --- a/duet/impl_test.py +++ b/duet/impl_test.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading +import time + +import pytest + import duet import duet.impl as impl @@ -78,3 +83,190 @@ def test_futures_not_flushed_if_tasks_ready(self): tasks = rs.get_all() assert tasks == [task2] assert not future.flushed + + +class FakeClockScheduler(impl.Scheduler): + """Scheduler with a manually controllable clock for deterministic testing.""" + + def __init__(self, now: float = 0.0) -> None: + super().__init__() + self.now = now + + def time(self) -> float: + return self.now + + +class TestSchedulerDeadlines: + def test_child_task_close_invalidates_inherited_deadline(self) -> None: + scheduler = FakeClockScheduler(now=100.0) + + async def child() -> None: + await duet.completed_future(None) + + async def main() -> None: + async with duet.new_scope(timeout=10.0) as scope: + scope.spawn(child) + scope.spawn(child) + + scheduler.spawn(main()) + while scheduler.active_tasks: + scheduler.tick() + + assert scheduler.get_next_deadline() is None + assert not scheduler._deadlines + assert scheduler._valid_deadlines == 0 + + def test_multiple_distinct_expired_deadlines_fire_in_single_tick(self) -> None: + scheduler = FakeClockScheduler(now=100.0) + timed_out = [] + + async def waiter(name: str, deadline: float) -> None: + try: + async with duet.deadline_scope(deadline): + await duet.AwaitableFuture() + except TimeoutError: + timed_out.append(name) + + scheduler.spawn(waiter("a", 101.0)) + scheduler.spawn(waiter("b", 102.0)) + scheduler.spawn(waiter("c", 110.0)) + + # Advance initial step so all tasks register their deadlines and wait. + scheduler.tick() + assert timed_out == [] + + # Jump past the first two distinct deadlines (101.0 and 102.0). + scheduler.now = 105.0 + scheduler.tick() + assert timed_out == ["a", "b"] + + # Jump past the third deadline. + scheduler.now = 115.0 + scheduler.tick() + assert timed_out == ["a", "b", "c"] + assert not scheduler.active_tasks + + def test_uninterruptible_expired_deadlines_do_not_block_later_deadline(self) -> None: + scheduler = FakeClockScheduler(now=100.0) + timed_out = [] + futures = [] + + async def waiter(name: str, deadline: float) -> None: + f: duet.AwaitableFuture[None] = duet.AwaitableFuture() + futures.append(f) + try: + async with duet.deadline_scope(deadline): + await f + except TimeoutError: + timed_out.append(name) + + t1 = scheduler.spawn(waiter("uninterruptible_1", 101.0)) + t2 = scheduler.spawn(waiter("uninterruptible_2", 102.0)) + t3 = scheduler.spawn(waiter("interruptible", 105.0)) + + scheduler.tick() + t1.interruptible = False + t2.interruptible = False + + # Jump past all three deadlines: t1 and t2 are no-ops, t3 must still wake. + scheduler.now = 106.0 + scheduler.tick() + assert timed_out == ["interruptible"] + assert t3.done + + # Clean up remaining uninterruptible tasks. + for f in futures: + f.try_set_result(None) + while scheduler.active_tasks: + scheduler.tick() + assert not scheduler._deadlines + assert scheduler._valid_deadlines == 0 + + def test_uninterruptible_expired_deadline_continues_waiting_for_future_deadline(self) -> None: + scheduler = impl.Scheduler() + timed_out = [] + futures = [] + + async def waiter(name: str, timeout: float) -> None: + f: duet.AwaitableFuture[None] = duet.AwaitableFuture() + futures.append(f) + try: + async with duet.timeout_scope(timeout): + await f + except TimeoutError: + timed_out.append(name) + + t1 = scheduler.spawn(waiter("uninterruptible_1", 0.02)) + t2 = scheduler.spawn(waiter("uninterruptible_2", 0.04)) + t3 = scheduler.spawn(waiter("interruptible", 0.08)) + + scheduler.tick() + t1.interruptible = False + t2.interruptible = False + + # One tick should drain t1 and t2 as they expire, then keep waiting until t3 expires. + start = time.time() + scheduler.tick() + elapsed = time.time() - start + assert timed_out == ["interruptible"] + assert t3.done + assert elapsed < 1.0 + + # Clean up remaining tasks. + for f in futures: + f.try_set_result(None) + while scheduler.active_tasks: + scheduler.tick() + assert not scheduler._deadlines + assert scheduler._valid_deadlines == 0 + + def test_invalidated_deadlines_are_compacted(self) -> None: + scheduler = FakeClockScheduler(now=100.0) + keep_open: duet.AwaitableFuture[None] = duet.AwaitableFuture() + + async def long_waiter() -> None: + async with duet.deadline_scope(200.0): + await keep_open + + async def churn() -> None: + for i in range(500): + # Deadlines later than 200.0 sit below long_waiter in the min-heap. + async with duet.deadline_scope(300.0 + i): + await duet.completed_future(None) + # Heap should have been compacted rather than holding all 500 invalid entries. + assert len(scheduler._deadlines) <= 64 + assert scheduler._valid_deadlines == 1 + keep_open.set_result(None) + + scheduler.spawn(long_waiter()) + scheduler.spawn(churn()) + while scheduler.active_tasks: + scheduler.tick() + assert not scheduler._deadlines + assert scheduler._valid_deadlines == 0 + + def test_interrupt_signal_while_waiting_on_deadline_does_not_timeout(self) -> None: + scheduler = impl.Scheduler() + + async def waiter() -> None: + async with duet.timeout_scope(10.0): + await duet.AwaitableFuture() + + task = scheduler.spawn(waiter()) + scheduler.tick() # Start task and register 10s deadline. + + def send_interrupt() -> None: + time.sleep(0.05) + scheduler._interrupted = True + scheduler._ready_tasks.interrupt() + + thread = threading.Thread(target=send_interrupt) + thread.start() + try: + with pytest.raises(KeyboardInterrupt): + scheduler.tick() + finally: + thread.join() + + assert task.done + assert not scheduler._deadlines