Bound download subprocess waits and retry transient failures on POST and ad-hoc fetches

gamdl/api/apple_music.py

@@ -194,6 +194,12 @@ async def create(
                     total=6,
                     backoff_factor=1,
                     status_forcelist=[429, 500, 502, 503, 504],
+                    # The only POSTs on this client are the rewindable JSON
+                    # webplayback and license-exchange fetches, which hard-fail
+                    # tracks on 429/5xx without this.
+                    allowed_methods=frozenset(
+                        {"GET", "HEAD", "PUT", "DELETE", "OPTIONS", "TRACE", "POST"}
+                    ),
                 )
             ),
         )

gamdl/downloader/base.py

@@ -15,7 +15,11 @@
 from ..interface.enums import CoverFormat
 from ..interface.interface import AppleMusicInterface
 from ..interface.types import MediaTags, PlaylistTags
-from ..utils import CustomStringFormatter, async_subprocess
+from ..utils import (
+    DOWNLOAD_TIMEOUT_SECONDS,
+    CustomStringFormatter,
+    async_subprocess,
+)
 from .constants import ILLEGAL_CHAR_REPLACEMENT, ILLEGAL_CHARS_RE, TEMP_PATH_TEMPLATE
 from .enums import DownloadMode
 
@@ -278,8 +282,13 @@ async def _download_ytdlp_async(
         )
         process.start()
 
+        deadline = asyncio.get_running_loop().time() + DOWNLOAD_TIMEOUT_SECONDS
         try:
             while process.is_alive():
+                if asyncio.get_running_loop().time() > deadline:
+                    raise RuntimeError(
+                        f"yt-dlp timed out after {DOWNLOAD_TIMEOUT_SECONDS} seconds"
+                    )
                 await asyncio.sleep(0.1)
 
             process.join()

gamdl/interface/base.py

@@ -7,6 +7,7 @@
 import httpx
 import structlog
 from async_lru import alru_cache
+from httpx_retries import Retry, RetryTransport
 from PIL import Image
 from pywidevine import PSSH, Cdm, Device
 from pywidevine.license_protocol_pb2 import WidevinePsshData
@@ -22,6 +23,14 @@
 
 logger = structlog.get_logger(__name__)
 
+# Manifest and cover fetches build ad-hoc clients; without a retry policy a
+# transient 429/5xx fails the whole track.
+_HTTP_RETRY = Retry(
+    total=3,
+    backoff_factor=1,
+    status_forcelist=[429, 500, 502, 503, 504],
+)
+
 
 class AppleMusicBaseInterface:
     def __init__(
@@ -90,7 +99,10 @@ async def get_response(
         url: str,
         valid_responses: list[int] = [200],
     ) -> httpx.Response:
-        async with httpx.AsyncClient(timeout=60.0) as client:
+        async with httpx.AsyncClient(
+            timeout=60.0,
+            transport=RetryTransport(retry=_HTTP_RETRY),
+        ) as client:
             try:
                 response = await client.get(url)
                 response.raise_for_status()
@@ -205,7 +217,10 @@ async def get_decryption_key(
     async def get_cover_bytes(self, cover_url: str) -> bytes | None:
         log = logger.bind(action="get_cover_bytes", cover_url=cover_url)
 
-        async with httpx.AsyncClient(timeout=30.0) as client:
+        async with httpx.AsyncClient(
+            timeout=30.0,
+            transport=RetryTransport(retry=_HTTP_RETRY),
+        ) as client:
             response = await client.get(cover_url, follow_redirects=True)
 
             if response.status_code == 404:

gamdl/utils.py

@@ -2,8 +2,17 @@
 import string
 import typing
 
+# Whole-track cap for external download processes so a wedged child cannot
+# hang the run forever. Generous on purpose: it bounds total transfer time,
+# including slow links and long lossless tracks, not just stalls.
+DOWNLOAD_TIMEOUT_SECONDS = 1800
 
-async def async_subprocess(*args: str, silent: bool = False) -> None:
+
+async def async_subprocess(
+    *args: str,
+    silent: bool = False,
+    timeout: float | None = DOWNLOAD_TIMEOUT_SECONDS,
+) -> None:
     if silent:
         additional_args = {
             "stdout": asyncio.subprocess.PIPE,
@@ -17,7 +26,14 @@ async def async_subprocess(*args: str, silent: bool = False) -> None:
         **additional_args,
     )
 
-    stdout, stderr = await proc.communicate()
+    try:
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+    except asyncio.TimeoutError:
+        proc.kill()
+        await proc.communicate()
+        raise RuntimeError(
+            f"Timed out after {timeout} seconds: {' '.join(str(arg) for arg in args)}"
+        ) from None
 
     if proc.returncode != 0:
         msg = (