DRU-611 -- Catch the errors Exec.kill raises in the first-byte killer

backend/druks/sandbox/host.py

@@ -14,7 +14,6 @@
 
 import asyncssh
 from drukbox_sdk import SandboxHost as SandboxHostRecord
-from drukbox_sdk.exceptions import SandboxAPIError, SandboxUnavailableError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from druks.core.models import uuid7_str
@@ -604,7 +603,7 @@ async def _watch_run(
                 if first_byte_task:
                     # Done: its answer. Still sleeping: cancelled, so no kill.
                     first_byte_task.cancel()
-                    with contextlib.suppress(asyncio.CancelledError, Exception):
+                    with contextlib.suppress(asyncio.CancelledError):
                         first_byte_killed = await first_byte_task
                 # The tails end with the exit code; this drains their last bytes.
                 await asyncio.gather(tee_out, tee_err, return_exceptions=True)
@@ -918,16 +917,14 @@ async def _first_byte_killer(
     except asyncio.CancelledError:
         return False
 
-    if len(stdout_buf) > 0 or len(stderr_buf) > 0:
+    if stdout_buf or stderr_buf:
         return False
 
     try:
         await run.kill()
-    except (SandboxAPIError, SandboxUnavailableError):
-        # If the kill RPC fails the overall timeout will eventually surface
-        # as HarnessTimeoutError; the killer's "best-effort" contract is
-        # explicit about that fallback.
-        logger.exception("first-byte killer failed to send kill")
+    except (asyncssh.Error, OSError):
+        # If the run survives, its overall timeout ends it.
+        logger.exception("first-byte killer failed to kill run %s", run.run_id)
         return False
     return True
 

backend/tests/test_sandboxed_harness.py

@@ -8,6 +8,7 @@
 from types import SimpleNamespace
 from typing import Any
 
+import asyncssh
 import pytest
 from conftest import CONFIG_PROBE, connect_provider, installation_key, make_jwt
 from druks.durable.enums import AgentCallStatus
@@ -284,6 +285,22 @@ async def test_first_byte_killer_kills_only_a_silent_run():
     assert stderr_run.kill_calls == 0
 
 
+@pytest.mark.parametrize(
+    "error", [asyncssh.ConnectionLost("channel closed"), ConnectionRefusedError()]
+)
+async def test_first_byte_killer_reports_a_failed_kill(error: Exception, caplog):
+    async def kill() -> None:
+        raise error
+
+    assert not await _first_byte_killer(
+        run=SimpleNamespace(run_id="rid", kill=kill),
+        stdout_buf=bytearray(),
+        stderr_buf=bytearray(),
+        delay_seconds=0,
+    )
+    assert "first-byte killer failed to kill run rid" in caplog.text
+
+
 async def test_sandbox_unreachable_translates_to_harness_error(
     ctx: SimpleNamespace,
 ):