Housekeeping: dependency bumps, lint debt, finished TODOs, dependabot

.github/dependabot.yml

@@ -0,0 +1,41 @@
+# Dependabot: weekly, grouped, one PR per ecosystem. The Substrate module is
+# kept out of the Go group on purpose: its bumps change the API the manifests
+# and the client are generated from and go through the bump procedure, not
+# through an automated PR. integrations/nemo-gym is not listed: its
+# ate-env-client dependency is a path source Dependabot cannot resolve, and
+# nothing in it is pinned.
+version: 2
+updates:
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+    cooldown:
+      default-days: 7 # a release has to be a week old before it is proposed
+    labels: [dependencies]
+    groups:
+      go-deps:
+        patterns: ["*"]
+        exclude-patterns: ["github.com/agent-substrate/substrate"]
+    ignore:
+      - dependency-name: github.com/agent-substrate/substrate
+  - package-ecosystem: pip
+    directory: /clients/python
+    schedule:
+      interval: weekly
+    cooldown:
+      default-days: 7 # a release has to be a week old before it is proposed
+    labels: [dependencies]
+    groups:
+      python-deps:
+        patterns: ["*"]
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    cooldown:
+      default-days: 7 # a release has to be a week old before it is proposed
+    labels: [dependencies]
+    groups:
+      actions:
+        patterns: ["*"]

.github/workflows/ci.yaml

@@ -60,3 +60,4 @@ jobs:
           python-version: "3.12"
       - run: python -m pip install -e 'clients/python[dev]'
       - run: python -m pytest clients/python/tests -q
+      - run: ruff check clients/python

clients/python/pyproject.toml

@@ -24,15 +24,16 @@ readme = "README.md"
 requires-python = ">=3.10"
 license = "Apache-2.0"
 dependencies = [
-    "grpcio>=1.83",
+    "grpcio>=1.84",
     "protobuf>=7.35",
 ]
 
 [project.optional-dependencies]
 dev = [
     "pytest>=8",
     "pytest-asyncio>=1.0",
-    "grpcio-tools>=1.83",
+    "grpcio-tools>=1.84",
+    "ruff==0.16.10",
 ]
 
 [project.urls]
@@ -47,3 +48,12 @@ ate_env = ["py.typed", "_gen/**/*.pyi"]
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 asyncio_mode = "auto"
+
+[tool.ruff]
+target-version = "py310"
+extend-exclude = ["src/ate_env/_gen"]
+
+[tool.ruff.lint]
+# pyflakes, the pycodestyle errors that are not about layout, import order,
+# and pyupgrade for the 3.10 floor. Formatting is not enforced.
+select = ["E4", "E7", "E9", "F", "I", "UP"]

clients/python/src/ate_env/__init__.py

@@ -34,12 +34,10 @@ async def main():
 
 # Layer 1: Core Client & Env
 from .client import DEFAULT_ATESPACE, Client
-from .env import Env, Process
 
 # Layer 2: High-Level Fleet Management & Abstractions
 from .config import FleetConfig
-from .fleet import SandboxFleet
-from .handle import SandboxHandle
+from .env import Env, Process
 
 # Exceptions
 from .errors import (
@@ -61,6 +59,8 @@ async def main():
     map_rpc_error,
 )
 from .exceptions import SandboxError
+from .fleet import SandboxFleet
+from .handle import SandboxHandle
 
 # Types
 from .types import (

clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2_grpc.py

@@ -19,7 +19,7 @@
 
 from . import env_pb2 as ateenv_dot_v1alpha_dot_env__pb2
 
-GRPC_GENERATED_VERSION = '1.81.1'
+GRPC_GENERATED_VERSION = '1.84.0'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 
@@ -40,10 +40,7 @@
 
 
 class EnvironmentServiceStub:
-    """TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService).
-    TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle.
-    TODO(jbd): Update remaining examples and documentation.
-    TODO(jbd): Add ListEnvironments to EnvironmentService.
+    """TODO(jbd): Add ListEnvironments to EnvironmentService.
 
     ============================================================================
     --- SERVICES ---
@@ -81,10 +78,7 @@ def __init__(self, channel):
 
 
 class EnvironmentServiceServicer:
-    """TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService).
-    TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle.
-    TODO(jbd): Update remaining examples and documentation.
-    TODO(jbd): Add ListEnvironments to EnvironmentService.
+    """TODO(jbd): Add ListEnvironments to EnvironmentService.
 
     ============================================================================
     --- SERVICES ---
@@ -153,10 +147,7 @@ def add_EnvironmentServiceServicer_to_server(servicer, server):
 
  # This class is part of an EXPERIMENTAL API.
 class EnvironmentService:
-    """TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService).
-    TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle.
-    TODO(jbd): Update remaining examples and documentation.
-    TODO(jbd): Add ListEnvironments to EnvironmentService.
+    """TODO(jbd): Add ListEnvironments to EnvironmentService.
 
     ============================================================================
     --- SERVICES ---

clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2.py

@@ -16,7 +16,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: ateenv/v1alpha/guest.proto
-# Protobuf Python Version: 6.33.5
+# Protobuf Python Version: 7.35.1
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -25,9 +25,9 @@
 from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
-    6,
-    33,
-    5,
+    7,
+    35,
+    1,
     '',
     'ateenv/v1alpha/guest.proto'
 )

clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2_grpc.py

@@ -19,7 +19,7 @@
 
 from . import guest_pb2 as ateenv_dot_v1alpha_dot_guest__pb2
 
-GRPC_GENERATED_VERSION = '1.81.1'
+GRPC_GENERATED_VERSION = '1.84.0'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 

clients/python/src/ate_env/backend/base.py

@@ -15,7 +15,6 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import Optional
 
 from ..types import EnvironmentSpec, RawSandboxInstance
 

clients/python/src/ate_env/backend/mock.py

@@ -17,7 +17,7 @@
 import collections
 import time
 import uuid
-from typing import Dict, List, Optional, Set
+from typing import Any
 
 from ..errors import SandboxStartError
 from ..types import EnvironmentSpec, RawSandboxInstance
@@ -30,10 +30,10 @@ class MockBackendDriver(BackendDriver):
     def __init__(self, fail_preflight: bool = False, fail_acquire: bool = False):
         self.fail_preflight = fail_preflight
         self.fail_acquire = fail_acquire
-        self.templates: Dict[str, EnvironmentSpec] = {}
-        self.warm_pools: Dict[str, int] = collections.defaultdict(int)
-        self.instances: Dict[str, RawSandboxInstance] = {}
-        self.reaped_runs: Set[str] = set()
+        self.templates: dict[str, EnvironmentSpec] = {}
+        self.warm_pools: dict[str, int] = collections.defaultdict(int)
+        self.instances: dict[str, RawSandboxInstance] = {}
+        self.reaped_runs: set[str] = set()
 
     def preflight(self) -> None:
         if self.fail_preflight:
@@ -79,6 +79,6 @@ def reap(self, run_id: str) -> int:
 
     def build_runtime(
         self, instance: RawSandboxInstance, data_plane: str = "mock"
-    ) -> tuple[Any, Optional[Any]]:
+    ) -> tuple[Any, Any | None]:
         from ..runtime.mock import MockRuntimeHook
         return MockRuntimeHook(), None

clients/python/src/ate_env/backend/substrate.py

@@ -20,10 +20,17 @@
 import threading
 import time
 import uuid
-from typing import Any, Dict, List, Mapping, Optional
+from collections.abc import Mapping
+from typing import Any
 
 from ..client import DEFAULT_ATESPACE, Client
-from ..errors import FailedPreconditionError, InfrastructureError, NotFoundError, PreflightError, SandboxStartError
+from ..errors import (
+    FailedPreconditionError,
+    InfrastructureError,
+    NotFoundError,
+    PreflightError,
+    SandboxStartError,
+)
 from ..loop_runner import LoopRunner
 from ..types import DataPlaneEndpoint, EnvironmentSpec, RawSandboxInstance
 from .base import BackendDriver
@@ -41,18 +48,18 @@ class SubstrateBackendDriver(BackendDriver):
 
     def __init__(
         self,
-        api_endpoint: Optional[str] = None,
+        api_endpoint: str | None = None,
         atespace: str = DEFAULT_ATESPACE,
         worker_family: str = "c2",
-        auth_token: Optional[str] = None,
+        auth_token: str | None = None,
         *,
-        client: Optional[Client] = None,
-        image_templates: Optional[Mapping[str, str]] = None,
-        default_template: Optional[str] = None,
-        labels: Optional[Mapping[str, str]] = None,
+        client: Client | None = None,
+        image_templates: Mapping[str, str] | None = None,
+        default_template: str | None = None,
+        labels: Mapping[str, str] | None = None,
         acquire_timeout_s: float = 180.0,
         ready_timeout_s: float = 180.0,
-        loop_runner: Optional[LoopRunner] = None,
+        loop_runner: LoopRunner | None = None,
     ):
         target = (
             api_endpoint
@@ -81,8 +88,8 @@ def __init__(
             self._owns_client = True
 
         self._lock = threading.Lock()
-        self._owned_actors: Dict[str, RawSandboxInstance] = {}
-        self._warm_paused_pool: Dict[str, List[str]] = {}
+        self._owned_actors: dict[str, RawSandboxInstance] = {}
+        self._warm_paused_pool: dict[str, list[str]] = {}
         self._background_tasks: set[asyncio.Task[Any]] = set()
 
     def _get_loop_runner(self) -> LoopRunner:
@@ -208,7 +215,7 @@ async def _acquire_internal(self, template_id: str, run_id: str) -> RawSandboxIn
         guest traffic, so ready_timeout_s does not measure readiness here.
         """
         while True:
-            actor_id: Optional[str] = None
+            actor_id: str | None = None
             with self._lock:
                 warm_list = self._warm_paused_pool.get(template_id)
                 if warm_list:
@@ -241,7 +248,7 @@ async def _acquire_internal(self, template_id: str, run_id: str) -> RawSandboxIn
                     template_id,
                     self.atespace,
                 )
-                last_err: Optional[Exception] = None
+                last_err: Exception | None = None
                 for attempt in range(4):
                     try:
                         await self.client.create(
@@ -277,7 +284,7 @@ def acquire(
         return self._sync(self.acquire_async(template_id, run_id, timeout_s=timeout_s))
 
     async def warm_pool_async(
-        self, template_id: str, replicas: int, wait: bool = True, run_id: Optional[str] = None
+        self, template_id: str, replicas: int, wait: bool = True, run_id: str | None = None
     ) -> None:
         """Pre-warm a pool of actors by creating and suspending them."""
         logger.info("Sizing warm pool for template %s to %d paused actors...", template_id, replicas)
@@ -290,7 +297,7 @@ async def warm_pool_async(
         sem = asyncio.Semaphore(16)
         effective_run_id = run_id or "warm-pool"
 
-        async def _warm_one() -> Optional[str]:
+        async def _warm_one() -> str | None:
             actor_id = f"warm-{uuid.uuid4().hex[:10]}"
             async with sem:
                 try:
@@ -335,7 +342,7 @@ def _done(t: asyncio.Task[Any]) -> None:
                     )
 
     def warm_pool(
-        self, template_id: str, replicas: int, wait: bool = True, run_id: Optional[str] = None
+        self, template_id: str, replicas: int, wait: bool = True, run_id: str | None = None
     ) -> None:
         self._sync(self.warm_pool_async(template_id, replicas, wait=wait, run_id=run_id))
 
@@ -393,7 +400,7 @@ def release(self, instance_id: str, recycle: bool = False) -> None:
     async def reap_async(self, run_id: str) -> int:
         """Force delete all environments tagged with this run_id and all warm pool actors."""
         with self._lock:
-            warm_actors: List[str] = []
+            warm_actors: list[str] = []
             for tid, actors in list(self._warm_paused_pool.items()):
                 warm_actors.extend(actors)
             self._warm_paused_pool.clear()
@@ -405,7 +412,7 @@ async def reap_async(self, run_id: str) -> int:
             to_reap = list(dict.fromkeys(to_reap + warm_actors))
 
         sem = asyncio.Semaphore(16)
-        failed_actors: List[str] = []
+        failed_actors: list[str] = []
         reaped_count = 0
         reap_lock = asyncio.Lock()
 
@@ -438,7 +445,7 @@ def reap(self, run_id: str) -> int:
 
     def build_runtime(
         self, instance: RawSandboxInstance, data_plane: str = "ate_env"
-    ) -> tuple[Any, Optional[Any]]:
+    ) -> tuple[Any, Any | None]:
         from ..runtime.substrate_env_client import SubstrateEnvClientRuntime
 
         ep = instance.data_planes.get(data_plane) or instance.data_planes.get("ate_env")

clients/python/src/ate_env/client.py

@@ -16,6 +16,8 @@
 
 from __future__ import annotations
 
+import collections
+
 import grpc
 import grpc.aio
 
@@ -37,9 +39,6 @@ def _normalize_target(endpoint: str) -> str:
     return target
 
 
-import collections
-
-
 class _ClientCallDetails(
     collections.namedtuple(
         "_ClientCallDetails",