From b69a10b131a4f4ddd2aa6f3f03ebfd921304791a Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:15:58 -0700 Subject: [PATCH 1/4] go.mod: mcp go-sdk 1.8.0, grpc 1.84.0, k8s.io 0.37.1 Build, vet and the full suite pass on the new versions; no code changes were needed. --- go.mod | 12 +++++++----- go.sum | 38 ++++++++++++-------------------------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index fba5ba9..744476b 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,19 @@ go 1.27.0 require ( github.com/agent-substrate/substrate v0.4.1-0.20261013175625-da35a983ed34 github.com/google/uuid v1.6.0 - github.com/modelcontextprotocol/go-sdk v1.4.1 + github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/spf13/cobra v1.10.2 - google.golang.org/grpc v1.83.2 + google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 - k8s.io/api v0.37.0 - k8s.io/apimachinery v0.37.0 + k8s.io/api v0.37.1 + k8s.io/apimachinery v0.37.1 sigs.k8s.io/yaml v1.6.0 ) require ( github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/go-logr/logr v1.4.4 // indirect - github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -30,8 +30,10 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/net v0.60.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect diff --git a/go.sum b/go.sum index 974facc..11e7bca 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,6 @@ github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+r github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= @@ -51,8 +49,8 @@ github.com/go-openapi/swag/yamlutils v0.28.0 h1:TV3JXH6DS46KUroDtMLAYHGkdWf5VDq3 github.com/go-openapi/swag/yamlutils v0.28.0/go.mod h1:x0q/yndZHEgk9Rx3DyDqzFUmHy55KTvIZldvF2dTJXs= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= @@ -60,8 +58,8 @@ github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7O github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -70,8 +68,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/modelcontextprotocol/go-sdk v1.4.1 h1:M4x9GyIPj+HoIlHNGpK2hq5o3BFhC+78PkEaldQRphc= -github.com/modelcontextprotocol/go-sdk v1.4.1/go.mod h1:Bo/mS87hPQqHSRkMv4dQq1XCu6zv4INdXnFZabkNU6s= +github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= +github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -113,18 +111,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= @@ -152,8 +138,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -161,12 +147,12 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnf gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4= -k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk= +k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM= +k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M= k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= -k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0= -k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE= +k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE= +k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw= k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= From a44feca09a6fd010aaef61c12b7488ea94ee3103 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:17:19 -0700 Subject: [PATCH 2/4] lint: staticcheck ST1005 in the CLI messages, ruff for the Python client - The three multi-line CLI error strings in manifest.go ended with a period, which staticcheck flags (ST1005); dropped. - clients/python gets a ruff configuration (pyflakes, non-layout pycodestyle, import order, pyupgrade for the 3.10 floor; generated stubs excluded), the one finding it produces is fixed, ruff is in the dev extra, and CI runs `ruff check clients/python` after the tests. --- .github/workflows/ci.yaml | 1 + clients/python/pyproject.toml | 10 ++++++++++ clients/python/tests/test_client.py | 8 +++++++- cmd/ate-env/manifest.go | 6 +++--- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 67dfcf5..9091916 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -60,3 +60,4 @@ jobs: python-version: "3.12" - run: python -m pip install -e 'clients/python[dev]' - run: python -m pytest clients/python/tests -q + - run: ruff check clients/python diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index f85ef51..2dedbfb 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -33,6 +33,7 @@ dev = [ "pytest>=8", "pytest-asyncio>=1.0", "grpcio-tools>=1.83", + "ruff==0.16.10", ] [project.urls] @@ -47,3 +48,12 @@ ate_env = ["py.typed", "_gen/**/*.pyi"] [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" + +[tool.ruff] +target-version = "py310" +extend-exclude = ["src/ate_env/_gen"] + +[tool.ruff.lint] +# pyflakes, the pycodestyle errors that are not about layout, import order, +# and pyupgrade for the 3.10 floor. Formatting is not enforced. +select = ["E4", "E7", "E9", "F", "I", "UP"] diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index b9abd4a..780eac6 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -17,7 +17,13 @@ import grpc.aio import pytest -from ate_env import Client, EnvironmentStatus, InvalidArgumentError, NotFoundError, RpcError +from ate_env import ( + Client, + EnvironmentStatus, + InvalidArgumentError, + NotFoundError, + RpcError, +) from ate_env._gen.ateenv.v1alpha import env_pb2 from ate_env.client import _normalize_target diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index 2238a37..39ca70e 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -62,12 +62,12 @@ deployed on your cluster. From a Substrate checkout at that version run "go run ./cmd/ate-setup publish worker-images": it pushes the images and prints the digest-pinned refs to use. Pin by digest: a registry's :latest is whatever was built last, and a stale worker image registers workers -with no capacity.`) +with no capacity`) } if c.apiImage == "" { return errors.New(`--api-image is required when ate-env is built from source; release binaries default it to the ate-env-api image published with the release -(see docs/release.md), or build and push your own with make images.`) +(see docs/release.md), or build and push your own with make images`) } return nil } @@ -143,7 +143,7 @@ func (c *templateConfig) resolveImages() error { if c.guestImage == "" { return errors.New(`--guest-image is required when ate-env is built from source; release binaries default it to the ate-env-guest image published with the release -(see docs/release.md), or build and push your own with make images.`) +(see docs/release.md), or build and push your own with make images`) } if c.snapshotsBucket == "" { return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") From 6b382b0744a82b47d364e78bb1e1ee2e060cb190 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:17:19 -0700 Subject: [PATCH 3/4] chore: drop finished TODOs, rootfs-image wording, grpcio floor 1.84, dependabot - env.proto: three of the four TODO(jbd) notes describe work that is done (the data plane lives in guest.proto, the REST lifecycle endpoints are gone, docs were updated); only ListEnvironments remains. client.go: the two atespace TODOs are done, atespace is a parameter everywhere. - Python stubs regenerated for the proto comment change. The generator is grpcio-tools 1.84, which stamps GRPC_GENERATED_VERSION 1.84.0 and makes the import-time check require grpcio >= 1.84; the client's and the nemo-gym integration's floors move from 1.83 to 1.84 to match. - docs/release.md says rootfs-image templates, the name since #70. - .github/dependabot.yml: weekly grouped updates for the Go module, the two Python projects and the workflow actions. The Substrate module is excluded: its bumps change the generated API and go through the bump procedure, not an automated PR. --- .github/dependabot.yml | 41 +++++++++++++++++++ clients/python/pyproject.toml | 4 +- .../_gen/ateenv/v1alpha/env_pb2_grpc.py | 17 ++------ .../ate_env/_gen/ateenv/v1alpha/guest_pb2.py | 8 ++-- .../_gen/ateenv/v1alpha/guest_pb2_grpc.py | 2 +- docs/release.md | 2 +- integrations/nemo-gym/pyproject.toml | 2 +- internal/ate/client.go | 3 -- proto/ateenv/v1alpha/env.proto | 3 -- 9 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..64038a9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,41 @@ +# Dependabot: weekly, grouped, one PR per ecosystem. The Substrate module is +# kept out of the Go group on purpose: its bumps change the API the manifests +# and the client are generated from and go through the bump procedure, not +# through an automated PR. integrations/nemo-gym is not listed: its +# ate-env-client dependency is a path source Dependabot cannot resolve, and +# nothing in it is pinned. +version: 2 +updates: + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 # a release has to be a week old before it is proposed + labels: [dependencies] + groups: + go-deps: + patterns: ["*"] + exclude-patterns: ["github.com/agent-substrate/substrate"] + ignore: + - dependency-name: github.com/agent-substrate/substrate + - package-ecosystem: pip + directory: /clients/python + schedule: + interval: weekly + cooldown: + default-days: 7 # a release has to be a week old before it is proposed + labels: [dependencies] + groups: + python-deps: + patterns: ["*"] + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 # a release has to be a week old before it is proposed + labels: [dependencies] + groups: + actions: + patterns: ["*"] diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 2dedbfb..5079717 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -24,7 +24,7 @@ readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" dependencies = [ - "grpcio>=1.83", + "grpcio>=1.84", "protobuf>=7.35", ] @@ -32,7 +32,7 @@ dependencies = [ dev = [ "pytest>=8", "pytest-asyncio>=1.0", - "grpcio-tools>=1.83", + "grpcio-tools>=1.84", "ruff==0.16.10", ] diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2_grpc.py b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2_grpc.py index 0723df8..32fdbcf 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2_grpc.py +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2_grpc.py @@ -19,7 +19,7 @@ from . import env_pb2 as ateenv_dot_v1alpha_dot_env__pb2 -GRPC_GENERATED_VERSION = '1.81.1' +GRPC_GENERATED_VERSION = '1.84.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False @@ -40,10 +40,7 @@ class EnvironmentServiceStub: - """TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService). - TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle. - TODO(jbd): Update remaining examples and documentation. - TODO(jbd): Add ListEnvironments to EnvironmentService. + """TODO(jbd): Add ListEnvironments to EnvironmentService. ============================================================================ --- SERVICES --- @@ -81,10 +78,7 @@ def __init__(self, channel): class EnvironmentServiceServicer: - """TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService). - TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle. - TODO(jbd): Update remaining examples and documentation. - TODO(jbd): Add ListEnvironments to EnvironmentService. + """TODO(jbd): Add ListEnvironments to EnvironmentService. ============================================================================ --- SERVICES --- @@ -153,10 +147,7 @@ def add_EnvironmentServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class EnvironmentService: - """TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService). - TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle. - TODO(jbd): Update remaining examples and documentation. - TODO(jbd): Add ListEnvironments to EnvironmentService. + """TODO(jbd): Add ListEnvironments to EnvironmentService. ============================================================================ --- SERVICES --- diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2.py b/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2.py index 07853ff..c4030dc 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2.py +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2.py @@ -16,7 +16,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: ateenv/v1alpha/guest.proto -# Protobuf Python Version: 6.33.5 +# Protobuf Python Version: 7.35.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -25,9 +25,9 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 6, - 33, - 5, + 7, + 35, + 1, '', 'ateenv/v1alpha/guest.proto' ) diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2_grpc.py b/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2_grpc.py index fbf0b04..93c6a30 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2_grpc.py +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/guest_pb2_grpc.py @@ -19,7 +19,7 @@ from . import guest_pb2 as ateenv_dot_v1alpha_dot_guest__pb2 -GRPC_GENERATED_VERSION = '1.81.1' +GRPC_GENERATED_VERSION = '1.84.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False diff --git a/docs/release.md b/docs/release.md index 60e6a7e..15226ff 100644 --- a/docs/release.md +++ b/docs/release.md @@ -83,7 +83,7 @@ workflow's `GITHUB_TOKEN`, under the repository. Once: - The guest's base image is pinned by digest in [`.ko.yaml`](../.ko.yaml). It is a `bash` image so that templates whose container is the guest itself have - a shell for `shell()`; for task-image templates only the static binary is + a shell for `shell()`; for rootfs-image templates only the static binary is used. Bump the pin deliberately and mention it in the release notes. - The API image uses ko's default distroless base. - The actions both workflows use are pinned by commit SHA, with the version in diff --git a/integrations/nemo-gym/pyproject.toml b/integrations/nemo-gym/pyproject.toml index 5e0f171..3a948f3 100644 --- a/integrations/nemo-gym/pyproject.toml +++ b/integrations/nemo-gym/pyproject.toml @@ -25,7 +25,7 @@ license = { text = "Apache-2.0" } requires-python = ">=3.10" # ate-env-client is this repo's Python client (clients/python); it is not on # PyPI yet, so install it from the checkout first: pip install ./clients/python -dependencies = ["ate-env-client>=0.1.0", "grpcio>=1.83"] +dependencies = ["ate-env-client>=0.1.0", "grpcio>=1.84"] [project.optional-dependencies] # The real deployment target. Without it the package falls back to local diff --git a/internal/ate/client.go b/internal/ate/client.go index 960fad4..75af159 100644 --- a/internal/ate/client.go +++ b/internal/ate/client.go @@ -362,6 +362,3 @@ func wrapGRPCError(err error) error { } return err } - -// TODO(jbd): Allow setting a non-default atespace. -// TODO(jbd): Allow setting default atespace. diff --git a/proto/ateenv/v1alpha/env.proto b/proto/ateenv/v1alpha/env.proto index 3ad8b16..034b684 100644 --- a/proto/ateenv/v1alpha/env.proto +++ b/proto/ateenv/v1alpha/env.proto @@ -24,9 +24,6 @@ package ateenv.v1alpha; option go_package = "github.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alpha"; -// TODO(jbd): Migrate data plane (shell/fs) to guest.proto (ProcessService/FileSystemService). -// TODO(jbd): Remove env legacy HTTP REST endpoints for environment lifecycle. -// TODO(jbd): Update remaining examples and documentation. // TODO(jbd): Add ListEnvironments to EnvironmentService. // ============================================================================ From a384a3b55137319f105073d8e3aa445390949756 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:00:10 -0700 Subject: [PATCH 4/4] lint: ruff fixes for the Python SDK merged in #80 The fleet, driver, runtime, strategies, config and their tests arrived after the ruff gate in this PR was written and carry 198 findings under its configuration. All mechanical: PEP 585/604 annotations (list[str], X | None) and typing imports moved to collections.abc, which the 3.10 floor and the future import already allow; unused imports and import order in tests; quoted annotations. Two by hand: backend/mock.py used Any without importing it, and client.py had `import collections` below a function. `uv run --extra dev pytest`: 118 passed. `ruff check clients/python`: clean. --- clients/python/src/ate_env/__init__.py | 6 +-- clients/python/src/ate_env/backend/base.py | 1 - clients/python/src/ate_env/backend/mock.py | 12 ++--- .../python/src/ate_env/backend/substrate.py | 45 +++++++++++-------- clients/python/src/ate_env/client.py | 5 +-- clients/python/src/ate_env/config.py | 26 +++++------ clients/python/src/ate_env/fleet.py | 45 +++++++++---------- clients/python/src/ate_env/handle.py | 17 ++++--- clients/python/src/ate_env/loop_runner.py | 3 +- clients/python/src/ate_env/runtime/base.py | 18 ++++---- clients/python/src/ate_env/runtime/mock.py | 19 ++++---- .../ate_env/runtime/substrate_env_client.py | 30 ++++++------- clients/python/src/ate_env/strategies.py | 28 ++++++------ clients/python/src/ate_env/types.py | 26 +++++------ clients/python/tests/test_data_planes.py | 4 +- clients/python/tests/test_exec_semantics.py | 2 - clients/python/tests/test_fleet_strategies.py | 1 + clients/python/tests/test_fleet_types.py | 3 +- clients/python/tests/test_substrate_driver.py | 1 + .../python/tests/test_substrate_env_client.py | 4 +- 20 files changed, 148 insertions(+), 148 deletions(-) diff --git a/clients/python/src/ate_env/__init__.py b/clients/python/src/ate_env/__init__.py index 0d8e056..1e3f643 100644 --- a/clients/python/src/ate_env/__init__.py +++ b/clients/python/src/ate_env/__init__.py @@ -34,12 +34,10 @@ async def main(): # Layer 1: Core Client & Env from .client import DEFAULT_ATESPACE, Client -from .env import Env, Process # Layer 2: High-Level Fleet Management & Abstractions from .config import FleetConfig -from .fleet import SandboxFleet -from .handle import SandboxHandle +from .env import Env, Process # Exceptions from .errors import ( @@ -61,6 +59,8 @@ async def main(): map_rpc_error, ) from .exceptions import SandboxError +from .fleet import SandboxFleet +from .handle import SandboxHandle # Types from .types import ( diff --git a/clients/python/src/ate_env/backend/base.py b/clients/python/src/ate_env/backend/base.py index 42e667c..2f9f5ef 100644 --- a/clients/python/src/ate_env/backend/base.py +++ b/clients/python/src/ate_env/backend/base.py @@ -15,7 +15,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Optional from ..types import EnvironmentSpec, RawSandboxInstance diff --git a/clients/python/src/ate_env/backend/mock.py b/clients/python/src/ate_env/backend/mock.py index 978fca0..23e06d9 100644 --- a/clients/python/src/ate_env/backend/mock.py +++ b/clients/python/src/ate_env/backend/mock.py @@ -17,7 +17,7 @@ import collections import time import uuid -from typing import Dict, List, Optional, Set +from typing import Any from ..errors import SandboxStartError from ..types import EnvironmentSpec, RawSandboxInstance @@ -30,10 +30,10 @@ class MockBackendDriver(BackendDriver): def __init__(self, fail_preflight: bool = False, fail_acquire: bool = False): self.fail_preflight = fail_preflight self.fail_acquire = fail_acquire - self.templates: Dict[str, EnvironmentSpec] = {} - self.warm_pools: Dict[str, int] = collections.defaultdict(int) - self.instances: Dict[str, RawSandboxInstance] = {} - self.reaped_runs: Set[str] = set() + self.templates: dict[str, EnvironmentSpec] = {} + self.warm_pools: dict[str, int] = collections.defaultdict(int) + self.instances: dict[str, RawSandboxInstance] = {} + self.reaped_runs: set[str] = set() def preflight(self) -> None: if self.fail_preflight: @@ -79,6 +79,6 @@ def reap(self, run_id: str) -> int: def build_runtime( self, instance: RawSandboxInstance, data_plane: str = "mock" - ) -> tuple[Any, Optional[Any]]: + ) -> tuple[Any, Any | None]: from ..runtime.mock import MockRuntimeHook return MockRuntimeHook(), None diff --git a/clients/python/src/ate_env/backend/substrate.py b/clients/python/src/ate_env/backend/substrate.py index 6e2e76a..52ff54c 100644 --- a/clients/python/src/ate_env/backend/substrate.py +++ b/clients/python/src/ate_env/backend/substrate.py @@ -20,10 +20,17 @@ import threading import time import uuid -from typing import Any, Dict, List, Mapping, Optional +from collections.abc import Mapping +from typing import Any from ..client import DEFAULT_ATESPACE, Client -from ..errors import FailedPreconditionError, InfrastructureError, NotFoundError, PreflightError, SandboxStartError +from ..errors import ( + FailedPreconditionError, + InfrastructureError, + NotFoundError, + PreflightError, + SandboxStartError, +) from ..loop_runner import LoopRunner from ..types import DataPlaneEndpoint, EnvironmentSpec, RawSandboxInstance from .base import BackendDriver @@ -41,18 +48,18 @@ class SubstrateBackendDriver(BackendDriver): def __init__( self, - api_endpoint: Optional[str] = None, + api_endpoint: str | None = None, atespace: str = DEFAULT_ATESPACE, worker_family: str = "c2", - auth_token: Optional[str] = None, + auth_token: str | None = None, *, - client: Optional[Client] = None, - image_templates: Optional[Mapping[str, str]] = None, - default_template: Optional[str] = None, - labels: Optional[Mapping[str, str]] = None, + client: Client | None = None, + image_templates: Mapping[str, str] | None = None, + default_template: str | None = None, + labels: Mapping[str, str] | None = None, acquire_timeout_s: float = 180.0, ready_timeout_s: float = 180.0, - loop_runner: Optional[LoopRunner] = None, + loop_runner: LoopRunner | None = None, ): target = ( api_endpoint @@ -81,8 +88,8 @@ def __init__( self._owns_client = True self._lock = threading.Lock() - self._owned_actors: Dict[str, RawSandboxInstance] = {} - self._warm_paused_pool: Dict[str, List[str]] = {} + self._owned_actors: dict[str, RawSandboxInstance] = {} + self._warm_paused_pool: dict[str, list[str]] = {} self._background_tasks: set[asyncio.Task[Any]] = set() def _get_loop_runner(self) -> LoopRunner: @@ -208,7 +215,7 @@ async def _acquire_internal(self, template_id: str, run_id: str) -> RawSandboxIn guest traffic, so ready_timeout_s does not measure readiness here. """ while True: - actor_id: Optional[str] = None + actor_id: str | None = None with self._lock: warm_list = self._warm_paused_pool.get(template_id) if warm_list: @@ -241,7 +248,7 @@ async def _acquire_internal(self, template_id: str, run_id: str) -> RawSandboxIn template_id, self.atespace, ) - last_err: Optional[Exception] = None + last_err: Exception | None = None for attempt in range(4): try: await self.client.create( @@ -277,7 +284,7 @@ def acquire( return self._sync(self.acquire_async(template_id, run_id, timeout_s=timeout_s)) async def warm_pool_async( - self, template_id: str, replicas: int, wait: bool = True, run_id: Optional[str] = None + self, template_id: str, replicas: int, wait: bool = True, run_id: str | None = None ) -> None: """Pre-warm a pool of actors by creating and suspending them.""" logger.info("Sizing warm pool for template %s to %d paused actors...", template_id, replicas) @@ -290,7 +297,7 @@ async def warm_pool_async( sem = asyncio.Semaphore(16) effective_run_id = run_id or "warm-pool" - async def _warm_one() -> Optional[str]: + async def _warm_one() -> str | None: actor_id = f"warm-{uuid.uuid4().hex[:10]}" async with sem: try: @@ -335,7 +342,7 @@ def _done(t: asyncio.Task[Any]) -> None: ) def warm_pool( - self, template_id: str, replicas: int, wait: bool = True, run_id: Optional[str] = None + self, template_id: str, replicas: int, wait: bool = True, run_id: str | None = None ) -> None: self._sync(self.warm_pool_async(template_id, replicas, wait=wait, run_id=run_id)) @@ -393,7 +400,7 @@ def release(self, instance_id: str, recycle: bool = False) -> None: async def reap_async(self, run_id: str) -> int: """Force delete all environments tagged with this run_id and all warm pool actors.""" with self._lock: - warm_actors: List[str] = [] + warm_actors: list[str] = [] for tid, actors in list(self._warm_paused_pool.items()): warm_actors.extend(actors) self._warm_paused_pool.clear() @@ -405,7 +412,7 @@ async def reap_async(self, run_id: str) -> int: to_reap = list(dict.fromkeys(to_reap + warm_actors)) sem = asyncio.Semaphore(16) - failed_actors: List[str] = [] + failed_actors: list[str] = [] reaped_count = 0 reap_lock = asyncio.Lock() @@ -438,7 +445,7 @@ def reap(self, run_id: str) -> int: def build_runtime( self, instance: RawSandboxInstance, data_plane: str = "ate_env" - ) -> tuple[Any, Optional[Any]]: + ) -> tuple[Any, Any | None]: from ..runtime.substrate_env_client import SubstrateEnvClientRuntime ep = instance.data_planes.get(data_plane) or instance.data_planes.get("ate_env") diff --git a/clients/python/src/ate_env/client.py b/clients/python/src/ate_env/client.py index ed1797a..dab33f1 100644 --- a/clients/python/src/ate_env/client.py +++ b/clients/python/src/ate_env/client.py @@ -16,6 +16,8 @@ from __future__ import annotations +import collections + import grpc import grpc.aio @@ -37,9 +39,6 @@ def _normalize_target(endpoint: str) -> str: return target -import collections - - class _ClientCallDetails( collections.namedtuple( "_ClientCallDetails", diff --git a/clients/python/src/ate_env/config.py b/clients/python/src/ate_env/config.py index a5404b8..86c099e 100644 --- a/clients/python/src/ate_env/config.py +++ b/clients/python/src/ate_env/config.py @@ -14,9 +14,9 @@ from __future__ import annotations -from dataclasses import MISSING, dataclass, field, fields import re -from typing import Any, Dict, List, Literal, Optional +from dataclasses import MISSING, dataclass, field, fields +from typing import Any, Literal BackendType = Literal["substrate", "mock"] StrategyType = Literal["none", "naive", "sliding", "pipelined"] @@ -72,26 +72,26 @@ class FleetConfig: data_plane: Literal["ate_env"] = "ate_env" tenancy: str = "ate-env" strategy: StrategyType = "sliding" - template: Optional[str] = None - image_templates: Dict[str, str] = field(default_factory=dict) + template: str | None = None + image_templates: dict[str, str] = field(default_factory=dict) # Sizing & Rollout parameters batch_size: int = 8 max_concurrent: int = 64 max_warmpool_replicas: int = 4 - window_size: Optional[int] = None + window_size: int | None = None # Hardware & placement (reserved for template placement and future scheduler extensions) - worker_family: Optional[str] = "c2" - node_selector: Dict[str, str] = field(default_factory=dict) - tolerations: List[Dict[str, Any]] = field(default_factory=list) - labels: Dict[str, str] = field(default_factory=dict) + worker_family: str | None = "c2" + node_selector: dict[str, str] = field(default_factory=dict) + tolerations: list[dict[str, Any]] = field(default_factory=list) + labels: dict[str, str] = field(default_factory=dict) # Timeouts & Breakers acquire_timeout_s: float = 180.0 ready_timeout_s: float = 180.0 step_timeout_s: float = 120.0 - auth_token: Optional[SecretStr] = None + auth_token: SecretStr | None = None def __init__(self, **kwargs: Any): valid_fields = {f.name: f for f in fields(self)} @@ -156,9 +156,9 @@ def __post_init__(self) -> None: if self.auth_token is not None and not isinstance(self.auth_token, SecretStr): object.__setattr__(self, "auth_token", SecretStr(str(self.auth_token))) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """Serialize configuration to dict, keeping auth_token as SecretStr for safe round-tripping.""" - res: Dict[str, Any] = {} + res: dict[str, Any] = {} for f in fields(self): val = getattr(self, f.name) res[f.name] = val @@ -167,7 +167,7 @@ def to_dict(self) -> Dict[str, Any]: model_dump = to_dict @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "FleetConfig": + def from_dict(cls, data: dict[str, Any]) -> FleetConfig: return cls(**data) model_validate = from_dict diff --git a/clients/python/src/ate_env/fleet.py b/clients/python/src/ate_env/fleet.py index 6e7e7bd..bad0ed8 100644 --- a/clients/python/src/ate_env/fleet.py +++ b/clients/python/src/ate_env/fleet.py @@ -18,22 +18,19 @@ import logging import threading import uuid -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from collections.abc import Callable, Sequence +from typing import Any from .backend.base import BackendDriver from .backend.mock import MockBackendDriver from .backend.substrate import SubstrateBackendDriver from .config import FleetConfig -from .errors import SandboxStartError from .handle import SandboxHandle from .runtime.base import RuntimeGuestHook -from .runtime.mock import MockRuntimeHook from .strategies import STRATEGIES from .types import ( DataPlaneEndpoint, - EnvironmentSpec, FleetPlan, - PlacementSpec, PlanEntry, RawSandboxInstance, Task, @@ -49,12 +46,12 @@ class SandboxFleet: task acquisition, and execution strategies across any backend driver. """ - def __init__(self, config: Optional[FleetConfig] = None, driver: Optional[BackendDriver] = None): + def __init__(self, config: FleetConfig | None = None, driver: BackendDriver | None = None): self.config = config or FleetConfig() self.run_id = uuid.uuid4().hex[:12] - self._tasks: List[Task] = [] - self._image_to_template: Dict[str, str] = {} - self._plan: Optional[FleetPlan] = None + self._tasks: list[Task] = [] + self._image_to_template: dict[str, str] = {} + self._plan: FleetPlan | None = None self._lock = threading.Lock() # Initialize Backend Driver @@ -78,7 +75,7 @@ def __init__(self, config: Optional[FleetConfig] = None, driver: Optional[Backen else: raise NotImplementedError(f"Backend '{self.config.backend}' is not yet supported or requires extras") - def __enter__(self) -> "SandboxFleet": + def __enter__(self) -> SandboxFleet: return self def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: @@ -89,7 +86,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: raise def close(self) -> None: - teardown_err: Optional[Exception] = None + teardown_err: Exception | None = None try: self.teardown() except Exception as e: @@ -105,12 +102,12 @@ def close(self) -> None: raise teardown_err @property - def tasks(self) -> List[Task]: + def tasks(self) -> list[Task]: return self._tasks - def load_tasks(self, tasks: List[Any]) -> None: + def load_tasks(self, tasks: list[Any]) -> None: """Load tasks into the fleet (supports list of Task or dicts).""" - normalized: List[Task] = [] + normalized: list[Task] = [] for t in tasks: if isinstance(t, Task): normalized.append(t) @@ -127,9 +124,9 @@ def load_tasks(self, tasks: List[Any]) -> None: logger.info("Fleet loaded %d tasks (%d unique images)", len(self._tasks), len(self.image_counts())) - def image_counts(self) -> Dict[str, int]: + def image_counts(self) -> dict[str, int]: """Return task count per unique image.""" - counts: Dict[str, int] = collections.defaultdict(int) + counts: dict[str, int] = collections.defaultdict(int) for t in self._tasks: counts[t.image] += 1 return dict(counts) @@ -140,7 +137,7 @@ def preflight(self) -> None: def plan(self) -> FleetPlan: """Compute provisioning sizing for all tasks.""" - entries: List[PlanEntry] = [] + entries: list[PlanEntry] = [] counts = self.image_counts() for img, task_count in counts.items(): template_id = self._ensure_template_for_image(img) @@ -155,7 +152,7 @@ def plan(self) -> FleetPlan: self._plan = FleetPlan(entries) return self._plan - def setup(self, tasks: Optional[List[Any]] = None) -> None: + def setup(self, tasks: list[Any] | None = None) -> None: """Run preflight, plan, and pre-warm all planned images.""" if tasks is not None: self.load_tasks(tasks) @@ -165,7 +162,7 @@ def setup(self, tasks: Optional[List[Any]] = None) -> None: if entry.replicas > 0: self.backend.warm_pool(entry.template_id, entry.replicas, wait=True) - def warm_images(self, images: List[str], replicas: Optional[int] = None, wait: bool = True) -> None: + def warm_images(self, images: list[str], replicas: int | None = None, wait: bool = True) -> None: """Warm up pools for specific images (used by windowed strategies).""" rep = replicas if replicas is not None else self.config.max_warmpool_replicas for img in images: @@ -181,8 +178,8 @@ def unwarm_image(self, image: str) -> None: def acquire( self, task: Task | str | Sequence[Task | str], - timeout_s: Optional[float] = None, - ) -> SandboxHandle | List[SandboxHandle]: + timeout_s: float | None = None, + ) -> SandboxHandle | list[SandboxHandle]: """Acquire one or more live sandboxes bound to tasks. If a list or tuple of tasks is given, returns a list of SandboxHandle. @@ -195,7 +192,7 @@ def acquire( def _acquire_one( self, task: Task | str, - timeout_s: Optional[float] = None, + timeout_s: float | None = None, ) -> SandboxHandle: task_obj: Task if isinstance(task, str): @@ -240,7 +237,7 @@ def _acquire_one( def _build_runtime( self, raw_inst: RawSandboxInstance - ) -> Tuple[RuntimeGuestHook, Optional[DataPlaneEndpoint]]: + ) -> tuple[RuntimeGuestHook, DataPlaneEndpoint | None]: """Build the RuntimeGuestHook from this sandbox's own data-plane coordinates.""" if hasattr(self.backend, "build_runtime"): return self.backend.build_runtime(raw_inst, data_plane=self.config.data_plane) @@ -272,7 +269,7 @@ def teardown(self) -> None: self.backend.reap(self.run_id) def run(self, process_fn: Callable[[Task, SandboxHandle], Any], - concurrency: Optional[int] = None) -> List[Any]: + concurrency: int | None = None) -> list[Any]: """Execute all tasks using the configured strategy.""" strat_fn = STRATEGIES.get(self.config.strategy) if not strat_fn: diff --git a/clients/python/src/ate_env/handle.py b/clients/python/src/ate_env/handle.py index 1373743..0199b49 100644 --- a/clients/python/src/ate_env/handle.py +++ b/clients/python/src/ate_env/handle.py @@ -15,7 +15,6 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional from .backend.base import BackendDriver from .errors import CommandExecutionError, CommandTimeoutError @@ -40,13 +39,13 @@ class SandboxHandle: run_id: str backend: BackendDriver = field(repr=False) runtime: RuntimeGuestHook = field(repr=False) - data_plane: Optional[DataPlaneEndpoint] = None + data_plane: DataPlaneEndpoint | None = None def exec( self, - command: str | List[str], + command: str | list[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, + env: dict[str, str] | None = None, timeout_s: float = 120.0, check: bool = True, ) -> ExecResult: @@ -75,9 +74,9 @@ def exec( async def exec_async( self, - command: str | List[str], + command: str | list[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, + env: dict[str, str] | None = None, timeout_s: float = 120.0, check: bool = True, ) -> ExecResult: @@ -107,7 +106,7 @@ async def read_file_bytes_async(self, path: str) -> bytes: def initialize( self, - init_script: Optional[str] = None, + init_script: str | None = None, cwd: str = "", timeout_s: float = 120.0, ) -> ExecResult: @@ -133,13 +132,13 @@ def recycle(self) -> None: finally: self.runtime.close() - def __enter__(self) -> "SandboxHandle": + def __enter__(self) -> SandboxHandle: return self def __exit__(self, exc_type, exc_val, exc_tb) -> None: self.release() - async def __aenter__(self) -> "SandboxHandle": + async def __aenter__(self) -> SandboxHandle: return self async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: diff --git a/clients/python/src/ate_env/loop_runner.py b/clients/python/src/ate_env/loop_runner.py index 34120f3..9eb54e3 100644 --- a/clients/python/src/ate_env/loop_runner.py +++ b/clients/python/src/ate_env/loop_runner.py @@ -17,7 +17,8 @@ import asyncio import concurrent.futures import threading -from typing import Any, Coroutine, TypeVar +from collections.abc import Coroutine +from typing import Any, TypeVar T = TypeVar("T") diff --git a/clients/python/src/ate_env/runtime/base.py b/clients/python/src/ate_env/runtime/base.py index d89b038..7ce9a6e 100644 --- a/clients/python/src/ate_env/runtime/base.py +++ b/clients/python/src/ate_env/runtime/base.py @@ -15,12 +15,12 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Dict, Iterator, List, Optional +from collections.abc import Iterator from ..types import ExecResult -def command_to_argv(command: str | List[str]) -> List[str]: +def command_to_argv(command: str | list[str]) -> list[str]: """Command contract shared by every runtime. A ``str`` is a shell command and always runs as ``bash -c ``. A @@ -45,8 +45,8 @@ class RuntimeGuestHook(ABC): """ @abstractmethod - def exec(self, command: str | List[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, timeout_s: float = 120.0) -> ExecResult: + def exec(self, command: str | list[str], cwd: str = "", + env: dict[str, str] | None = None, timeout_s: float = 120.0) -> ExecResult: """Execute command inside the guest container/actor. Returns an ExecResult only if the guest ran the command and it either @@ -56,8 +56,8 @@ def exec(self, command: str | List[str], cwd: str = "", """ @abstractmethod - def stream_process(self, argv: List[str], cwd: str = "", - env: Optional[Dict[str, str]] = None) -> Iterator[str]: + def stream_process(self, argv: list[str], cwd: str = "", + env: dict[str, str] | None = None) -> Iterator[str]: """Stream real-time output chunks from a running guest process.""" @abstractmethod @@ -68,8 +68,8 @@ def write_file(self, path: str, content: bytes | str) -> None: def read_file_bytes(self, path: str) -> bytes: """Read binary file content from the guest filesystem.""" - async def exec_async(self, command: str | List[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, timeout_s: float = 120.0) -> ExecResult: + async def exec_async(self, command: str | list[str], cwd: str = "", + env: dict[str, str] | None = None, timeout_s: float = 120.0) -> ExecResult: """Asynchronously execute command inside the guest container/actor.""" import asyncio return await asyncio.to_thread(self.exec, command, cwd=cwd, env=env, timeout_s=timeout_s) @@ -84,7 +84,7 @@ async def read_file_bytes_async(self, path: str) -> bytes: import asyncio return await asyncio.to_thread(self.read_file_bytes, path) - def initialize(self, init_script: Optional[str] = None, cwd: str = "", + def initialize(self, init_script: str | None = None, cwd: str = "", timeout_s: float = 120.0) -> ExecResult: """Run post-boot initialization script inside the guest.""" if not init_script: diff --git a/clients/python/src/ate_env/runtime/mock.py b/clients/python/src/ate_env/runtime/mock.py index 0d936c2..c81a195 100644 --- a/clients/python/src/ate_env/runtime/mock.py +++ b/clients/python/src/ate_env/runtime/mock.py @@ -14,8 +14,7 @@ from __future__ import annotations -import io -from typing import Dict, Iterator, List, Optional, Union +from collections.abc import Iterator from ..types import ExecResult from .base import RuntimeGuestHook @@ -25,17 +24,17 @@ class MockRuntimeHook(RuntimeGuestHook): """In-memory mock guest hook implementing file operations and command responses.""" def __init__(self): - self.files: Dict[str, bytes] = {} - self.executed_commands: List[str] = [] - self.custom_responses: Dict[str, Union[ExecResult, BaseException]] = {} + self.files: dict[str, bytes] = {} + self.executed_commands: list[str] = [] + self.custom_responses: dict[str, ExecResult | BaseException] = {} def set_response(self, command_substring: str, - result: Union[ExecResult, BaseException]) -> None: + result: ExecResult | BaseException) -> None: """Preset a result, or an exception (e.g. SandboxUnavailableError) to raise.""" self.custom_responses[command_substring] = result - def exec(self, command: str | List[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, timeout_s: float = 120.0) -> ExecResult: + def exec(self, command: str | list[str], cwd: str = "", + env: dict[str, str] | None = None, timeout_s: float = 120.0) -> ExecResult: cmd_str = command if isinstance(command, str) else " ".join(command) self.executed_commands.append(cmd_str) @@ -60,8 +59,8 @@ def exec(self, command: str | List[str], cwd: str = "", return ExecResult(exit_code=0, stdout=f"Mock exec ok: {cmd_str}\n", stderr="") - def stream_process(self, argv: List[str], cwd: str = "", - env: Optional[Dict[str, str]] = None) -> Iterator[str]: + def stream_process(self, argv: list[str], cwd: str = "", + env: dict[str, str] | None = None) -> Iterator[str]: cmd_str = " ".join(argv) yield f"Process started: {cmd_str}\n" yield "Process completed.\n" diff --git a/clients/python/src/ate_env/runtime/substrate_env_client.py b/clients/python/src/ate_env/runtime/substrate_env_client.py index 247fc28..0dc75fc 100644 --- a/clients/python/src/ate_env/runtime/substrate_env_client.py +++ b/clients/python/src/ate_env/runtime/substrate_env_client.py @@ -20,7 +20,8 @@ import queue import threading import time -from typing import Any, AsyncIterator, Dict, Iterator, List, Optional +from collections.abc import AsyncIterator, Iterator +from typing import Any import grpc @@ -28,7 +29,6 @@ from ..env import Env as AteEnv from ..errors import ( CommandStartError, - EnvError, InfrastructureError, InvalidArgumentError, NotFoundError, @@ -54,11 +54,11 @@ def __init__( self, env_id: str, *, - endpoint: Optional[str] = None, + endpoint: str | None = None, atespace: str = "ate-env", - client: Optional[AteClient] = None, - env: Optional[AteEnv] = None, - loop_runner: Optional[LoopRunner] = None, + client: AteClient | None = None, + env: AteEnv | None = None, + loop_runner: LoopRunner | None = None, ): if not env_id: raise ValueError("env_id is required for SubstrateEnvClientRuntime") @@ -134,9 +134,9 @@ def _sync(self, coro: Any) -> Any: async def exec_async( self, - command: str | List[str], + command: str | list[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, + env: dict[str, str] | None = None, timeout_s: float = 120.0, ) -> ExecResult: """Native async execution honoring command contract: argv list runs directly, string runs in shell.""" @@ -247,9 +247,9 @@ async def _run_argv() -> ShellResult: def exec( self, - command: str | List[str], + command: str | list[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, + env: dict[str, str] | None = None, timeout_s: float = 120.0, ) -> ExecResult: """Synchronous wrapper for exec_async.""" @@ -257,9 +257,9 @@ def exec( async def stream_process_async( self, - argv: List[str], + argv: list[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, + env: dict[str, str] | None = None, ) -> AsyncIterator[str]: """Stream real-time output chunks from running process inside guest.""" self._check_open() @@ -277,13 +277,13 @@ async def stream_process_async( def stream_process( self, - argv: List[str], + argv: list[str], cwd: str = "", - env: Optional[Dict[str, str]] = None, + env: dict[str, str] | None = None, ) -> Iterator[str]: """Synchronously stream output chunks in real time.""" self._check_open() - q: queue.Queue[tuple[Optional[str], Optional[Exception]]] = queue.Queue() + q: queue.Queue[tuple[str | None, Exception | None]] = queue.Queue() async def _pump(): try: diff --git a/clients/python/src/ate_env/strategies.py b/clients/python/src/ate_env/strategies.py index 6652c22..1b6392a 100644 --- a/clients/python/src/ate_env/strategies.py +++ b/clients/python/src/ate_env/strategies.py @@ -16,9 +16,9 @@ import collections import logging -import time +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any, Callable, Dict, List, Optional +from typing import Any from .handle import SandboxHandle from .types import Task @@ -26,10 +26,10 @@ logger = logging.getLogger("ate_env.strategies") -def process_parallel(fleet: Any, tasks: List[Task], process_fn: Callable[[Task, SandboxHandle], Any], - concurrency: int) -> List[Any]: +def process_parallel(fleet: Any, tasks: list[Task], process_fn: Callable[[Task, SandboxHandle], Any], + concurrency: int) -> list[Any]: """Execute acquire -> process_fn -> release across tasks up to concurrency.""" - results: List[Any] = [None] * len(tasks) + results: list[Any] = [None] * len(tasks) def _execute_single(task: Task) -> Any: handle = fleet.acquire(task) @@ -61,7 +61,7 @@ def _execute_single(task: Task) -> Any: def run_none(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], - concurrency: int, teardown: bool = True) -> List[Any]: + concurrency: int, teardown: bool = True) -> list[Any]: """No pre-warming: on-demand 1:1 acquisition per task.""" try: return process_parallel(fleet, fleet.tasks, process_fn, concurrency) @@ -71,7 +71,7 @@ def run_none(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], def run_naive(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], - concurrency: int, teardown: bool = True) -> List[Any]: + concurrency: int, teardown: bool = True) -> list[Any]: """Pre-warm every image up front, process all tasks in parallel, tear down.""" try: fleet.setup() @@ -82,18 +82,18 @@ def run_naive(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], def run_sliding(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], - concurrency: int, teardown: bool = True) -> List[Any]: + concurrency: int, teardown: bool = True) -> list[Any]: """Keep only a sliding window of image pools warm at a time (bounded footprint).""" fleet.preflight() fleet.plan() window = fleet.config.window_size or fleet.config.batch_size images = list(fleet.image_counts().keys()) - by_image: Dict[str, List[tuple[int, Task]]] = collections.defaultdict(list) + by_image: dict[str, list[tuple[int, Task]]] = collections.defaultdict(list) for i, t in enumerate(fleet.tasks): by_image[t.image].append((i, t)) - results: List[Any] = [None] * len(fleet.tasks) + results: list[Any] = [None] * len(fleet.tasks) try: for start in range(0, len(images), window): batch = images[start:start + window] @@ -117,7 +117,7 @@ def run_sliding(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], def run_pipelined(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], - concurrency: int, teardown: bool = True) -> List[Any]: + concurrency: int, teardown: bool = True) -> list[Any]: """ Double-buffered pipelined sliding window. @@ -129,11 +129,11 @@ def run_pipelined(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], window = fleet.config.window_size or fleet.config.batch_size images = list(fleet.image_counts().keys()) - by_image: Dict[str, List[tuple[int, Task]]] = collections.defaultdict(list) + by_image: dict[str, list[tuple[int, Task]]] = collections.defaultdict(list) for i, t in enumerate(fleet.tasks): by_image[t.image].append((i, t)) - results: List[Any] = [None] * len(fleet.tasks) + results: list[Any] = [None] * len(fleet.tasks) batches = [images[s:s + window] for s in range(0, len(images), window)] prefetch_executor = ThreadPoolExecutor(max_workers=1) @@ -171,7 +171,7 @@ def run_pipelined(fleet: Any, process_fn: Callable[[Task, SandboxHandle], Any], return results -STRATEGIES: Dict[str, Callable[..., List[Any]]] = { +STRATEGIES: dict[str, Callable[..., list[Any]]] = { "none": run_none, "naive": run_naive, "sliding": run_sliding, diff --git a/clients/python/src/ate_env/types.py b/clients/python/src/ate_env/types.py index 1377b19..c2ed06e 100644 --- a/clients/python/src/ate_env/types.py +++ b/clients/python/src/ate_env/types.py @@ -24,7 +24,7 @@ import hashlib from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Literal from ._gen.ateenv.v1alpha import env_pb2, guest_pb2 @@ -217,9 +217,9 @@ class ResourceLimits: @dataclass(frozen=True) class PlacementSpec: """Placement constraints and worker hardware selectors.""" - node_selector: Dict[str, str] = field(default_factory=dict) - tolerations: List[Dict[str, Any]] = field(default_factory=list) - worker_family: Optional[str] = None # e.g., "c2", "n2", "c3" + node_selector: dict[str, str] = field(default_factory=dict) + tolerations: list[dict[str, Any]] = field(default_factory=list) + worker_family: str | None = None # e.g., "c2", "n2", "c3" @dataclass(frozen=True) @@ -229,7 +229,7 @@ class EnvironmentSpec: runtime_bundle: str = "default-guest" limits: ResourceLimits = field(default_factory=ResourceLimits) placement: PlacementSpec = field(default_factory=PlacementSpec) - snapshot_storage_uri: Optional[str] = None + snapshot_storage_uri: str | None = None def template_key(self) -> str: """Derive canonical template ID including CPU family and bundle digest.""" @@ -245,8 +245,8 @@ class Task: """Single workload/task definition (e.g. one SWE-bench instance).""" id: str image: str - metadata: Dict[str, Any] = field(default_factory=dict) - template: Optional[str] = None + metadata: dict[str, Any] = field(default_factory=dict) + template: str | None = None @dataclass @@ -267,7 +267,7 @@ class ExecResult: terminated by an external signal (such as an out-of-memory killer) near its deadline. """ - exit_code: Optional[int] + exit_code: int | None stdout: str stderr: str duration_s: float = 0.0 @@ -300,7 +300,7 @@ class DataPlaneEndpoint: """ protocol: Literal["grpc"] = "grpc" address: str = "" - headers: Dict[str, str] = field(default_factory=dict) + headers: dict[str, str] = field(default_factory=dict) def __repr__(self) -> str: shown = {k: ("" if k.lower() in _SENSITIVE_HEADERS else v) @@ -316,9 +316,9 @@ class RawSandboxInstance: endpoint: str template_id: str run_id: str - metadata: Dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) # Keyed by data-plane name matching FleetConfig.data_plane ("router", "grpc"). - data_planes: Dict[str, DataPlaneEndpoint] = field(default_factory=dict) + data_planes: dict[str, DataPlaneEndpoint] = field(default_factory=dict) @dataclass @@ -332,11 +332,11 @@ class PlanEntry: class FleetPlan: """The complete provisioning plan computed by SandboxFleet.plan().""" - def __init__(self, entries: List[PlanEntry]): + def __init__(self, entries: list[PlanEntry]): self.entries = entries self._by_image = {e.image: e for e in entries} - def for_image(self, image: str) -> Optional[PlanEntry]: + def for_image(self, image: str) -> PlanEntry | None: return self._by_image.get(image) @property diff --git a/clients/python/tests/test_data_planes.py b/clients/python/tests/test_data_planes.py index cd5bc28..47110db 100644 --- a/clients/python/tests/test_data_planes.py +++ b/clients/python/tests/test_data_planes.py @@ -17,11 +17,9 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock -import pytest -from ate_env import DataPlaneEndpoint, FleetConfig, SandboxFleet +from ate_env import FleetConfig, SandboxFleet from ate_env.backend.substrate import SubstrateBackendDriver -from ate_env.errors import SandboxStartError from ate_env.runtime.mock import MockRuntimeHook from ate_env.runtime.substrate_env_client import SubstrateEnvClientRuntime from ate_env.types import EnvironmentSpec diff --git a/clients/python/tests/test_exec_semantics.py b/clients/python/tests/test_exec_semantics.py index 68d22d9..db116ea 100644 --- a/clients/python/tests/test_exec_semantics.py +++ b/clients/python/tests/test_exec_semantics.py @@ -15,7 +15,6 @@ """Exec result contract, strict-mode errors, command contract, and file helpers.""" import pickle -import subprocess import pytest @@ -31,7 +30,6 @@ ) from ate_env.exceptions import TimeoutError as SdkTimeoutError from ate_env.runtime.base import ( - RuntimeGuestHook, command_to_argv, ) from ate_env.runtime.mock import MockRuntimeHook diff --git a/clients/python/tests/test_fleet_strategies.py b/clients/python/tests/test_fleet_strategies.py index 8bb7b75..a317c10 100644 --- a/clients/python/tests/test_fleet_strategies.py +++ b/clients/python/tests/test_fleet_strategies.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest + from ate_env.config import FleetConfig from ate_env.fleet import SandboxFleet from ate_env.types import Task diff --git a/clients/python/tests/test_fleet_types.py b/clients/python/tests/test_fleet_types.py index 836ac90..03a056e 100644 --- a/clients/python/tests/test_fleet_types.py +++ b/clients/python/tests/test_fleet_types.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest -from ate_env.types import EnvironmentSpec, PlacementSpec, ResourceLimits, Task +from ate_env.types import EnvironmentSpec, PlacementSpec, Task def test_environment_spec_template_key_hashing(): diff --git a/clients/python/tests/test_substrate_driver.py b/clients/python/tests/test_substrate_driver.py index c402000..a5d5d58 100644 --- a/clients/python/tests/test_substrate_driver.py +++ b/clients/python/tests/test_substrate_driver.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio + import pytest from ate_env import FleetConfig, SandboxFleet, Task diff --git a/clients/python/tests/test_substrate_env_client.py b/clients/python/tests/test_substrate_env_client.py index 80b98b4..27458a5 100644 --- a/clients/python/tests/test_substrate_env_client.py +++ b/clients/python/tests/test_substrate_env_client.py @@ -15,7 +15,8 @@ from __future__ import annotations import asyncio -from unittest.mock import AsyncMock, patch +from unittest.mock import patch + import grpc import pytest @@ -28,6 +29,7 @@ SandboxUnavailableError, ) from ate_env.runtime.substrate_env_client import SubstrateEnvClientRuntime + from .fakes import FakeProc