From 14bbd8012b4f0ba95b9fe2439f74be93da25e598 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Mon, 12 Oct 2026 16:33:54 +0000 Subject: [PATCH 01/20] Singleton subresource Get and Delete requests take a name (#2452) #### Breaking change `GetActorEgressPolicy`, `DeleteActorEgressPolicy`, `GetAtespaceAccessPolicy`, `DeleteAtespaceAccessPolicy` and `GetGlobalAccessPolicy` now require `name` set to `default`. Please add the `breaking-change` label. Fixes #2363 Adds `name` to those requests, per the style guide. Any other value is `INVALID_ARGUMENT`. `resources.SingletonName` replaces the repeated `"default"` literals. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .../internal/apivalidation/access_policy.go | 24 ++- .../apivalidation/access_policy_test.go | 35 +++- cmd/ateapi/internal/apivalidation/common.go | 10 + .../internal/apivalidation/egress_policy.go | 16 +- .../apivalidation/egress_policy_test.go | 47 ++++- .../apivalidation/zz_generated.validation.go | 189 ++++++++++++++++++ .../internal/controlapi/access_policy_test.go | 7 +- .../internal/controlapi/egress_policy_test.go | 6 +- .../functionaltest/access_policy_test.go | 18 +- .../functionaltest/egress_policy_test.go | 19 +- .../internal/router/egress/egress_test.go | 6 +- .../internal/router/egress/policycache.go | 2 +- cmd/kubectl-ate/internal/cmd/egress_policy.go | 12 +- .../internal/cmd/egress_policy_test.go | 25 +-- docs/api-style-guide.md | 2 - internal/e2e/egresspolicy.go | 5 +- internal/resources/singleton.go | 19 ++ pkg/proto/ateapipb/ateapi.pb.go | 104 ++++++++-- pkg/proto/ateapipb/ateapi.proto | 41 +++- 19 files changed, 501 insertions(+), 86 deletions(-) create mode 100644 internal/resources/singleton.go diff --git a/cmd/ateapi/internal/apivalidation/access_policy.go b/cmd/ateapi/internal/apivalidation/access_policy.go index 1f281ceb27..2871c9f45a 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy.go +++ b/cmd/ateapi/internal/apivalidation/access_policy.go @@ -40,10 +40,9 @@ func ValidateCreateGlobalAccessPolicyRequest(ctx context.Context, req *ateapipb. return append(errs, validateBindings(bindingsPath, req.GetAccessPolicy(), validGlobalRoles)...) } -// ValidateGetGlobalAccessPolicyRequest accepts every request: the message has -// no fields, so validation-gen emits no validator for it. -func ValidateGetGlobalAccessPolicyRequest(_ context.Context, _ *ateapipb.GetGlobalAccessPolicyRequest) field.ErrorList { - return nil +func ValidateGetGlobalAccessPolicyRequest(ctx context.Context, req *ateapipb.GetGlobalAccessPolicyRequest) field.ErrorList { + op := operation.Operation{Type: operation.Create} + return Validate_GetGlobalAccessPolicyRequest(ctx, op, nil, req, nil) } func ValidateUpdateGlobalAccessPolicyRequest(ctx context.Context, req *ateapipb.UpdateGlobalAccessPolicyRequest) field.ErrorList { @@ -101,10 +100,19 @@ func ValidateCustom_AccessPolicy_Metadata(_ context.Context, _ operation.Operati if meta.GetAtespace() != "" { errs = append(errs, field.Forbidden(root.Child("atespace"), "must not be set")) } - if name := meta.GetName(); name != "" && name != "default" { - errs = append(errs, field.Invalid(root.Child("name"), name, `must be "default"`).WithOrigin("custom=default")) - } - return errs + return append(errs, validateSingletonName(root.Child("name"), meta.GetName())...) +} + +func ValidateCustom_GetGlobalAccessPolicyRequest_Name(_ context.Context, _ operation.Operation, p *field.Path, name, _ *string) field.ErrorList { + return validateSingletonName(p, *name) +} + +func ValidateCustom_GetAtespaceAccessPolicyRequest_Name(_ context.Context, _ operation.Operation, p *field.Path, name, _ *string) field.ErrorList { + return validateSingletonName(p, *name) +} + +func ValidateCustom_DeleteAtespaceAccessPolicyRequest_Name(_ context.Context, _ operation.Operation, p *field.Path, name, _ *string) field.ErrorList { + return validateSingletonName(p, *name) } // validateBindings checks that each binding names an allowed role at most once, diff --git a/cmd/ateapi/internal/apivalidation/access_policy_test.go b/cmd/ateapi/internal/apivalidation/access_policy_test.go index 9ac5dcad47..d229c84df2 100644 --- a/cmd/ateapi/internal/apivalidation/access_policy_test.go +++ b/cmd/ateapi/internal/apivalidation/access_policy_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -170,7 +171,15 @@ func TestValidateGetGlobalAccessPolicyRequest(t *testing.T) { wantError field.ErrorList }{{ name: "valid", - req: &ateapipb.GetGlobalAccessPolicyRequest{}, + req: &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}, + }, { + name: "missing name", + req: &ateapipb.GetGlobalAccessPolicyRequest{}, + wantError: field.ErrorList{field.Required(field.NewPath("name"), "")}, + }, { + name: "wrong name", + req: &ateapipb.GetGlobalAccessPolicyRequest{Name: "other"}, + wantError: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("custom=default")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -298,11 +307,19 @@ func TestValidateGetAtespaceAccessPolicyRequest(t *testing.T) { wantError field.ErrorList }{{ name: "valid", - req: &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}, + req: &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, Name: resources.SingletonName}, }, { name: "missing atespace", - req: &ateapipb.GetAtespaceAccessPolicyRequest{}, + req: &ateapipb.GetAtespaceAccessPolicyRequest{Name: resources.SingletonName}, wantError: field.ErrorList{field.Required(field.NewPath("atespace"), "")}, + }, { + name: "missing name", + req: &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}, + wantError: field.ErrorList{field.Required(field.NewPath("name"), "")}, + }, { + name: "wrong name", + req: &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, Name: "other"}, + wantError: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("custom=default")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -372,11 +389,19 @@ func TestValidateDeleteAtespaceAccessPolicyRequest(t *testing.T) { wantError field.ErrorList }{{ name: "valid", - req: &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}, + req: &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, Name: resources.SingletonName}, }, { name: "missing atespace", - req: &ateapipb.DeleteAtespaceAccessPolicyRequest{}, + req: &ateapipb.DeleteAtespaceAccessPolicyRequest{Name: resources.SingletonName}, wantError: field.ErrorList{field.Required(field.NewPath("atespace"), "")}, + }, { + name: "missing name", + req: &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}, + wantError: field.ErrorList{field.Required(field.NewPath("name"), "")}, + }, { + name: "wrong name", + req: &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}, Name: "other"}, + wantError: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("custom=default")}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/apivalidation/common.go b/cmd/ateapi/internal/apivalidation/common.go index cee0a6f548..d99c419428 100644 --- a/cmd/ateapi/internal/apivalidation/common.go +++ b/cmd/ateapi/internal/apivalidation/common.go @@ -16,6 +16,7 @@ package apivalidation import ( "context" + "fmt" "net/netip" "github.com/agent-substrate/substrate/internal/resources" @@ -31,6 +32,15 @@ func ateDeepEqual[T any](a, b T) bool { return resources.DeepEqual(a, b) } +// validateSingletonName requires the name of a singleton subresource to be +// resources.SingletonName. An empty name is left to the required tag. +func validateSingletonName(fldPath *field.Path, name string) field.ErrorList { + if name == "" || name == resources.SingletonName { + return nil + } + return field.ErrorList{field.Invalid(fldPath, name, fmt.Sprintf("must be %q", resources.SingletonName)).WithOrigin("custom=default")} +} + // ValidateCustom_ResourceMetadata checks the server-stamped timestamps: each, // when set, must be a valid google.protobuf.Timestamp, and update_time must // not precede create_time. Both fields are scrubbed from input, so a diff --git a/cmd/ateapi/internal/apivalidation/egress_policy.go b/cmd/ateapi/internal/apivalidation/egress_policy.go index df9961ee43..7a6bdabce7 100644 --- a/cmd/ateapi/internal/apivalidation/egress_policy.go +++ b/cmd/ateapi/internal/apivalidation/egress_policy.go @@ -73,13 +73,15 @@ func validateEgressPolicyParentAtespace(actor *ateapipb.ObjectRef, policy *ateap } func ValidateCustom_EgressPolicy_Metadata(_ context.Context, _ operation.Operation, root *field.Path, meta, _ *ateapipb.ResourceMetadata) field.ErrorList { - if meta == nil || meta.Name == "" { - return nil // regular DV will handle it - } - if meta.Name != "default" { - return field.ErrorList{field.Invalid(root.Child("name"), meta.Name, `must be "default"`).WithOrigin("custom=default")} - } - return nil + return validateSingletonName(root.Child("name"), meta.GetName()) +} + +func ValidateCustom_GetActorEgressPolicyRequest_Name(_ context.Context, _ operation.Operation, p *field.Path, name, _ *string) field.ErrorList { + return validateSingletonName(p, *name) +} + +func ValidateCustom_DeleteActorEgressPolicyRequest_Name(_ context.Context, _ operation.Operation, p *field.Path, name, _ *string) field.ErrorList { + return validateSingletonName(p, *name) } // ValidateCustom_EgressPolicy_Rules rejects two rules that tie on a pattern diff --git a/cmd/ateapi/internal/apivalidation/egress_policy_test.go b/cmd/ateapi/internal/apivalidation/egress_policy_test.go index 99008748c3..b45b0a1774 100644 --- a/cmd/ateapi/internal/apivalidation/egress_policy_test.go +++ b/cmd/ateapi/internal/apivalidation/egress_policy_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/defaults" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/validation/field" @@ -192,6 +193,7 @@ func TestValidateGetActorEgressPolicyRequest(t *testing.T) { validReq := func() *ateapipb.GetActorEgressPolicyRequest { return &ateapipb.GetActorEgressPolicyRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor"}, + Name: resources.SingletonName, } } tests := []struct { @@ -203,7 +205,7 @@ func TestValidateGetActorEgressPolicyRequest(t *testing.T) { req: validReq(), }, { name: "missing actor", - req: &ateapipb.GetActorEgressPolicyRequest{}, + req: &ateapipb.GetActorEgressPolicyRequest{Name: resources.SingletonName}, want: field.ErrorList{ field.Required(field.NewPath("actor"), ""), }, @@ -247,6 +249,26 @@ func TestValidateGetActorEgressPolicyRequest(t *testing.T) { want: field.ErrorList{ field.Invalid(field.NewPath("actor", "name"), nil, "").WithOrigin("format=k8s-short-name"), }, + }, { + name: "missing singleton name", + req: func() *ateapipb.GetActorEgressPolicyRequest { + r := validReq() + r.Name = "" + return r + }(), + want: field.ErrorList{ + field.Required(field.NewPath("name"), ""), + }, + }, { + name: "wrong singleton name", + req: func() *ateapipb.GetActorEgressPolicyRequest { + r := validReq() + r.Name = "other" + return r + }(), + want: field.ErrorList{ + field.Invalid(field.NewPath("name"), nil, "").WithOrigin("custom=default"), + }, }} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -321,6 +343,7 @@ func TestValidateDeleteActorEgressPolicyRequest(t *testing.T) { validReq := func() *ateapipb.DeleteActorEgressPolicyRequest { return &ateapipb.DeleteActorEgressPolicyRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor"}, + Name: resources.SingletonName, } } tests := []struct { @@ -332,7 +355,7 @@ func TestValidateDeleteActorEgressPolicyRequest(t *testing.T) { req: validReq(), }, { name: "missing actor", - req: &ateapipb.DeleteActorEgressPolicyRequest{}, + req: &ateapipb.DeleteActorEgressPolicyRequest{Name: resources.SingletonName}, want: field.ErrorList{ field.Required(field.NewPath("actor"), ""), }, @@ -376,6 +399,26 @@ func TestValidateDeleteActorEgressPolicyRequest(t *testing.T) { want: field.ErrorList{ field.Invalid(field.NewPath("actor", "name"), nil, "").WithOrigin("format=k8s-short-name"), }, + }, { + name: "missing singleton name", + req: func() *ateapipb.DeleteActorEgressPolicyRequest { + r := validReq() + r.Name = "" + return r + }(), + want: field.ErrorList{ + field.Required(field.NewPath("name"), ""), + }, + }, { + name: "wrong singleton name", + req: func() *ateapipb.DeleteActorEgressPolicyRequest { + r := validReq() + r.Name = "other" + return r + }(), + want: field.ErrorList{ + field.Invalid(field.NewPath("name"), nil, "").WithOrigin("custom=default"), + }, }} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go index 25b7bee5f0..a5fc763555 100644 --- a/cmd/ateapi/internal/apivalidation/zz_generated.validation.go +++ b/cmd/ateapi/internal/apivalidation/zz_generated.validation.go @@ -2604,6 +2604,42 @@ func Validate_DeleteActorEgressPolicyRequest( errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) } + { // field ateapipb.DeleteActorEgressPolicyRequest.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_DeleteActorEgressPolicyRequest_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.DeleteActorEgressPolicyRequest) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + { // field ateapipb.DeleteActorEgressPolicyRequest.Options fn := func( fldPath *field.Path, @@ -2891,6 +2927,42 @@ func Validate_DeleteAtespaceAccessPolicyRequest( errs = append(errs, fn(fldPath.Child("atespace"), obj.Atespace, oldVal, oldObj != nil)...) } + { // field ateapipb.DeleteAtespaceAccessPolicyRequest.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_DeleteAtespaceAccessPolicyRequest_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.DeleteAtespaceAccessPolicyRequest) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + { // field ateapipb.DeleteAtespaceAccessPolicyRequest.Options fn := func( fldPath *field.Path, @@ -3957,6 +4029,42 @@ func Validate_GetActorEgressPolicyRequest( errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) } + { // field ateapipb.GetActorEgressPolicyRequest.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_GetActorEgressPolicyRequest_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.GetActorEgressPolicyRequest) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + return errs } @@ -4126,6 +4234,42 @@ func Validate_GetAtespaceAccessPolicyRequest( errs = append(errs, fn(fldPath.Child("atespace"), obj.Atespace, oldVal, oldObj != nil)...) } + { // field ateapipb.GetAtespaceAccessPolicyRequest.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_GetAtespaceAccessPolicyRequest_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.GetAtespaceAccessPolicyRequest) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + return errs } @@ -4188,6 +4332,51 @@ func Validate_GetAtespaceRequest( return errs } +// Validate_GetGlobalAccessPolicyRequest validates an instance of GetGlobalAccessPolicyRequest according +// to declarative validation rules in the API schema. +func Validate_GetGlobalAccessPolicyRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.GetGlobalAccessPolicyRequest) (errs field.ErrorList) { + + { // field ateapipb.GetGlobalAccessPolicyRequest.Name + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // custom validation + if e := ValidateCustom_GetGlobalAccessPolicyRequest_Name(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + if e := validate.ShortName(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.GetGlobalAccessPolicyRequest) *string { + return &oldObj.Name + }) + errs = append(errs, fn(fldPath.Child("name"), &obj.Name, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_GetTagRequest validates an instance of GetTagRequest according // to declarative validation rules in the API schema. func Validate_GetTagRequest( diff --git a/cmd/ateapi/internal/controlapi/access_policy_test.go b/cmd/ateapi/internal/controlapi/access_policy_test.go index 5b28e3b40f..ca93d15782 100644 --- a/cmd/ateapi/internal/controlapi/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/access_policy_test.go @@ -23,6 +23,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/principal" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -65,7 +66,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { return interceptor(c, req, &grpc.UnaryServerInfo{FullMethod: method}, handler) } getGlobal := func(c context.Context) (*ateapipb.AccessPolicy, error) { - got, err := invoke(c, ateapipb.Control_GetGlobalAccessPolicy_FullMethodName, &ateapipb.GetGlobalAccessPolicyRequest{}, func(c context.Context, r any) (any, error) { + got, err := invoke(c, ateapipb.Control_GetGlobalAccessPolicy_FullMethodName, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}, func(c context.Context, r any) (any, error) { return svc.GetGlobalAccessPolicy(c, r.(*ateapipb.GetGlobalAccessPolicyRequest)) }) if err != nil { @@ -201,7 +202,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { } restartedInterceptor := authz.UnaryServerInterceptor(restarted, true) getGlobalAfterRestart := func(c context.Context) error { - _, err := restartedInterceptor(c, &ateapipb.GetGlobalAccessPolicyRequest{}, &grpc.UnaryServerInfo{FullMethod: ateapipb.Control_GetGlobalAccessPolicy_FullMethodName}, func(c context.Context, r any) (any, error) { + _, err := restartedInterceptor(c, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}, &grpc.UnaryServerInfo{FullMethod: ateapipb.Control_GetGlobalAccessPolicy_FullMethodName}, func(c context.Context, r any) (any, error) { return svc.GetGlobalAccessPolicy(c, r.(*ateapipb.GetGlobalAccessPolicyRequest)) }) return err @@ -238,6 +239,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { getSpacePolReq := &ateapipb.GetAtespaceAccessPolicyRequest{ Atespace: &ateapipb.ObjectRef{Name: "team-alpha"}, + Name: resources.SingletonName, } if _, err := invoke(aliceCtx, ateapipb.Control_GetAtespaceAccessPolicy_FullMethodName, getSpacePolReq, func(c context.Context, r any) (any, error) { return svc.GetAtespaceAccessPolicy(c, r.(*ateapipb.GetAtespaceAccessPolicyRequest)) @@ -292,6 +294,7 @@ func TestAccessPolicy_GlobalAndAtespaceGovernance(t *testing.T) { } deleteSpacePolReq := &ateapipb.DeleteAtespaceAccessPolicyRequest{ Atespace: &ateapipb.ObjectRef{Name: "team-alpha"}, + Name: resources.SingletonName, } if _, err := invoke(charlieCtx, ateapipb.Control_DeleteAtespaceAccessPolicy_FullMethodName, deleteSpacePolReq, func(c context.Context, r any) (any, error) { return svc.DeleteAtespaceAccessPolicy(c, r.(*ateapipb.DeleteAtespaceAccessPolicyRequest)) diff --git a/cmd/ateapi/internal/controlapi/egress_policy_test.go b/cmd/ateapi/internal/controlapi/egress_policy_test.go index e454e04395..6e782a3337 100644 --- a/cmd/ateapi/internal/controlapi/egress_policy_test.go +++ b/cmd/ateapi/internal/controlapi/egress_policy_test.go @@ -46,11 +46,13 @@ func TestActorEgressPolicy(t *testing.T) { if _, err := service.GetActorEgressPolicy(t.Context(), &ateapipb.GetActorEgressPolicyRequest{ Actor: actorRef, + Name: resources.SingletonName, }); apierror.Code(err) != codes.NotFound { t.Fatalf("policy before create status = %v, want NotFound", apierror.Code(err)) } if _, err := service.GetActorEgressPolicy(t.Context(), &ateapipb.GetActorEgressPolicyRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "missing-actor"}, + Name: resources.SingletonName, }); apierror.Code(err) != codes.NotFound { t.Fatalf("missing parent status = %v, want NotFound", apierror.Code(err)) } @@ -93,7 +95,7 @@ func TestActorEgressPolicy(t *testing.T) { if md := created.GetMetadata(); md.GetName() != "default" || md.GetAtespace() != testAtespace || md.GetUid() == "" || md.GetVersion() != 1 || md.GetCreateTime() == nil || md.GetUpdateTime() == nil { t.Fatalf("created metadata = %v", md) } - got, err := service.GetActorEgressPolicy(t.Context(), &ateapipb.GetActorEgressPolicyRequest{Actor: actorRef}) + got, err := service.GetActorEgressPolicy(t.Context(), &ateapipb.GetActorEgressPolicyRequest{Actor: actorRef, Name: resources.SingletonName}) if err != nil || !proto.Equal(got, created) { t.Fatalf("policy after create = %v, %v; want %v", got, err, created) } @@ -138,12 +140,14 @@ func TestActorEgressPolicy(t *testing.T) { } deleted, err := service.DeleteActorEgressPolicy(t.Context(), &ateapipb.DeleteActorEgressPolicyRequest{ Actor: actorRef, + Name: resources.SingletonName, }) if err != nil || !proto.Equal(deleted, updated) { t.Fatalf("deleted policy = %v, %v; want %v", deleted, err, updated) } if _, err := service.GetActorEgressPolicy(t.Context(), &ateapipb.GetActorEgressPolicyRequest{ Actor: actorRef, + Name: resources.SingletonName, }); apierror.Code(err) != codes.NotFound { t.Fatalf("policy after delete status = %v, want NotFound", apierror.Code(err)) } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go b/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go index 3ea02410f0..70d15ed8db 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/access_policy_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "google.golang.org/grpc/codes" @@ -38,7 +39,7 @@ func TestGlobalAccessPolicy_CRUD(t *testing.T) { defer tc.cleanup() ctx := context.Background() - _, err := tc.client.GetGlobalAccessPolicy(ctx, &ateapipb.GetGlobalAccessPolicyRequest{}) + _, err := tc.client.GetGlobalAccessPolicy(ctx, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}) wantCode(t, "Get before create", err, codes.NotFound) created, err := tc.client.CreateGlobalAccessPolicy(ctx, &ateapipb.CreateGlobalAccessPolicyRequest{ @@ -62,7 +63,7 @@ func TestGlobalAccessPolicy_CRUD(t *testing.T) { }) wantCode(t, "second Create", err, codes.AlreadyExists) - got, err := tc.client.GetGlobalAccessPolicy(ctx, &ateapipb.GetGlobalAccessPolicyRequest{}) + got, err := tc.client.GetGlobalAccessPolicy(ctx, &ateapipb.GetGlobalAccessPolicyRequest{Name: resources.SingletonName}) if err != nil { t.Fatalf("Get failed: %v", err) } @@ -108,7 +109,7 @@ func TestAtespaceAccessPolicy_CRUD(t *testing.T) { }) wantCode(t, "Create on missing atespace", err, codes.FailedPrecondition) - _, err = tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref}) + _, err = tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) wantCode(t, "Get before create", err, codes.NotFound) created, err := tc.client.CreateAtespaceAccessPolicy(ctx, &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: ref, AccessPolicy: policy}) @@ -118,7 +119,7 @@ func TestAtespaceAccessPolicy_CRUD(t *testing.T) { _, err = tc.client.CreateAtespaceAccessPolicy(ctx, &ateapipb.CreateAtespaceAccessPolicyRequest{Atespace: ref, AccessPolicy: policy}) wantCode(t, "second Create", err, codes.AlreadyExists) - got, err := tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref}) + got, err := tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) if err != nil { t.Fatalf("Get failed: %v", err) } @@ -138,16 +139,17 @@ func TestAtespaceAccessPolicy_CRUD(t *testing.T) { _, err = tc.client.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{ Atespace: ref, + Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Version: created.GetMetadata().GetVersion()}, }) wantCode(t, "Delete with stale version", err, codes.Aborted) - if _, err := tc.client.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: ref}); err != nil { + if _, err := tc.client.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}); err != nil { t.Fatalf("Delete failed: %v", err) } - _, err = tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref}) + _, err = tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) wantCode(t, "Get after delete", err, codes.NotFound) - _, err = tc.client.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: ref}) + _, err = tc.client.DeleteAtespaceAccessPolicy(ctx, &ateapipb.DeleteAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) wantCode(t, "second Delete", err, codes.NotFound) // Deleting the atespace removes its policy with it. @@ -157,6 +159,6 @@ func TestAtespaceAccessPolicy_CRUD(t *testing.T) { if _, err := tc.client.DeleteAtespace(ctx, &ateapipb.DeleteAtespaceRequest{Atespace: ref}); err != nil { t.Fatalf("DeleteAtespace failed: %v", err) } - _, err = tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref}) + _, err = tc.client.GetAtespaceAccessPolicy(ctx, &ateapipb.GetAtespaceAccessPolicyRequest{Atespace: ref, Name: resources.SingletonName}) wantCode(t, "Get after DeleteAtespace", err, codes.NotFound) } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go b/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go index 2c77fada5a..0ac4692738 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/egress_policy_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "google.golang.org/grpc/codes" @@ -94,7 +95,7 @@ func TestCreateActorEgressPolicy_Success(t *testing.T) { t.Errorf("timestamps = (%v, %v), want server-generated values", md.GetCreateTime(), md.GetUpdateTime()) } - got, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + got, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) if err != nil { t.Fatalf("GetActorEgressPolicy failed: %v", err) } @@ -122,7 +123,7 @@ func TestCreateActorEgressPolicy_Errors(t *testing.T) { func TestGetActorEgressPolicy_NotFound(t *testing.T) { tc, actor := setupEgressPolicyActor(t, "ns-get-egress-policy-missing") - _, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + _, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) assertGrpcError(t, err, codes.NotFound, "EgressPolicy for actor "+actor.GetAtespace()+"/"+actor.GetName()+" not found") } @@ -152,7 +153,7 @@ func TestUpdateActorEgressPolicy(t *testing.T) { t.Errorf("updated rules = %v, want one tls_passthrough rule", updated.GetRules()) } - got, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + got, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) if err != nil { t.Fatalf("GetActorEgressPolicy failed: %v", err) } @@ -190,14 +191,14 @@ func TestDeleteActorEgressPolicy(t *testing.T) { tc, actor := setupEgressPolicyActor(t, "ns-delete-egress-policy") created := createEgressPolicy(t, tc, actor) - deleted, err := tc.client.DeleteActorEgressPolicy(context.Background(), &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor}) + deleted, err := tc.client.DeleteActorEgressPolicy(context.Background(), &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) if err != nil { t.Fatalf("DeleteActorEgressPolicy failed: %v", err) } if diff := cmp.Diff(created, deleted, protocmp.Transform()); diff != "" { t.Errorf("deleted policy mismatch (-created +deleted):\n%s", diff) } - _, err = tc.client.DeleteActorEgressPolicy(context.Background(), &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor}) + _, err = tc.client.DeleteActorEgressPolicy(context.Background(), &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) assertGrpcError(t, err, codes.NotFound, "EgressPolicy not found") } @@ -208,7 +209,7 @@ func TestDeleteActor_CascadesEgressPolicy(t *testing.T) { if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: actor}); err != nil { t.Fatalf("DeleteActor failed: %v", err) } - _, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + _, err := tc.client.GetActorEgressPolicy(context.Background(), &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) assertGrpcError(t, err, codes.NotFound, "EgressPolicy for actor "+actor.GetAtespace()+"/"+actor.GetName()+" not found") } @@ -230,7 +231,7 @@ func TestDeleteActorEgressPolicy_Preconditions(t *testing.T) { tc, actor := setupEgressPolicyActor(t, "ns-delete-egress-policy-preconditions") ctx := context.Background() del := func(opts *ateapipb.DeleteOptions) error { - _, err := tc.client.DeleteActorEgressPolicy(ctx, &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Options: opts}) + _, err := tc.client.DeleteActorEgressPolicy(ctx, &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName, Options: opts}) return err } @@ -241,7 +242,7 @@ func TestDeleteActorEgressPolicy_Preconditions(t *testing.T) { assertGrpcError(t, del(&ateapipb.DeleteOptions{Uid: uid, Version: version + 1}), codes.Aborted, "EgressPolicy version conflict") assertGrpcError(t, del(&ateapipb.DeleteOptions{Uid: foreignUID}), codes.Aborted, "EgressPolicy UID conflict") assertGrpcError(t, del(&ateapipb.DeleteOptions{Uid: foreignUID, Version: version}), codes.Aborted, "EgressPolicy UID conflict") - if _, err := tc.client.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}); err != nil { + if _, err := tc.client.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}); err != nil { t.Fatalf("a refused delete removed the policy: %v", err) } @@ -256,6 +257,6 @@ func TestDeleteActorEgressPolicy_Preconditions(t *testing.T) { if err := del(&ateapipb.DeleteOptions{Uid: policy.GetMetadata().GetUid(), Version: policy.GetMetadata().GetVersion()}); err != nil { t.Fatalf("DeleteActorEgressPolicy with both guards: %v", err) } - _, err := tc.client.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + _, err := tc.client.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) assertGrpcError(t, err, codes.NotFound, "EgressPolicy for actor "+actor.GetAtespace()+"/"+actor.GetName()+" not found") } diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 3ca10de2eb..1b1febf633 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -44,6 +44,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/egresspolicy" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -198,8 +199,11 @@ func (m *egressMockClient) GetActor(context.Context, *ateapipb.GetActorRequest, return m.actor, nil } -func (m *egressMockClient) GetActorEgressPolicy(ctx context.Context, _ *ateapipb.GetActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { +func (m *egressMockClient) GetActorEgressPolicy(ctx context.Context, req *ateapipb.GetActorEgressPolicyRequest, _ ...grpc.CallOption) (*ateapipb.EgressPolicy, error) { m.policyCalls.Add(1) + if req.GetName() != resources.SingletonName { + return nil, status.Errorf(codes.InvalidArgument, "name must be %q", resources.SingletonName) + } if m.policyGate != nil { select { case <-m.policyGate: diff --git a/cmd/atenet/internal/router/egress/policycache.go b/cmd/atenet/internal/router/egress/policycache.go index 1feb80f385..954d91a174 100644 --- a/cmd/atenet/internal/router/egress/policycache.go +++ b/cmd/atenet/internal/router/egress/policycache.go @@ -137,7 +137,7 @@ func (c *policyCache) cached(ref resources.ActorRef) (*egresspolicy.Policy, bool // fetch loads and compiles one actor's policy and stores the result. A nil // policy with a nil error is the stored form of "no policy". func (c *policyCache) fetch(ctx context.Context, ref resources.ActorRef) (*egresspolicy.Policy, error) { - resp, err := c.client.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: ref.ToObjectRef()}) + resp, err := c.client.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: ref.ToObjectRef(), Name: resources.SingletonName}) var policy *egresspolicy.Policy switch { case err == nil: diff --git a/cmd/kubectl-ate/internal/cmd/egress_policy.go b/cmd/kubectl-ate/internal/cmd/egress_policy.go index 019891ced4..9ecae7f0a2 100644 --- a/cmd/kubectl-ate/internal/cmd/egress_policy.go +++ b/cmd/kubectl-ate/internal/cmd/egress_policy.go @@ -23,6 +23,7 @@ import ( "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" "google.golang.org/grpc" @@ -129,7 +130,6 @@ func manifestToJSON(data []byte) ([]byte, error) { // metadata.name the manifest leaves empty, then checks that both match the // command line: the atespace flag and the fixed policy name. func overrideEgressPolicyMetadata(policy *ateapipb.EgressPolicy, atespace string) error { - const policyNameDefault = "default" if policy.Metadata == nil { policy.Metadata = &ateapipb.ResourceMetadata{} } @@ -137,13 +137,13 @@ func overrideEgressPolicyMetadata(policy *ateapipb.EgressPolicy, atespace string policy.Metadata.Atespace = atespace } if policy.Metadata.Name == "" { - policy.Metadata.Name = policyNameDefault + policy.Metadata.Name = resources.SingletonName } if policy.Metadata.Atespace != atespace { return fmt.Errorf("manifest metadata.atespace %q does not match --atespace %q", policy.Metadata.Atespace, atespace) } - if policy.Metadata.Name != policyNameDefault { - return fmt.Errorf("manifest metadata.name %q must be %q", policy.Metadata.Name, policyNameDefault) + if policy.Metadata.Name != resources.SingletonName { + return fmt.Errorf("manifest metadata.name %q must be %q", policy.Metadata.Name, resources.SingletonName) } return nil } @@ -198,7 +198,7 @@ type getEgressPolicyRunner struct { } func (r *getEgressPolicyRunner) Run(ctx context.Context) error { - policy, err := r.getter.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: r.actor}) + policy, err := r.getter.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: r.actor, Name: resources.SingletonName}) if status.Code(err) == codes.NotFound { if err := requireActor(ctx, r.getter, r.actor); err != nil { return err @@ -372,7 +372,7 @@ type deleteEgressPolicyRunner struct { } func (r *deleteEgressPolicyRunner) Run(ctx context.Context) error { - _, err := r.deleter.DeleteActorEgressPolicy(ctx, &ateapipb.DeleteActorEgressPolicyRequest{Actor: r.actor, Options: r.options}) + _, err := r.deleter.DeleteActorEgressPolicy(ctx, &ateapipb.DeleteActorEgressPolicyRequest{Actor: r.actor, Name: resources.SingletonName, Options: r.options}) if status.Code(err) == codes.NotFound { if err := requireActor(ctx, r.deleter, r.actor); err != nil { return err diff --git a/cmd/kubectl-ate/internal/cmd/egress_policy_test.go b/cmd/kubectl-ate/internal/cmd/egress_policy_test.go index eddef33aef..455689a2fe 100644 --- a/cmd/kubectl-ate/internal/cmd/egress_policy_test.go +++ b/cmd/kubectl-ate/internal/cmd/egress_policy_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "google.golang.org/grpc" @@ -510,7 +511,7 @@ func TestGetEgressPolicyRunner_Run(t *testing.T) { name: "table by default", outputFmt: "table", getter: &fakeEgressPolicyGetter{policy: policy}, - wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor}, + wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}, wantOut: `ATESPACE ACTOR RULES VERSION AGE team-a c1 1 1 5m `, @@ -519,7 +520,7 @@ team-a c1 1 1 5m name: "yaml", outputFmt: "yaml", getter: &fakeEgressPolicyGetter{policy: policy}, - wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor}, + wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}, wantOut: `metadata: atespace: team-a createTime: "2026-01-01T11:55:00Z" @@ -536,7 +537,7 @@ rules: name: "no policy on an existing actor writes a note and succeeds", outputFmt: "yaml", getter: &fakeEgressPolicyGetter{err: status.Error(codes.NotFound, "EgressPolicy not found")}, - wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor}, + wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}, wantActorReq: &ateapipb.GetActorRequest{Actor: actor}, wantErrOut: "actor \"c1\" in atespace \"team-a\" has no egress policy\n", }, @@ -544,7 +545,7 @@ rules: name: "missing actor fails", outputFmt: "yaml", getter: &fakeEgressPolicyGetter{err: status.Error(codes.NotFound, "EgressPolicy not found"), fakeActorReader: fakeActorReader{actorErr: status.Error(codes.NotFound, "Actor team-a/c1 not found")}}, - wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor}, + wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}, wantActorReq: &ateapipb.GetActorRequest{Actor: actor}, wantErr: `actor "c1" in atespace "team-a" not found`, }, @@ -552,7 +553,7 @@ rules: name: "actor lookup error wraps", outputFmt: "yaml", getter: &fakeEgressPolicyGetter{err: status.Error(codes.NotFound, "EgressPolicy not found"), fakeActorReader: fakeActorReader{actorErr: status.Error(codes.PermissionDenied, "denied")}}, - wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor}, + wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}, wantActorReq: &ateapipb.GetActorRequest{Actor: actor}, wantErr: `failed to get actor "c1" in atespace "team-a": rpc error: code = PermissionDenied desc = denied`, }, @@ -560,7 +561,7 @@ rules: name: "other error wraps", outputFmt: "table", getter: &fakeEgressPolicyGetter{err: status.Error(codes.Unavailable, "api-server down")}, - wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor}, + wantReq: &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}, wantErr: `failed to get egress policy for actor "c1" in atespace "team-a": rpc error: code = Unavailable desc = api-server down`, }, } @@ -966,7 +967,7 @@ func TestDeleteEgressPolicyRunner_Run(t *testing.T) { Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "default", Version: 1}, Rules: []*ateapipb.EgressRule{{Http: &ateapipb.HTTPRule{Hostnames: []string{"api.example.com"}}}}, } - unguarded := &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor} + unguarded := &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName} wantActorReq := &ateapipb.GetActorRequest{Actor: actor} policyNotFound := status.Error(codes.NotFound, "EgressPolicy not found") const uid = "9a2b1c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d" @@ -991,21 +992,21 @@ func TestDeleteEgressPolicyRunner_Run(t *testing.T) { name: "uid and version populate options", deleter: &fakeEgressPolicyDeleter{policy: deleted}, options: &ateapipb.DeleteOptions{Uid: uid, Version: 3}, - wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Options: &ateapipb.DeleteOptions{Uid: uid, Version: 3}}, + wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Uid: uid, Version: 3}}, wantOut: confirmed, }, { name: "uid alone", deleter: &fakeEgressPolicyDeleter{policy: deleted}, options: &ateapipb.DeleteOptions{Uid: uid}, - wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Options: &ateapipb.DeleteOptions{Uid: uid}}, + wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Uid: uid}}, wantOut: confirmed, }, { name: "version alone", deleter: &fakeEgressPolicyDeleter{policy: deleted}, options: &ateapipb.DeleteOptions{Version: 3}, - wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Options: &ateapipb.DeleteOptions{Version: 3}}, + wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Version: 3}}, wantOut: confirmed, }, { @@ -1019,7 +1020,7 @@ func TestDeleteEgressPolicyRunner_Run(t *testing.T) { name: "guarded delete not found still reads actor", deleter: &fakeEgressPolicyDeleter{err: policyNotFound}, options: &ateapipb.DeleteOptions{Uid: uid, Version: 3}, - wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Options: &ateapipb.DeleteOptions{Uid: uid, Version: 3}}, + wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Uid: uid, Version: 3}}, wantActorReq: wantActorReq, wantErr: `actor "c1" in atespace "team-a" has no egress policy`, }, @@ -1041,7 +1042,7 @@ func TestDeleteEgressPolicyRunner_Run(t *testing.T) { name: "aborted conflict wraps", deleter: &fakeEgressPolicyDeleter{err: status.Error(codes.Aborted, "EgressPolicy version conflict")}, options: &ateapipb.DeleteOptions{Version: 3}, - wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Options: &ateapipb.DeleteOptions{Version: 3}}, + wantReq: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName, Options: &ateapipb.DeleteOptions{Version: 3}}, wantErr: `failed to delete egress policy for actor "c1" in atespace "team-a": rpc error: code = Aborted desc = EgressPolicy version conflict`, }, { diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index 6bfb643ccf..c4af026579 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -139,8 +139,6 @@ Rules: - A subresource may be managed by the system, such as `ActorAssignment`. The system creates and deletes it as part of other operations, and the API exposes only List and, optionally, Get. - Subresource's methods follow section #3. Update replaces the whole object with `uid` and `version` preconditions, Delete takes `DeleteOptions`, and List is paginated. -TODO: The existing Get and Delete requests for singleton subresources do not carry `name` yet. They identify the subresource by its parent alone. - ### 2.5 Singletons A **singleton** is a resource type that allows at most one instance per parent. For example, each `Actor` has at most one `EgressPolicy`, and each `Atespace` has at most one `AccessPolicy`, as does Global. diff --git a/internal/e2e/egresspolicy.go b/internal/e2e/egresspolicy.go index 1e10a372a8..d250d83b41 100644 --- a/internal/e2e/egresspolicy.go +++ b/internal/e2e/egresspolicy.go @@ -21,6 +21,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -106,7 +107,7 @@ func replaceHeaderEffects(header, prefix, credentialURI string) *ateapipb.HttpRu func EnsureEgressPolicy(t *testing.T, ctx context.Context, clients *Clients, actor *ateapipb.ObjectRef, rules ...*ateapipb.EgressRule) { t.Helper() policy := &ateapipb.EgressPolicy{ - Metadata: &ateapipb.ResourceMetadata{Atespace: actor.GetAtespace(), Name: "default"}, + Metadata: &ateapipb.ResourceMetadata{Atespace: actor.GetAtespace(), Name: resources.SingletonName}, Rules: rules, } _, err := clients.SubstrateAPI.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{ @@ -119,7 +120,7 @@ func EnsureEgressPolicy(t *testing.T, ctx context.Context, clients *Clients, act } return } - existing, err := clients.SubstrateAPI.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}) + existing, err := clients.SubstrateAPI.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actor, Name: resources.SingletonName}) if err != nil { t.Fatalf("GetActorEgressPolicy for %s/%s: %v", actor.GetAtespace(), actor.GetName(), err) } diff --git a/internal/resources/singleton.go b/internal/resources/singleton.go new file mode 100644 index 0000000000..efcced564d --- /dev/null +++ b/internal/resources/singleton.go @@ -0,0 +1,19 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +// SingletonName is the name of every singleton subresource, such as an Actor's +// EgressPolicy. +const SingletonName = "default" diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 597cbfaef2..d92ef51c95 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -5388,7 +5388,13 @@ type GetActorEgressPolicyRequest struct { // // +k8s:required // +k8s:subfield(atespace)=+k8s:required - Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + // Name of the singleton egress policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5430,6 +5436,13 @@ func (x *GetActorEgressPolicyRequest) GetActor() *ObjectRef { return nil } +func (x *GetActorEgressPolicyRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + // CreateActorEgressPolicyRequest creates an egress policy for an Actor. // // +k8s:customValidation # atespaces must match @@ -5564,10 +5577,16 @@ type DeleteActorEgressPolicyRequest struct { // +k8s:required // +k8s:subfield(atespace)=+k8s:required Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + // Name of the singleton egress policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` // Optional per-delete preconditions. // // +k8s:optional - Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + Options *DeleteOptions `protobuf:"bytes,3,opt,name=options,proto3" json:"options,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5609,6 +5628,13 @@ func (x *DeleteActorEgressPolicyRequest) GetActor() *ObjectRef { return nil } +func (x *DeleteActorEgressPolicyRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + func (x *DeleteActorEgressPolicyRequest) GetOptions() *DeleteOptions { if x != nil { return x.Options @@ -8133,7 +8159,13 @@ func (x *Binding) GetMembers() []string { // GetGlobalAccessPolicyRequest retrieves the deployment-wide global access policy singleton. type GetGlobalAccessPolicyRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // Name of the singleton access policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8168,6 +8200,13 @@ func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{111} } +func (x *GetGlobalAccessPolicyRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + // CreateGlobalAccessPolicyRequest creates the deployment-wide global access policy singleton. type CreateGlobalAccessPolicyRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -8273,7 +8312,13 @@ type GetAtespaceAccessPolicyRequest struct { // // +k8s:required // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix - Atespace *ObjectRef `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Atespace *ObjectRef `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // Name of the singleton access policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8315,6 +8360,13 @@ func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { return nil } +func (x *GetAtespaceAccessPolicyRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + // CreateAtespaceAccessPolicyRequest creates an access policy for an Atespace. type CreateAtespaceAccessPolicyRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -8445,10 +8497,16 @@ type DeleteAtespaceAccessPolicyRequest struct { // +k8s:required // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix Atespace *ObjectRef `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + // Name of the singleton access policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` // Optional per-delete preconditions. // // +k8s:optional - Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + Options *DeleteOptions `protobuf:"bytes,3,opt,name=options,proto3" json:"options,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8490,6 +8548,13 @@ func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { return nil } +func (x *DeleteAtespaceAccessPolicyRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + func (x *DeleteAtespaceAccessPolicyRequest) GetOptions() *DeleteOptions { if x != nil { return x.Options @@ -8768,18 +8833,20 @@ const file_ateapi_proto_rawDesc = "" + "\x12DeleteActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tany_state\x18\x02 \x01(\bR\banyState\x12/\n" + - "\aoptions\x18\x03 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"F\n" + + "\aoptions\x18\x03 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"Z\n" + "\x1bGetActorEgressPolicyRequest\x12'\n" + - "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\"\x84\x01\n" + + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\"\x84\x01\n" + "\x1eCreateActorEgressPolicyRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x129\n" + "\regress_policy\x18\x02 \x01(\v2\x14.ateapi.EgressPolicyR\fegressPolicy\"\x84\x01\n" + "\x1eUpdateActorEgressPolicyRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x129\n" + - "\regress_policy\x18\x02 \x01(\v2\x14.ateapi.EgressPolicyR\fegressPolicy\"z\n" + + "\regress_policy\x18\x02 \x01(\v2\x14.ateapi.EgressPolicyR\fegressPolicy\"\x8e\x01\n" + "\x1eDeleteActorEgressPolicyRequest\x12'\n" + - "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12/\n" + - "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"4\n" + + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12/\n" + + "\aoptions\x18\x03 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"4\n" + "\rGetTagRequest\x12#\n" + "\x03tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x03tag\"\x8b\x01\n" + "\x13MintActorJWTRequest\x12'\n" + @@ -8916,23 +8983,26 @@ const file_ateapi_proto_rawDesc = "" + "\bbindings\x18\x02 \x03(\v2\x0f.ateapi.BindingR\bbindings\"7\n" + "\aBinding\x12\x12\n" + "\x04role\x18\x01 \x01(\tR\x04role\x12\x18\n" + - "\amembers\x18\x02 \x03(\tR\amembers\"\x1e\n" + - "\x1cGetGlobalAccessPolicyRequest\"\\\n" + + "\amembers\x18\x02 \x03(\tR\amembers\"2\n" + + "\x1cGetGlobalAccessPolicyRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"\\\n" + "\x1fCreateGlobalAccessPolicyRequest\x129\n" + "\raccess_policy\x18\x01 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\\\n" + "\x1fUpdateGlobalAccessPolicyRequest\x129\n" + - "\raccess_policy\x18\x01 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"O\n" + + "\raccess_policy\x18\x01 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"c\n" + "\x1eGetAtespaceAccessPolicyRequest\x12-\n" + - "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\"\x8d\x01\n" + + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\"\x8d\x01\n" + "!CreateAtespaceAccessPolicyRequest\x12-\n" + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x129\n" + "\raccess_policy\x18\x02 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\x8d\x01\n" + "!UpdateAtespaceAccessPolicyRequest\x12-\n" + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x129\n" + - "\raccess_policy\x18\x02 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\x83\x01\n" + + "\raccess_policy\x18\x02 \x01(\v2\x14.ateapi.AccessPolicyR\faccessPolicy\"\x97\x01\n" + "!DeleteAtespaceAccessPolicyRequest\x12-\n" + - "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x12/\n" + - "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*a\n" + + "\batespace\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12/\n" + + "\aoptions\x18\x03 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions*a\n" + "\rSnapshotOwner\x12\x1e\n" + "\x1aSNAPSHOT_OWNER_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14SNAPSHOT_OWNER_ACTOR\x10\x01\x12\x16\n" + diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 232bd44dcd..a0269940fb 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -1712,6 +1712,13 @@ message GetActorEgressPolicyRequest { // +k8s:required // +k8s:subfield(atespace)=+k8s:required ObjectRef actor = 1; + + // Name of the singleton egress policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + string name = 2; } // CreateActorEgressPolicyRequest creates an egress policy for an Actor. @@ -1756,10 +1763,17 @@ message DeleteActorEgressPolicyRequest { // +k8s:subfield(atespace)=+k8s:required ObjectRef actor = 1; + // Name of the singleton egress policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + string name = 2; + // Optional per-delete preconditions. // // +k8s:optional - DeleteOptions options = 2; + DeleteOptions options = 3; } message GetTagRequest { @@ -2543,7 +2557,14 @@ message Binding { } // GetGlobalAccessPolicyRequest retrieves the deployment-wide global access policy singleton. -message GetGlobalAccessPolicyRequest {} +message GetGlobalAccessPolicyRequest { + // Name of the singleton access policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + string name = 1; +} // CreateGlobalAccessPolicyRequest creates the deployment-wide global access policy singleton. message CreateGlobalAccessPolicyRequest { @@ -2570,6 +2591,13 @@ message GetAtespaceAccessPolicyRequest { // +k8s:required // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix ObjectRef atespace = 1; + + // Name of the singleton access policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + string name = 2; } // CreateAtespaceAccessPolicyRequest creates an access policy for an Atespace. @@ -2610,8 +2638,15 @@ message DeleteAtespaceAccessPolicyRequest { // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix ObjectRef atespace = 1; + // Name of the singleton access policy. Must be "default". + // + // +k8s:required + // +k8s:format=k8s-short-name + // +k8s:customValidation # must be "default" + string name = 2; + // Optional per-delete preconditions. // // +k8s:optional - DeleteOptions options = 2; + DeleteOptions options = 3; } From cfedc83dfb5e620d977f2cc15c324accba0e1d41 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Mon, 12 Oct 2026 17:29:27 +0000 Subject: [PATCH 02/20] Fix a couple issues with plural commands in kubectl-ate (#2509) 1) Make docs consistent for actor-templates wrt everything else. 2) Normalize all CLI help text on singular nouns with plurals as aliases 3) Don't accept plural commands where we can't handle more than 1 item #### Breaking change Some CLI usage of plurals breaks, but it wasn't DOING plural things. - [X] Tests pass - [X] Appropriate changes to documentation are included in the PR --- cmd/kubectl-ate/README.md | 8 +++---- cmd/kubectl-ate/internal/cmd/actor.go | 13 +++++------ cmd/kubectl-ate/internal/cmd/atespace.go | 4 ++-- cmd/kubectl-ate/internal/cmd/egress_policy.go | 19 +++++++-------- cmd/kubectl-ate/internal/cmd/tag.go | 23 ++++++++----------- cmd/kubectl-ate/internal/cmd/worker.go | 8 +++---- docs/observability.md | 12 +++++----- 7 files changed, 40 insertions(+), 47 deletions(-) diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index f021248030..e9ab9dbb8a 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -290,18 +290,18 @@ kubectl ate delete egress-policy -a --uid --versio ```bash # Print the logs an actor has produced on its current worker. # -a/--atespace is required, since an actor is addressed by (atespace, name). -kubectl ate logs actors my-actor -a +kubectl ate logs actor my-actor -a # Follow the logs with -f. The stream is aggregated across worker # reassignments, so the same actor stays queryable as it teleports between pods. -kubectl ate logs actors my-actor -a -f +kubectl ate logs actor my-actor -a -f # Show only one container's logs with -c/--container. -kubectl ate logs actors my-actor -a -c my-container +kubectl ate logs actor my-actor -a -c my-container # Pick a class of lines with --source: containers (every container's output, # no lifecycle events) or lifecycle (only the actor's lifecycle events). -kubectl ate logs actors my-actor -a --source=lifecycle +kubectl ate logs actor my-actor -a --source=lifecycle ``` Logs are streamable only while the actor is bound to a worker (i.e., `ACTOR_STATE_RUNNING`). For history across worker migrations, route through a centralized log backend (Cloud Logging, Loki, etc.); see `docs/observability.md`. diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index ab993b6638..5cb3bacebf 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -59,8 +59,8 @@ var ( ) var getActorsCmd = &cobra.Command{ - Use: "actors ", - Aliases: []string{"actor"}, + Use: "actor ", + Aliases: []string{"actors"}, Short: "List all actors or get one or more actors", RunE: func(cmd *cobra.Command, args []string) error { if len(args) > 0 { @@ -254,11 +254,10 @@ var suspendActorCmd = &cobra.Command{ } var logsActorsCmd = &cobra.Command{ - Use: "actors ", - Aliases: []string{"actor"}, - Short: "Stream logs for a specific actor", - Args: cobra.ExactArgs(1), - RunE: runLogsActor, + Use: "actor ", + Short: "Stream logs for a specific actor", + Args: cobra.ExactArgs(1), + RunE: runLogsActor, } func buildCreateActorRequest(actorName, atespace, template, tag string) (*ateapipb.CreateActorRequest, error) { diff --git a/cmd/kubectl-ate/internal/cmd/atespace.go b/cmd/kubectl-ate/internal/cmd/atespace.go index 741af23155..37cedad0ff 100644 --- a/cmd/kubectl-ate/internal/cmd/atespace.go +++ b/cmd/kubectl-ate/internal/cmd/atespace.go @@ -24,8 +24,8 @@ import ( ) var getAtespacesCmd = &cobra.Command{ - Use: "atespaces [name ...]", - Aliases: []string{"atespace"}, + Use: "atespace [name ...]", + Aliases: []string{"atespaces"}, Short: "List all atespaces or get one or more atespaces", RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() diff --git a/cmd/kubectl-ate/internal/cmd/egress_policy.go b/cmd/kubectl-ate/internal/cmd/egress_policy.go index 9ecae7f0a2..0657125407 100644 --- a/cmd/kubectl-ate/internal/cmd/egress_policy.go +++ b/cmd/kubectl-ate/internal/cmd/egress_policy.go @@ -51,9 +51,8 @@ var getEgressPolicyCmd = &cobra.Command{ } var createEgressPolicyCmd = &cobra.Command{ - Use: "egress-policy -f ", - Aliases: []string{"egress-policies"}, - Short: "Create an actor's egress policy from a manifest", + Use: "egress-policy -f ", + Short: "Create an actor's egress policy from a manifest", Long: `Create the egress policy of an actor from a manifest file. The manifest is a YAML or JSON EgressPolicy, as printed by @@ -64,9 +63,8 @@ Its metadata may be omitted.`, } var updateEgressPolicyCmd = &cobra.Command{ - Use: "egress-policy -f ", - Aliases: []string{"egress-policies"}, - Short: "Replace an actor's egress policy from a manifest", + Use: "egress-policy -f ", + Short: "Replace an actor's egress policy from a manifest", Long: `Replace the egress policy of an actor from a manifest file. The manifest is a YAML or JSON EgressPolicy, as printed by @@ -78,11 +76,10 @@ rejects the update when they no longer match those of the stored egress policy.` } var deleteEgressPolicyCmd = &cobra.Command{ - Use: "egress-policy ", - Aliases: []string{"egress-policies"}, - Short: "Delete the egress policy of an actor", - Args: cobra.ExactArgs(1), - RunE: runDeleteEgressPolicy, + Use: "egress-policy ", + Short: "Delete the egress policy of an actor", + Args: cobra.ExactArgs(1), + RunE: runDeleteEgressPolicy, } // egressPolicyFromManifest parses a single protojson-shaped YAML or JSON diff --git a/cmd/kubectl-ate/internal/cmd/tag.go b/cmd/kubectl-ate/internal/cmd/tag.go index e687346448..00270a0fc9 100644 --- a/cmd/kubectl-ate/internal/cmd/tag.go +++ b/cmd/kubectl-ate/internal/cmd/tag.go @@ -38,8 +38,8 @@ var ( ) var getTagsCmd = &cobra.Command{ - Use: "tags [tag-name ...]", - Aliases: []string{"tag"}, + Use: "tag [tag-name ...]", + Aliases: []string{"tags"}, Short: "List or get tags", RunE: func(cmd *cobra.Command, args []string) error { if getTagAllAtespacesFlag && getTagAtespaceFlag != "" { @@ -94,9 +94,8 @@ var getTagsCmd = &cobra.Command{ } var createTagCmd = &cobra.Command{ - Use: "tag ", - Aliases: []string{"tags"}, - Short: "Tag the external snapshot a suspended actor holds", + Use: "tag ", + Short: "Tag the external snapshot a suspended actor holds", Long: "Tag the external snapshot a suspended actor holds.\n\n" + "The tag gets its own copy of that snapshot, so suspending the actor\n" + "again or deleting it cannot collect what the tag names.", @@ -128,10 +127,9 @@ var createTagCmd = &cobra.Command{ } var updateTagCmd = &cobra.Command{ - Use: "tag ", - Aliases: []string{"tags"}, - Short: "Publish or unpublish a tag", - Args: cobra.ExactArgs(1), + Use: "tag ", + Short: "Publish or unpublish a tag", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { scope, err := parseTagScope(updateTagScopeFlag) if err != nil { @@ -154,10 +152,9 @@ var updateTagCmd = &cobra.Command{ } var deleteTagCmd = &cobra.Command{ - Use: "tag ", - Aliases: []string{"tags"}, - Short: "Delete a tag", - Args: cobra.ExactArgs(1), + Use: "tag ", + Short: "Delete a tag", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() client, err := ateclient.NewClient(ctx, kubeconfig, k8sContext, endpoint, tokenFile, traceEnabled) diff --git a/cmd/kubectl-ate/internal/cmd/worker.go b/cmd/kubectl-ate/internal/cmd/worker.go index 55e281f6cd..14798778ae 100644 --- a/cmd/kubectl-ate/internal/cmd/worker.go +++ b/cmd/kubectl-ate/internal/cmd/worker.go @@ -44,15 +44,15 @@ var ( ) var getWorkersCmd = &cobra.Command{ - Use: "workers ", - Aliases: []string{"worker"}, + Use: "worker ", + Aliases: []string{"workers"}, Short: "List all workers or get one or more workers", RunE: runGetWorkers, } var topWorkersCmd = &cobra.Command{ - Use: "workers", - Aliases: []string{"worker"}, + Use: "worker", + Aliases: []string{"workers"}, Short: "Display resource (CPU/Memory) usage of workers", Args: cobra.NoArgs, RunE: runTopWorkers, diff --git a/docs/observability.md b/docs/observability.md index a69839434b..042b8007ba 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -28,7 +28,7 @@ Agent Substrate captures container standard output/error, wraps them into struct For quick, on-demand debugging of an active actor, use the Agent Substrate CLI: ```bash -kubectl ate logs actors --atespace [--follow / -f] +kubectl ate logs actor --atespace [--follow / -f] ``` `--atespace` (short form `-a`) is required: actor names are only unique within @@ -40,7 +40,7 @@ an atespace, so an actor is always addressed by `(atespace, name)`. If an actor is suspended or not assigned to a worker pod, the CLI informs you immediately: ```bash -$ kubectl ate logs actors test -a demo +$ kubectl ate logs actor test -a demo Error: actor test is not currently running on any worker pod ``` @@ -48,7 +48,7 @@ Error: actor test is not currently running on any worker pod When an active actor is assigned to a worker pod, the CLI outputs clean, uniform JSON lines stripped of Substrate metadata, perfectly matching standard `kubectl logs` behavior: ```bash -$ kubectl ate logs actors test -a demo +$ kubectl ate logs actor test -a demo {"time":"2026-05-22T21:49:15.23700774Z","message":"Actor started"} {"time":"2026-05-22T21:49:15.23700774Z","level":"INFO","msg":"Starting counter server on port 80"} {"time":"2026-05-22T21:49:15.255765354Z","count":0,"fshash":"mCY7G4S318ztOUojPTF2NA/W+ZSmWyr+T5K3udFuP50","level":"INFO","msg":"Count"} @@ -59,7 +59,7 @@ $ kubectl ate logs actors test -a demo To stream actor logs in real-time, append the `--follow` (or `-f`) flag. The CLI is fully actor-aware, automatically resuming the stream if the actor is suspended or migrates to a different worker pod: ```bash -$ kubectl ate logs actors test -a demo -f +$ kubectl ate logs actor test -a demo -f Actor is currently running on pod ate-demo-counter/counter-d8f99-m7d96 {"time":"2026-05-22T21:49:15.255765354Z","count":0,"fshash":"mCY7...","level":"INFO","msg":"Count"} {"time":"2026-05-22T21:49:25.263744806Z","count":1,"fshash":"mCY7...","level":"INFO","msg":"Count"} @@ -71,13 +71,13 @@ Actor is currently running on pod ate-demo-counter/counter-ab123-x4y5z An actor can run several containers. By default every line is shown, including the synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`). `--container` (short form `-c`) restricts the output to the named container's logs: ```bash -kubectl ate logs actors -a -c +kubectl ate logs actor -a -c ``` `--source` selects a class of lines instead of a container. `containers` shows every container's output with the lifecycle events removed; `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): ```bash -$ kubectl ate logs actors test -a demo --source=lifecycle +$ kubectl ate logs actor test -a demo --source=lifecycle {"time":"2026-10-07T21:31:48.648599369Z","message":"Actor checkpointing","span_id":"c475...","trace_flags":"01","trace_id":"e617..."} {"time":"2026-10-07T21:31:48.791435032Z","message":"Actor checkpointed","span_id":"c475...","trace_flags":"01","trace_id":"e617..."} {"time":"2026-10-07T21:31:59.772566601Z","message":"Actor restoring","span_id":"a68c...","trace_flags":"01","trace_id":"b387..."} From 73882a089088112cab2450ee6eb4d3583287b1bc Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Mon, 12 Oct 2026 18:02:33 +0000 Subject: [PATCH 03/20] e2e: skip the egress credential injection test (#2518) #2513 - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- internal/e2e/suites/egresscredinject/egresscredinject_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/e2e/suites/egresscredinject/egresscredinject_test.go b/internal/e2e/suites/egresscredinject/egresscredinject_test.go index 23ad344edc..d15d84a455 100644 --- a/internal/e2e/suites/egresscredinject/egresscredinject_test.go +++ b/internal/e2e/suites/egresscredinject/egresscredinject_test.go @@ -92,6 +92,8 @@ var withPlaceholder = []string{"header=" + url.QueryEscape("Authorization:"+plac // hack/install-ate-kind.sh --deploy-atenet --credential-provider='{"name":"k8s.io"}' // hack/run-e2e-kind.sh ./internal/e2e/suites/egresscredinject -v -args --no-color func TestActorEgressCredentialInjection(t *testing.T) { + t.Skip("disabled until the echo origin is in-cluster: httpbin.org outages fail CI for every PR (issue 2513)") + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") if err != nil { t.Fatalf("CheckEnv failed: %v", err) From 00593ef013b5c88bd37008d2a488ac957e3d2c93 Mon Sep 17 00:00:00 2001 From: Chenyi Wang Date: Mon, 12 Oct 2026 18:13:54 +0000 Subject: [PATCH 04/20] atecontroller: add --worker-sync-concurrency flag (#2334) ### Summary Expose the `WorkerPoolSyncer` worker goroutine count via `--worker-sync-concurrency` (default `2`, preserving existing behavior) instead of a hardcoded constant, so large-scale benchmarks can raise worker-registration concurrency without patching Go sources at runtime. ### Testing `go test ./cmd/atecontroller/...` - [x] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/atecontroller/internal/workersync/syncer.go | 15 +++++++-------- .../internal/workersync/syncer_test.go | 4 ++-- cmd/atecontroller/main.go | 9 ++++++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 62cf8c144d..f60291ab1d 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -37,11 +37,6 @@ import ( "k8s.io/client-go/util/workqueue" ) -// syncerWorkerCount is the number of goroutines draining the work queue. The -// queue never hands the same key to two workers concurrently, so per-key -// ordering is preserved. -const syncerWorkerCount = 2 - // workerPodLabel names the WorkerPool a worker pod belongs to. Its presence is // also what marks a pod as a worker pod at all, so it doubles as the selector // the pod informer is narrowed by. @@ -96,13 +91,16 @@ func (k workerKey) logAttrs() []any { // // Informer event handlers only enqueue keys; worker goroutines reconcile each // key against the current informer cache state, requeuing with rate-limited -// backoff on transient failures such as a lost version precondition. +// backoff on transient failures such as a lost version precondition. The queue +// never hands the same key to two workers concurrently, so per-key ordering is +// preserved regardless of workerCount. type WorkerPoolSyncer struct { client ateapipb.ControlClient pods corev1client.PodsGetter workerInformer cache.SharedIndexInformer workerPoolInformer cache.SharedIndexInformer queue workqueue.TypedRateLimitingInterface[workerKey] + workerCount int // Exponential backoff schedule for retrying a failed page of the startup // registered-worker scan. Per-syncer rather than package-level so a test @@ -113,13 +111,14 @@ type WorkerPoolSyncer struct { // NewWorkerPoolSyncer creates a new WorkerPoolSyncer. pods is used to delete // worker pods that have reached a terminal phase. -func NewWorkerPoolSyncer(client ateapipb.ControlClient, pods corev1client.PodsGetter, workerInformer, workerPoolInformer cache.SharedIndexInformer) *WorkerPoolSyncer { +func NewWorkerPoolSyncer(client ateapipb.ControlClient, pods corev1client.PodsGetter, workerInformer, workerPoolInformer cache.SharedIndexInformer, workerCount int) *WorkerPoolSyncer { return &WorkerPoolSyncer{ client: client, pods: pods, workerInformer: workerInformer, workerPoolInformer: workerPoolInformer, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[workerKey]()), + workerCount: workerCount, listBackoff: defaultListBackoff, listCap: defaultListCap, } @@ -176,7 +175,7 @@ func (s *WorkerPoolSyncer) Start(ctx context.Context) { slog.ErrorContext(ctx, "Syncer: failed to sync informer cache") return } - for range syncerWorkerCount { + for range s.workerCount { go wait.UntilWithContext(ctx, s.runWorker, time.Second) } diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index 642e2e7eeb..d8a1f77788 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -130,7 +130,7 @@ func setupSyncerTest(t *testing.T, ctx context.Context, api *fakeControl, initPo // Start before the factory: the informer's initial list is what seeds the // queue with the pods that already exist. - NewWorkerPoolSyncer(api, fakeK8s.CoreV1(), workerInformer, workerPoolInformer).Start(ctx) + NewWorkerPoolSyncer(api, fakeK8s.CoreV1(), workerInformer, workerPoolInformer, 2).Start(ctx) workerFactory.Start(ctx.Done()) workerFactory.WaitForCacheSync(ctx.Done()) @@ -149,7 +149,7 @@ func setupReconcileTest(t *testing.T, api *fakeControl, initPools ...*atev1alpha _, workerInformer := WorkerPodInformer(fakeK8s) workerPoolInformer, poolIndexer := newWorkerPoolInformer(t, initPools...) - return NewWorkerPoolSyncer(api, fakeK8s.CoreV1(), workerInformer, workerPoolInformer), workerInformer.GetIndexer(), poolIndexer + return NewWorkerPoolSyncer(api, fakeK8s.CoreV1(), workerInformer, workerPoolInformer, 2), workerInformer.GetIndexer(), poolIndexer } // seedPod puts a pod in the syncer's cache as though the informer had delivered diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index f624bbec90..f9a4bbd298 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -82,6 +82,8 @@ var ( ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required.") + + workerSyncConcurrency = pflag.Int("worker-sync-concurrency", 2, "Number of concurrent worker goroutines reconciling worker pods into the Worker registry. Defaults to 2.") ) func init() { @@ -111,6 +113,11 @@ func main() { slog.InfoContext(ctx, "atecontroller starting", slog.String("version", version.Version)) ctrl.SetLogger(newControllerRuntimeLogger(slog.Default().Handler())) + if *workerSyncConcurrency <= 0 { + setupLog.Error(nil, "invalid flag", "flag", "--worker-sync-concurrency", "reason", "must be positive", "value", *workerSyncConcurrency) + os.Exit(1) + } + // Both providers must be registered before the ateapi client below: // otelgrpc.NewClientHandler captures the global tracer and meter providers at // construction, so a later init leaves it bound to the no-op ones. @@ -265,7 +272,7 @@ func main() { // Start registers the informer event handlers, so it has to run before the // factory does: the initial list then synthesizes an Add for every pod that // already exists, and no explicit startup re-list is needed. - workersync.NewWorkerPoolSyncer(ateapiClient, k8sClient.CoreV1(), workerPodInformer, workerPoolInformer.Informer()).Start(runCtx) + workersync.NewWorkerPoolSyncer(ateapiClient, k8sClient.CoreV1(), workerPodInformer, workerPoolInformer.Informer(), *workerSyncConcurrency).Start(runCtx) workerPodInformerFactory.Start(runCtx.Done()) ateFactory.Start(runCtx.Done()) From 33388934d3d45bc744b521aa26a2c90755cae89e Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Mon, 12 Oct 2026 18:47:44 +0000 Subject: [PATCH 05/20] ate-api-server no longer links the GCS and S3 SDKs (#2398) Fixes #2373 ate-api-server no longer links the GCS and S3 SDKs. The plugin socket helpers move to `internal/pluginsocket`, and `hack/verify/ateapi-deps.sh` guards it. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --------- Co-authored-by: Haven Xia --- cmd/ateapi/internal/controlapi/workflow.go | 6 +-- cmd/ateapi/main.go | 6 +-- cmd/atelet/main.go | 10 ++-- cmd/atelet/snapshot_transfer.go | 6 +-- cmd/atelet/snapshot_transfer_test.go | 7 +-- cmd/snapshot-plugin/main.go | 7 +-- hack/verify/ateapi-deps.sh | 50 +++++++++++++++++++ internal/objectstoreplugin/plugin_test.go | 21 ++------ .../calls_test.go | 2 +- .../pluginsocket.go} | 15 +++--- .../pluginsocket_test.go} | 20 +++++++- 11 files changed, 104 insertions(+), 46 deletions(-) create mode 100755 hack/verify/ateapi-deps.sh rename internal/{objectstoreplugin => pluginsocket}/calls_test.go (98%) rename internal/{objectstoreplugin/socket.go => pluginsocket/pluginsocket.go} (90%) rename internal/{objectstoreplugin/socket_test.go => pluginsocket/pluginsocket_test.go} (92%) diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index c15605e867..dbcb0bdd78 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -26,7 +26,7 @@ import ( "github.com/agent-substrate/substrate/internal/actorevent" "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/ateattr" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/internal/resources" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -147,14 +147,14 @@ func NewActorWorkflow( // snapshot plugin. func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) - return objectstoreplugin.CallError(err) + return pluginsocket.CallError(err) } // copySnapshot copies every object under src to dst through the control // snapshot plugin. func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) - return objectstoreplugin.CallError(err) + return pluginsocket.CallError(err) } // actorWorkflowStore enumerates the exact storage methods needed by diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index e58e23cc36..41a1687b5c 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -39,8 +39,8 @@ import ( "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/oidcdiscovery" + "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/internal/volume" @@ -259,7 +259,7 @@ func main() { serverboot.Fatal(ctx, "Failed to create metric instruments", err) } - snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket, objectstoreplugin.ReadyWait) + snapshotPluginConn, err := pluginsocket.Dial(*snapshotPluginSocket, pluginsocket.ReadyWait) if err != nil { serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) } @@ -267,7 +267,7 @@ func main() { // A plugin that never serves would fail every call. Fail at startup // instead, giving the sidecar time to come up. readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) - err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) + err = pluginsocket.WaitReady(readyCtx, snapshotPluginConn) cancelReady() if err != nil { serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 09ab84adff..9d05477df1 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -43,9 +43,9 @@ import ( "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/nodepath" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/otlprelay" + "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -283,7 +283,7 @@ func main() { go trustBundles.Informer().Run(stopCh) cache.WaitForCacheSync(stopCh, trustBundles.Informer().HasSynced) - snapshotPluginConn, err := objectstoreplugin.Dial(*snapshotPluginSocket, objectstoreplugin.ReadyWait) + snapshotPluginConn, err := pluginsocket.Dial(*snapshotPluginSocket, pluginsocket.ReadyWait) if err != nil { serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) } @@ -291,7 +291,7 @@ func main() { // A plugin that never serves would fail every call. Fail at startup // instead, giving the sidecar time to come up. readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) - err = objectstoreplugin.WaitReady(readyCtx, snapshotPluginConn) + err = pluginsocket.WaitReady(readyCtx, snapshotPluginConn) cancelReady() if err != nil { serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) @@ -874,7 +874,7 @@ func shouldHaveSnapshots(req *ateletpb.CheckpointRequest) bool { // uploadExternalCheckpoint uploads the checkpoint CheckpointWorkload just // took. Its plugin calls pass grpc.WaitForReady, so they wait out a plugin -// outage for as long as ctx allows rather than only objectstoreplugin's ready +// outage for as long as ctx allows rather than only pluginsocket's ready // wait, and their errors are not reported as retryable: see Checkpoint. func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error { uri, err := resources.ParseSnapshotURI(req.GetExternalConfig().GetSnapshotUri()) @@ -1013,7 +1013,7 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp // The local snapshot stays until the upload succeeds, so a retry can // upload it again. - return rec.SandboxClass, objectstoreplugin.CallError(s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName())) + return rec.SandboxClass, pluginsocket.CallError(s.uploadSnapshot(ctx, uri, localDir, rec, req.GetActorTemplateAtespace(), req.GetActorTemplateName())) } func readSnapshotManifest(dir string) ([]byte, error) { diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index 485490d1b5..57a4924e3a 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -20,7 +20,7 @@ import ( "os" "path/filepath" - "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/pluginsocket" objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" ) @@ -39,12 +39,12 @@ func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDi WritePath: dstDir, Files: files, }) - return objectstoreplugin.CallError(err) + return pluginsocket.CallError(err) } // uploadSnapshotFiles uploads the named files in srcDir to the snapshot, // passing opts to the plugin call. It returns the plugin's error as is: an -// upload that can be retried reports it through objectstoreplugin.CallError, +// upload that can be retried reports it through pluginsocket.CallError, // one that cannot does not. func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { if len(files) == 0 { diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 82d7c3ca94..4dd11c5a17 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -32,6 +32,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/objectstoreplugin" "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" + "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/objectstorage" objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" @@ -252,7 +253,7 @@ func newPluginSidecar(t *testing.T, store objectstorage.ObjectStorage) *pluginSi // start may run on any goroutine, so it reports failures with Error. func (s *pluginSidecar) start() { - lis, err := objectstoreplugin.Listen(s.path) + lis, err := pluginsocket.Listen(s.path) if err != nil { s.t.Errorf("Listen: %v", err) return @@ -286,14 +287,14 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { store := &orderedObjectStorage{} sidecar := newPluginSidecar(t, store) sidecar.start() - conn, err := objectstoreplugin.Dial(sidecar.path, wait) + conn, err := pluginsocket.Dial(sidecar.path, wait) if err != nil { t.Fatal(err) } defer conn.Close() ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - if err := objectstoreplugin.WaitReady(ctx, conn); err != nil { + if err := pluginsocket.WaitReady(ctx, conn); err != nil { t.Fatalf("WaitReady = %v", err) } s := &AteomHerder{ diff --git a/cmd/snapshot-plugin/main.go b/cmd/snapshot-plugin/main.go index 1d95e9e394..f2a2e7d1c1 100644 --- a/cmd/snapshot-plugin/main.go +++ b/cmd/snapshot-plugin/main.go @@ -33,6 +33,7 @@ import ( "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/objectstore" "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/pkg/objectstorage" objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" @@ -105,7 +106,7 @@ func main() { } healthpb.RegisterHealthServer(srv, health.NewServer()) - lis, err := objectstoreplugin.Listen(*socket) + lis, err := pluginsocket.Listen(*socket) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } @@ -149,12 +150,12 @@ func newObjectStore(ctx context.Context) (objectstore.Store, error) { // healthcheck dials the plugin at socket and waits up to timeout for it to // report that it is serving. func healthcheck(ctx context.Context, socket string, timeout time.Duration) error { - conn, err := objectstoreplugin.Dial(socket, objectstoreplugin.ReadyWait) + conn, err := pluginsocket.Dial(socket, pluginsocket.ReadyWait) if err != nil { return err } defer conn.Close() ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - return objectstoreplugin.WaitReady(ctx, conn) + return pluginsocket.WaitReady(ctx, conn) } diff --git a/hack/verify/ateapi-deps.sh b/hack/verify/ateapi-deps.sh new file mode 100755 index 0000000000..d41874be5d --- /dev/null +++ b/hack/verify/ateapi-deps.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ate-api-server moves snapshots through the snapshot plugin sidecar, so it +# must not link the object-store SDKs. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +banned=( + cloud.google.com/go/storage + github.com/aws/aws-sdk-go-v2/service/s3 +) + +# ko ships ate-api-server for every platform in .ko.yaml. +platforms=() +while IFS= read -r platform; do + platforms+=("${platform}") +done < <(awk '/^defaultPlatforms:/ {in_list = 1; next} in_list && $1 == "-" {print $2; next} {in_list = 0}' .ko.yaml) +if [[ ${#platforms[@]} -eq 0 ]]; then + echo "no defaultPlatforms in .ko.yaml." >&2 + exit 1 +fi + +status=0 +for platform in "${platforms[@]}"; do + deps="$(GOOS="${platform%/*}" GOARCH="${platform#*/}" CGO_ENABLED=0 go list -deps ./cmd/ateapi)" + for pkg in "${banned[@]}"; do + if grep -Fxq -- "${pkg}" <<<"${deps}"; then + echo "cmd/ateapi (${platform}) must not depend on ${pkg}." >&2 + status=1 + fi + done +done +exit "${status}" diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go index eafac6f033..88eb1389d3 100644 --- a/internal/objectstoreplugin/plugin_test.go +++ b/internal/objectstoreplugin/plugin_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/pkg/objectstorage" objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" @@ -119,7 +120,7 @@ func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(sockDir) }) - lis, err := Listen(filepath.Join(sockDir, "plugin.sock")) + lis, err := pluginsocket.Listen(filepath.Join(sockDir, "plugin.sock")) if err != nil { t.Fatal(err) } @@ -134,7 +135,7 @@ func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { go srv.Serve(lis) t.Cleanup(srv.Stop) - conn, err := Dial(filepath.Join(sockDir, "plugin.sock"), ReadyWait) + conn, err := pluginsocket.Dial(filepath.Join(sockDir, "plugin.sock"), pluginsocket.ReadyWait) if err != nil { t.Fatal(err) } @@ -145,23 +146,9 @@ func serve(t *testing.T, backend *memObjects, root string) *grpc.ClientConn { func TestWaitReady(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - if err := WaitReady(ctx, serve(t, newMemObjects(), t.TempDir())); err != nil { + if err := pluginsocket.WaitReady(ctx, serve(t, newMemObjects(), t.TempDir())); err != nil { t.Errorf("WaitReady against a serving plugin = %v", err) } - - // A socket nobody listens on must fail when ctx ends, not hang: WaitReady - // waits for the plugin as long as ctx allows. - conn, err := Dial(filepath.Join(t.TempDir(), "missing.sock"), ReadyWait) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - short, cancelShort := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancelShort() - err = WaitReady(short, conn) - if status.Code(err) != codes.DeadlineExceeded { - t.Errorf("WaitReady against a missing socket = %v, want %s", err, codes.DeadlineExceeded) - } } func TestUploadFetchRoundTrip(t *testing.T) { diff --git a/internal/objectstoreplugin/calls_test.go b/internal/pluginsocket/calls_test.go similarity index 98% rename from internal/objectstoreplugin/calls_test.go rename to internal/pluginsocket/calls_test.go index dd272f9919..d9ae9cc717 100644 --- a/internal/objectstoreplugin/calls_test.go +++ b/internal/pluginsocket/calls_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package objectstoreplugin +package pluginsocket import ( "errors" diff --git a/internal/objectstoreplugin/socket.go b/internal/pluginsocket/pluginsocket.go similarity index 90% rename from internal/objectstoreplugin/socket.go rename to internal/pluginsocket/pluginsocket.go index a5881aea04..e6fedc41da 100644 --- a/internal/objectstoreplugin/socket.go +++ b/internal/pluginsocket/pluginsocket.go @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package objectstoreplugin +// Package pluginsocket connects a server to its plugin sidecar over a Unix +// socket. It depends on no plugin implementation, so a server that only calls +// a plugin does not link the plugin's storage SDKs. +package pluginsocket import ( "context" @@ -97,7 +100,7 @@ func Dial(path string, readyWait time.Duration) (*grpc.ClientConn, error) { grpc.WithUnaryInterceptor(boundedWaitForReady(readyWait)), ) if err != nil { - return nil, fmt.Errorf("while dialing snapshot plugin at %s: %w", path, err) + return nil, fmt.Errorf("while dialing plugin at %s: %w", path, err) } return conn, nil } @@ -136,7 +139,7 @@ func awaitReady(ctx context.Context, cc *grpc.ClientConn, wait time.Duration) er if ctx.Err() != nil { return status.FromContextError(ctx.Err()).Err() } - return status.Errorf(codes.Unavailable, "snapshot plugin at %s is not ready after %s (connection %s)", cc.Target(), wait, state) + return status.Errorf(codes.Unavailable, "plugin at %s is not ready after %s (connection %s)", cc.Target(), wait, state) } } } @@ -148,10 +151,10 @@ func awaitReady(ctx context.Context, cc *grpc.ClientConn, wait time.Duration) er func WaitReady(ctx context.Context, conn *grpc.ClientConn) error { resp, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}, grpc.WaitForReady(true)) if err != nil { - return fmt.Errorf("while waiting for the snapshot plugin at %s: %w", conn.Target(), err) + return fmt.Errorf("while waiting for the plugin at %s: %w", conn.Target(), err) } if resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { - return fmt.Errorf("snapshot plugin at %s is %s", conn.Target(), resp.GetStatus()) + return fmt.Errorf("plugin at %s is %s", conn.Target(), resp.GetStatus()) } return nil } @@ -162,7 +165,7 @@ func WaitReady(ctx context.Context, conn *grpc.ClientConn) error { // rather than treating it as Internal. Any other error is returned unchanged. func CallError(err error) error { if status.Code(err) == codes.Unavailable { - return apierror.Unavailable("snapshot plugin: %w", err) + return apierror.Unavailable("plugin: %w", err) } return err } diff --git a/internal/objectstoreplugin/socket_test.go b/internal/pluginsocket/pluginsocket_test.go similarity index 92% rename from internal/objectstoreplugin/socket_test.go rename to internal/pluginsocket/pluginsocket_test.go index 18a8f6f659..b26e83a490 100644 --- a/internal/objectstoreplugin/socket_test.go +++ b/internal/pluginsocket/pluginsocket_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package objectstoreplugin +package pluginsocket import ( "context" @@ -138,7 +138,7 @@ func (s *sidecar) lose(conn *grpc.ClientConn) { } } -var cleanupReq = &objectstorev1.CleanupSnapshotRequest{SnapshotUri: testURI} +var cleanupReq = &objectstorev1.CleanupSnapshotRequest{SnapshotUri: "gs://bucket/root/snap1"} // A plugin lost after startup fails a call with Unavailable once the ready // wait runs out, not when the caller's deadline does. @@ -266,3 +266,19 @@ func TestWaitReadyOutlastsReadyWait(t *testing.T) { t.Errorf("WaitReady for a plugin that starts after the ready wait = %v, want success", err) } } + +// A socket nobody listens on must fail when ctx ends, not hang: WaitReady +// waits for the plugin as long as ctx allows. +func TestWaitReadyEndsWithContext(t *testing.T) { + conn, err := Dial(filepath.Join(t.TempDir(), "missing.sock"), ReadyWait) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + err = WaitReady(ctx, conn) + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("WaitReady against a missing socket = %v, want %s", err, codes.DeadlineExceeded) + } +} From 052c3487128984abf04f83872b3d925c5bb36a26 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 12 Oct 2026 19:16:17 +0000 Subject: [PATCH 06/20] Add actor egress benchmarking tool (#2505) Add tool for benchmarking Substrate egress. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --------- Signed-off-by: Yan Avlasov --- benchmarking/nighthawk-egress/README.md | 107 ++++++ .../nighthawk-egress/deploy-traffic-driver.sh | 209 +++++++++++ benchmarking/nighthawk-egress/deploy.sh | 94 +++++ .../nighthawk-egress/egress-policy.yaml | 40 +++ .../nighthawk-egress/nighthawk-server.yaml | 328 ++++++++++++++++++ .../traffic-driver-invoker.yaml | 158 +++++++++ .../traffic-driver-template.yaml.tmpl | 66 ++++ .../nighthawk-egress/traffic-driver.yaml.tmpl | 38 ++ cmd/ate-setup/internal/config/config.go | 6 + cmd/ate-setup/internal/config/config_test.go | 25 ++ cmd/ate-setup/internal/config/load.go | 7 + cmd/ate-setup/internal/config/options_test.go | 6 + cmd/ate-setup/internal/config/setting.go | 5 + cmd/ate-setup/internal/steps/overlay.go | 49 +++ cmd/ate-setup/internal/steps/overlay_test.go | 72 ++++ .../internal/steps/substrate_test.go | 4 + .../internal/cmd/egress_policy_test.go | 43 +++ docs/operator-install.md | 1 + 18 files changed, 1258 insertions(+) create mode 100644 benchmarking/nighthawk-egress/README.md create mode 100755 benchmarking/nighthawk-egress/deploy-traffic-driver.sh create mode 100755 benchmarking/nighthawk-egress/deploy.sh create mode 100644 benchmarking/nighthawk-egress/egress-policy.yaml create mode 100644 benchmarking/nighthawk-egress/nighthawk-server.yaml create mode 100644 benchmarking/nighthawk-egress/traffic-driver-invoker.yaml create mode 100644 benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl create mode 100644 benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl diff --git a/benchmarking/nighthawk-egress/README.md b/benchmarking/nighthawk-egress/README.md new file mode 100644 index 0000000000..2a14dfc04c --- /dev/null +++ b/benchmarking/nighthawk-egress/README.md @@ -0,0 +1,107 @@ +# Nighthawk Egress Benchmark + +Benchmarks the Agent Substrate egress dataplane (`atunnel` -> `atenet-egress`) using [Nighthawk](https://github.com/envoyproxy/nighthawk). + +## Architecture + +```mermaid +flowchart LR + client["nighthawk_client
(host)"] + invoker["traffic-driver-invoker
(:19980, adds ate-target-actor)"] + router["atenet-router
(:80)"] + + subgraph worker["WorkerPod (ateom-gvisor)"] + atunnel-ingress["ingress atunnel"] + actor["Actor: nighthawk/tester
(nighthawk_service :80)"] + atunnel-egress["egress atunnel"] + end + + egress["atenet-egress
(HTTP / HTTPS MITM / TLS passthrough)"] + server["nighthawk-server
(:80, :443, :8443)"] + + client -->|"start test"| invoker + invoker -->|"gRPC + ate-target-actor: nighthawk/tester"| router + router --> atunnel-ingress + atunnel-ingress --> actor + actor ==>|"test traffic"| atunnel-egress + atunnel-egress ==>|"mTLS"| egress + egress ==> server +``` + +## Prerequisites + +To measure single CPU performance characteristics of Envoy dataplane set E2E_ATE_ENVOY_CONCURRENCY=1 environment +variable when deploying ATE. + +1. **Cluster with Agent Substrate deployed**: + - **Existing cluster** (with `.ate-dev-env.sh` configured): + ```bash + hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}' + ``` + - **Local `kind` cluster**: + ```bash + hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"enabled":false}' + ``` + - To set the `atenet-egress` Envoy worker thread count (`--concurrency`), set `E2E_ATE_ENVOY_CONCURRENCY` when deploying: + ```bash + E2E_ATE_ENVOY_CONCURRENCY=1 hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"enabled":false}' + ``` +2. **`kubectl-ate` installed in `PATH`**: + ```bash + go install ./cmd/kubectl-ate + ``` +3. **Docker** running locally (used to run `nighthawk_client`). + +## Setup + +### 1. Deploy `nighthawk-server` and `traffic-driver-invoker` + +```bash +./benchmarking/nighthawk-egress/deploy.sh +``` + +### 2. Deploy the worker pool, actor template, and `nighthawk/tester` actor + +```bash +./benchmarking/nighthawk-egress/deploy-traffic-driver.sh +``` + +### 3. Port-forward `traffic-driver-invoker` + +In a separate terminal, forward local port `19980` to the `traffic-driver-invoker` service: + +```bash +kubectl -n ate-system port-forward svc/traffic-driver-invoker 19980:19980 +``` + +## Running the Benchmark + +Run `nighthawk_client` against the port-forwarded `traffic-driver-invoker` endpoint (`grpc://127.0.0.1:19980`), which routes the gRPC request to `nighthawk_service` inside the `nighthawk/tester` actor: + +```bash +docker run --rm --network=host --entrypoint /usr/local/bin/nighthawk_client envoyproxy/nighthawk-dev:latest --nighthawk-service grpc://127.0.0.1:19980 --rps 1000 --concurrency 1 --protocol http1 --duration 60 http://nighthawk-server.ate-system.svc.cluster.local:80 +``` + +See [Nighthawk CLI help](https://github.com/envoyproxy/nighthawk#using-the-nighthawk-client-cli) for all command line options. + +### Target URLs and Default Egress Policy + +| Egress Mode | Target URL | Port | +|---|---|---| +| Plaintext HTTP | `http://nighthawk-server.ate-system.svc.cluster.local:80` | `80` | +| HTTPS (TLS MITM) | `https://nighthawk-server.ate-system.svc.cluster.local:443` | `443` | +| TLS Passthrough | `https://nighthawk-server.ate-system.svc.cluster.local:8443` | `8443` | + +## Cleanup + +Delete the `nighthawk/tester` actor, actor template, atespace, and worker pool: + +```bash +./benchmarking/nighthawk-egress/deploy-traffic-driver.sh --delete +``` + +Delete `nighthawk-server` and `traffic-driver-invoker` from `ate-system`: + +```bash +./benchmarking/nighthawk-egress/deploy.sh --delete +``` diff --git a/benchmarking/nighthawk-egress/deploy-traffic-driver.sh b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh new file mode 100755 index 0000000000..af5698742f --- /dev/null +++ b/benchmarking/nighthawk-egress/deploy-traffic-driver.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +cd "${REPO_ROOT}" + +if [[ -f "${REPO_ROOT}/.ate-dev-env.sh" ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then + # shellcheck source=/dev/null + source "${REPO_ROOT}/.ate-dev-env.sh" +fi + +ATESPACE="nighthawk" +TEMPLATE_NAME="traffic-driver" +ACTOR_NAME="tester" +POOL_MANIFEST="${SCRIPT_DIR}/traffic-driver.yaml.tmpl" +MANIFEST="${SCRIPT_DIR}/traffic-driver-template.yaml.tmpl" +EGRESS_POLICY_MANIFEST="${SCRIPT_DIR}/egress-policy.yaml" +KUBECTL_ATE_BIN="kubectl-ate" +WAIT_TIMEOUT_SECS=300 +CONTEXT="${KUBECTL_CONTEXT:-}" +BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" +KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " --deploy Deploy the ${ATESPACE}/${TEMPLATE_NAME} worker pool, actor template, and ${ACTOR_NAME} actor (default)" + echo " --delete Delete the ${ACTOR_NAME} actor, template, atespace, and worker pool" + echo " --bucket-name BUCKET GCS bucket name for actor snapshots (default: \$BUCKET_NAME)" + echo " --ko-docker-repo REPO Container registry for ko image builds (default: \$KO_DOCKER_REPO)" + echo " --context CONTEXT kubectl context to use (default: \$KUBECTL_CONTEXT or current context)" + echo " --wait-timeout SECONDS Timeout waiting for worker pool and golden snapshot readiness (default: ${WAIT_TIMEOUT_SECS})" + echo " -h, --help Show this help message" +} + +action="deploy" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --deploy) + action="deploy" + ;; + --delete) + action="delete" + ;; + --bucket-name) + shift + BUCKET_NAME="$1" + ;; + --bucket-name=*) + BUCKET_NAME="${1#*=}" + ;; + --ko-docker-repo) + shift + KO_DOCKER_REPO="$1" + ;; + --ko-docker-repo=*) + KO_DOCKER_REPO="${1#*=}" + ;; + --context) + shift + CONTEXT="$1" + ;; + --context=*) + CONTEXT="${1#*=}" + ;; + --wait-timeout) + shift + WAIT_TIMEOUT_SECS="$1" + ;; + --wait-timeout=*) + WAIT_TIMEOUT_SECS="${1#*=}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Error: Unknown option: $1" >&2 + usage + exit 1 + ;; + esac + shift +done + +if ! [[ "${WAIT_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then + echo "Error: --wait-timeout must be a whole number of seconds like 300, got '${WAIT_TIMEOUT_SECS}'" >&2 + exit 1 +fi + +run_kubectl() { + local args=() + if [[ -n "${CONTEXT}" ]]; then + args+=(--context "${CONTEXT}") + fi + kubectl "${args[@]}" "$@" +} + +run_kubectl_ate() { + local args=() + if [[ -n "${CONTEXT}" ]]; then + args+=(--context "${CONTEXT}") + fi + "${KUBECTL_ATE_BIN}" "${args[@]}" "$@" +} + +wait_template_ready() { + echo "Waiting for actor template ${ATESPACE}/${TEMPLATE_NAME} golden snapshot (timeout: ${WAIT_TIMEOUT_SECS}s)..." + local deadline=$((SECONDS + WAIT_TIMEOUT_SECS)) + while true; do + local json + json="$(run_kubectl_ate get actor-template "${TEMPLATE_NAME}" -a "${ATESPACE}" -o json)" + local err_msg + err_msg="$(jq -r '.status.goldenSnapshotStatus.errorMessage // empty' <<<"${json}")" + if [[ -n "${err_msg}" ]]; then + echo "Error: ActorTemplate ${ATESPACE}/${TEMPLATE_NAME} golden snapshot failed: ${err_msg}" >&2 + return 1 + fi + local tag + tag="$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}")" + if [[ -n "${tag}" ]]; then + echo " ${ATESPACE}/${TEMPLATE_NAME} ready (golden tag: ${tag})" + return 0 + fi + if (( SECONDS >= deadline )); then + echo "Error: Timed out waiting for ActorTemplate ${ATESPACE}/${TEMPLATE_NAME} golden snapshot" >&2 + echo "${json}" >&2 + return 1 + fi + sleep 2 + done +} + +deploy() { + if [[ -z "${BUCKET_NAME}" ]]; then + echo "Error: BUCKET_NAME is not set. Pass --bucket-name or configure .ate-dev-env.sh." >&2 + exit 1 + fi + if [[ -z "${KO_DOCKER_REPO}" ]]; then + echo "Error: KO_DOCKER_REPO is not set. Pass --ko-docker-repo or configure .ate-dev-env.sh." >&2 + exit 1 + fi + export KO_DOCKER_REPO + + echo "Deploying worker pool from ${POOL_MANIFEST}..." + hack/run-tool.sh ko resolve -f "${POOL_MANIFEST}" | run_kubectl apply -f - + echo "Waiting for worker pool ${ATESPACE}/${TEMPLATE_NAME} to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + run_kubectl wait --for=create "deployment/${TEMPLATE_NAME}" \ + --namespace="${ATESPACE}" --timeout="${WAIT_TIMEOUT_SECS}s" + run_kubectl rollout status "deployment/${TEMPLATE_NAME}" \ + --namespace="${ATESPACE}" --timeout="${WAIT_TIMEOUT_SECS}s" + + echo "Ensuring atespace ${ATESPACE} exists..." + run_kubectl_ate create atespace "${ATESPACE}" >/dev/null 2>&1 \ + || run_kubectl_ate get atespace "${ATESPACE}" >/dev/null + + # Remove any existing actor and template before redeploying. + run_kubectl_ate suspend actor "${ACTOR_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true + run_kubectl_ate delete actor "${ACTOR_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true + run_kubectl_ate delete actor-template "${TEMPLATE_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true + + echo "Creating actor template ${ATESPACE}/${TEMPLATE_NAME} from ${MANIFEST}..." + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" "${MANIFEST}" \ + | run_kubectl_ate create actor-template -f - + + wait_template_ready + + echo "Creating actor ${ATESPACE}/${ACTOR_NAME} from template ${TEMPLATE_NAME}..." + run_kubectl_ate create actor "${ACTOR_NAME}" -a "${ATESPACE}" --template "${TEMPLATE_NAME}" + + echo "Applying egress policy to actor ${ATESPACE}/${ACTOR_NAME} from ${EGRESS_POLICY_MANIFEST}..." + run_kubectl_ate create egress-policy "${ACTOR_NAME}" -a "${ATESPACE}" -f "${EGRESS_POLICY_MANIFEST}" + + echo "Resuming actor ${ATESPACE}/${ACTOR_NAME}..." + run_kubectl_ate resume actor "${ACTOR_NAME}" -a "${ATESPACE}" +} + +delete() { + echo "Suspending and deleting actor ${ATESPACE}/${ACTOR_NAME}..." + run_kubectl_ate suspend actor "${ACTOR_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true + run_kubectl_ate delete actor "${ACTOR_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true + + echo "Deleting actor template ${ATESPACE}/${TEMPLATE_NAME}..." + run_kubectl_ate delete actor-template "${TEMPLATE_NAME}" -a "${ATESPACE}" >/dev/null 2>&1 || true + run_kubectl_ate delete atespace "${ATESPACE}" >/dev/null 2>&1 \ + || echo "atespace ${ATESPACE} not deleted (may not exist or is not empty)" + + echo "Deleting worker pool from ${POOL_MANIFEST}..." + run_kubectl delete --ignore-not-found -f "${POOL_MANIFEST}" +} + +"${action}" diff --git a/benchmarking/nighthawk-egress/deploy.sh b/benchmarking/nighthawk-egress/deploy.sh new file mode 100755 index 0000000000..713e76dc0a --- /dev/null +++ b/benchmarking/nighthawk-egress/deploy.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVER_MANIFEST="${SCRIPT_DIR}/nighthawk-server.yaml" +INVOKER_MANIFEST="${SCRIPT_DIR}/traffic-driver-invoker.yaml" +WAIT_TIMEOUT_SECS=120 +CONTEXT="${KUBECTL_CONTEXT:-}" + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " --deploy Deploy nighthawk-server and traffic-driver-invoker in ate-system (default)" + echo " --delete Delete nighthawk-server and traffic-driver-invoker from ate-system" + echo " --context CONTEXT kubectl context to use (default: \$KUBECTL_CONTEXT or current context)" + echo " --wait-timeout SECONDS Rollout wait timeout in seconds (default: ${WAIT_TIMEOUT_SECS})" + echo " -h, --help Show this help message" +} + +action="deploy" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --deploy) + action="deploy" + ;; + --delete) + action="delete" + ;; + --context) + shift + CONTEXT="$1" + ;; + --context=*) + CONTEXT="${1#*=}" + ;; + --wait-timeout) + shift + WAIT_TIMEOUT_SECS="$1" + ;; + --wait-timeout=*) + WAIT_TIMEOUT_SECS="${1#*=}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Error: Unknown option: $1" >&2 + usage + exit 1 + ;; + esac + shift +done + +if ! [[ "${WAIT_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then + echo "Error: --wait-timeout must be a whole number of seconds like 120, got '${WAIT_TIMEOUT_SECS}'" >&2 + exit 1 +fi + +kubectl_cmd=(kubectl) +if [[ -n "${CONTEXT}" ]]; then + kubectl_cmd+=(--context "${CONTEXT}") +fi + +if [[ "${action}" == "deploy" ]]; then + echo "Deploying Nighthawk server from ${SERVER_MANIFEST}..." + "${kubectl_cmd[@]}" apply -f "${SERVER_MANIFEST}" + echo "Deploying traffic-driver-invoker from ${INVOKER_MANIFEST}..." + "${kubectl_cmd[@]}" apply -f "${INVOKER_MANIFEST}" + echo "Waiting for deployment/nighthawk-server in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + "${kubectl_cmd[@]}" -n ate-system rollout status deployment/nighthawk-server --timeout="${WAIT_TIMEOUT_SECS}s" + echo "Waiting for deployment/traffic-driver-invoker in ate-system to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + "${kubectl_cmd[@]}" -n ate-system rollout status deployment/traffic-driver-invoker --timeout="${WAIT_TIMEOUT_SECS}s" +elif [[ "${action}" == "delete" ]]; then + echo "Deleting Nighthawk server and traffic-driver-invoker..." + "${kubectl_cmd[@]}" delete --ignore-not-found -f "${INVOKER_MANIFEST}" -f "${SERVER_MANIFEST}" +fi diff --git a/benchmarking/nighthawk-egress/egress-policy.yaml b/benchmarking/nighthawk-egress/egress-policy.yaml new file mode 100644 index 0000000000..5e4bca28a1 --- /dev/null +++ b/benchmarking/nighthawk-egress/egress-policy.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Substrate EgressPolicy resource (protojson-shaped ateapipb.EgressPolicy, +# NOT a Kubernetes CRD) for Nighthawk traffic-driver actors in the `nighthawk` +# atespace. Applied to an actor with: +# kubectl-ate create egress-policy -a nighthawk -f egress-policy.yaml +metadata: + atespace: nighthawk + name: default +rules: +- http: + hostnames: + - "*" + ports: + numbers: + - 80 +- https: + hostnames: + - "*" + ports: + numbers: + - 443 +- tlsPassthrough: + hostnames: + - "*" + ports: + numbers: + - 8443 diff --git a/benchmarking/nighthawk-egress/nighthawk-server.yaml b/benchmarking/nighthawk-egress/nighthawk-server.yaml new file mode 100644 index 0000000000..b30073b9b3 --- /dev/null +++ b/benchmarking/nighthawk-egress/nighthawk-server.yaml @@ -0,0 +1,328 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# In-cluster Envoy Nighthawk test server used as the origin target for egress +# benchmarks. Exposes plaintext HTTP on :80 and :8080 and TLS-terminated HTTPS +# on :443 and :8443 using a serving certificate issued by the cluster's +# servicedns pod-certificate signer (SAN: nighthawk-server.ate-system.svc). +apiVersion: v1 +kind: ConfigMap +metadata: + name: nighthawk-server + namespace: ate-system +data: + nighthawk.yaml: | + # SDS requires a node id and cluster. + node: + id: nighthawk-server + cluster: nighthawk-server + + admin: + address: + socket_address: { address: 127.0.0.1, port_value: 10001 } + + static_resources: + listeners: + - name: http_80 + address: + socket_address: { address: "::", ipv4_compat: true, port_value: 80 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: false + codec_type: AUTO + stat_prefix: ingress_http_80 + route_config: + name: local_route + virtual_hosts: + - name: service + domains: ["*"] + http_filters: + - name: time-tracking + typed_config: + "@type": type.googleapis.com/nighthawk.server.TimeTrackingConfiguration + emit_previous_request_delta_in_response_header: x-origin-request-receipt-delta + - name: dynamic-delay + typed_config: + "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration + static_delay: 0s + - name: test-server + typed_config: + "@type": type.googleapis.com/nighthawk.server.ResponseOptions + response_body_size: 10 + v3_response_headers: + - { header: { key: "x-nh", value: "1" } } + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false + + - name: http_8080 + address: + socket_address: { address: "::", ipv4_compat: true, port_value: 8080 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: false + codec_type: AUTO + stat_prefix: ingress_http_8080 + route_config: + name: local_route + virtual_hosts: + - name: service + domains: ["*"] + http_filters: + - name: time-tracking + typed_config: + "@type": type.googleapis.com/nighthawk.server.TimeTrackingConfiguration + emit_previous_request_delta_in_response_header: x-origin-request-receipt-delta + - name: dynamic-delay + typed_config: + "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration + static_delay: 0s + - name: test-server + typed_config: + "@type": type.googleapis.com/nighthawk.server.ResponseOptions + response_body_size: 10 + v3_response_headers: + - { header: { key: "x-nh", value: "1" } } + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false + + - name: https_443 + address: + socket_address: { address: "::", ipv4_compat: true, port_value: 443 } + filter_chains: + - transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + alpn_protocols: ["h2", "http/1.1"] + tls_certificate_sds_secret_configs: + - name: servicedns_serving_cert + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/nighthawk/sds-servicedns-cert.yaml + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: false + codec_type: AUTO + stat_prefix: ingress_https_443 + route_config: + name: local_route + virtual_hosts: + - name: service + domains: ["*"] + http_filters: + - name: time-tracking + typed_config: + "@type": type.googleapis.com/nighthawk.server.TimeTrackingConfiguration + emit_previous_request_delta_in_response_header: x-origin-request-receipt-delta + - name: dynamic-delay + typed_config: + "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration + static_delay: 0s + - name: test-server + typed_config: + "@type": type.googleapis.com/nighthawk.server.ResponseOptions + response_body_size: 10 + v3_response_headers: + - { header: { key: "x-nh", value: "1" } } + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false + + - name: https_8443 + address: + socket_address: { address: "::", ipv4_compat: true, port_value: 8443 } + filter_chains: + - transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + alpn_protocols: ["h2", "http/1.1"] + tls_certificate_sds_secret_configs: + - name: servicedns_serving_cert + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/nighthawk/sds-servicedns-cert.yaml + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: false + codec_type: AUTO + stat_prefix: ingress_https_8443 + route_config: + name: local_route + virtual_hosts: + - name: service + domains: ["*"] + http_filters: + - name: time-tracking + typed_config: + "@type": type.googleapis.com/nighthawk.server.TimeTrackingConfiguration + emit_previous_request_delta_in_response_header: x-origin-request-receipt-delta + - name: dynamic-delay + typed_config: + "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration + static_delay: 0s + - name: test-server + typed_config: + "@type": type.googleapis.com/nighthawk.server.ResponseOptions + response_body_size: 10 + v3_response_headers: + - { header: { key: "x-nh", value: "1" } } + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false + + # SDS resource for the Nighthawk server's TLS serving cert. Kubelet rotates + # the projected volume by swapping its ..data symlink, triggering the + # watched_directory reload on the combined leaf-cert + private-key bundle. + sds-servicedns-cert.yaml: | + resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: servicedns_serving_cert + tls_certificate: + certificate_chain: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } + private_key: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } + watched_directory: { path: /run/servicedns.podcert.ate.dev } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nighthawk-server + namespace: ate-system + labels: + app: nighthawk-server +spec: + replicas: 1 + selector: + matchLabels: + app: nighthawk-server + template: + metadata: + labels: + app: nighthawk-server + spec: + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + fsGroup: 65532 + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + containers: + - name: nighthawk-server + # Same envoyproxy/nighthawk-dev digest as benchmarking/nighthawk-ingress/Dockerfile. + image: envoyproxy/nighthawk-dev@sha256:7729902afea9d83f593b240690ba7d6e8fff04f6215ada538cfb4c1452b2553e + command: + - /usr/local/bin/nighthawk_test_server + args: + - -c + - /etc/nighthawk/nighthawk.yaml + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: http-alt + containerPort: 8080 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: https-alt + containerPort: 8443 + protocol: TCP + readinessProbe: + httpGet: + path: / + port: http-alt + periodSeconds: 5 + volumeMounts: + - name: config + mountPath: /etc/nighthawk + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + volumes: + - name: config + configMap: + name: nighthawk-server + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +# Selecting Service in ate-system required by servicedns.podcert.ate.dev/identity +# to mint a serving certificate with SAN nighthawk-server.ate-system.svc. +apiVersion: v1 +kind: Service +metadata: + name: nighthawk-server + namespace: ate-system +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: nighthawk-server + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + - name: http-alt + port: 8080 + targetPort: http-alt + protocol: TCP + - name: https + port: 443 + targetPort: https + protocol: TCP + - name: https-alt + port: 8443 + targetPort: https-alt + protocol: TCP diff --git a/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml b/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml new file mode 100644 index 0000000000..799ad103bc --- /dev/null +++ b/benchmarking/nighthawk-egress/traffic-driver-invoker.yaml @@ -0,0 +1,158 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# In-cluster Envoy proxy that listens for plaintext HTTP/1.1 and HTTP/2 +# (h2c/gRPC) on :19980, injects `ate-target-actor: nighthawk/tester` on every +# request, and forwards upstream to atenet-router.ate-system.svc.cluster.local:80. +# +# TODO(yanavlasov): this is a temporary workaround for nighthawk's cli's inability +# to add headers when invoking traffic driver gRPC service. So this Envoy adds the +# `ate-target-actor: nighthawk/tester` header so that cli can reach the traffic driver. +apiVersion: v1 +kind: ConfigMap +metadata: + name: traffic-driver-invoker + namespace: ate-system +data: + envoy.yaml: | + static_resources: + listeners: + - name: listener_http_19980 + address: + socket_address: + address: "::" + ipv4_compat: true + port_value: 19980 + listener_filters: + - name: envoy.filters.listener.http_inspector + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.listener.http_inspector.v3.HttpInspector + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http_19980 + codec_type: AUTO + route_config: + name: local_route + request_headers_to_add: + - header: + key: ate-target-actor + value: nighthawk/tester + append_action: OVERWRITE_IF_EXISTS_OR_ADD + virtual_hosts: + - name: backend + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: atenet_router + timeout: 0s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: atenet_router + type: STRICT_DNS + connect_timeout: 5s + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + use_downstream_protocol_config: + http_protocol_options: {} + http2_protocol_options: {} + load_assignment: + cluster_name: atenet_router + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: atenet-router.ate-system.svc.cluster.local + port_value: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: traffic-driver-invoker + namespace: ate-system + labels: + app: traffic-driver-invoker +spec: + replicas: 1 + selector: + matchLabels: + app: traffic-driver-invoker + template: + metadata: + labels: + app: traffic-driver-invoker + spec: + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + containers: + - name: envoy + # Same envoyproxy/envoy digest as cmd/dataplane/envoy/Dockerfile. + image: envoyproxy/envoy:v1.39.1@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 + command: + - /usr/local/bin/envoy + args: + - -c + - /etc/envoy/envoy.yaml + - --concurrency + - "1" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + ports: + - name: http + containerPort: 19980 + protocol: TCP + readinessProbe: + tcpSocket: + port: http + periodSeconds: 5 + volumeMounts: + - name: config + mountPath: /etc/envoy + readOnly: true + volumes: + - name: config + configMap: + name: traffic-driver-invoker +--- +apiVersion: v1 +kind: Service +metadata: + name: traffic-driver-invoker + namespace: ate-system +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: traffic-driver-invoker + ports: + - name: http + port: 19980 + targetPort: http + protocol: TCP diff --git a/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl new file mode 100644 index 0000000000..6434b5c6ee --- /dev/null +++ b/benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Substrate ActorTemplate resource (protojson-shaped ateapipb.ActorTemplate, +# NOT the ate.dev/v1alpha1 CRD) for the Nighthawk egress traffic-driver actor. +# Applied with `kubectl-ate create actor-template -f -` by +# deploy-traffic-driver.sh; the `nighthawk` atespace must exist. +# +# Runs Envoy's `nighthawk_service` gRPC server on port 80 (the default actor +# ingress port forwarded by atunnel) so benchmark runners can invoke +# ExecutionStream through atenet-router to drive HTTP/HTTPS egress load from +# inside the gVisor actor sandbox. +metadata: + atespace: nighthawk + name: traffic-driver +workerSelector: + matchLabels: + workload: traffic-driver +containers: +- name: nighthawk-service + # Same envoyproxy/nighthawk-dev digest as nighthawk-server.yaml and + # benchmarking/nighthawk-ingress/Dockerfile. + image: envoyproxy/nighthawk-dev@sha256:7729902afea9d83f593b240690ba7d6e8fff04f6215ada538cfb4c1452b2553e + command: + - /usr/local/bin/nighthawk_service + args: + - --listen + - 0.0.0.0:80 + - --service + - traffic-generator-service + env: + - name: SSL_CERT_FILE + value: /run/ate/egress-trust-bundle/trust-bundle.pem + - name: SSL_CERT_DIR + value: /run/ate/egress-trust-bundle/ + volumeMounts: + - name: system-info + mountPath: /run/ate +volumes: +- name: system-info + systemInfo: + dataSources: + # Trust anchors for per-SNI leaf certificates minted by the egress gateway + # when terminating HTTPS egress traffic. + - trustBundle: + names: + - egress-mitm.ate.dev + - system-roots.ate.dev + path: egress-trust-bundle/trust-bundle.pem +sandboxConfig: + sandboxClass: SANDBOX_CLASS_GVISOR + configName: gvisor-default +snapshotConfig: + preferredFidelity: SNAPSHOT_FIDELITY_MEMORY + storageLocation: gs://${BUCKET_NAME}/nighthawk/traffic-driver/ diff --git a/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl new file mode 100644 index 0000000000..155df4dc9a --- /dev/null +++ b/benchmarking/nighthawk-egress/traffic-driver.yaml.tmpl @@ -0,0 +1,38 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Worker pool for the Nighthawk egress traffic-driver actor. The ActorTemplate +# is a substrate resource, not a CRD: it lives in +# traffic-driver-template.yaml.tmpl and is created through the ate API with +# `kubectl-ate create actor-template`. + +apiVersion: v1 +kind: Namespace +metadata: + name: nighthawk + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: traffic-driver + namespace: nighthawk + labels: + workload: traffic-driver +spec: + replicas: 1 + workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + sandboxClasses: + - name: gvisor diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 8050a58aea..0b415b25c3 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -180,6 +180,9 @@ type Config struct { // PodcertWorkersPerSigner overrides WORKERS_PER_SIGNER on podcertificate-controller. PodcertWorkersPerSigner int + // EnvoyConcurrency overrides --concurrency on the envoy container in atenet-egress. + EnvoyConcurrency int + // ClusterSize is the footprint profile (ATE_INSTALL_CLUSTER_SIZE): size0 // or size10. ClusterSize string @@ -416,6 +419,9 @@ func (c *Config) ScriptEnv() []string { if c.PodcertWorkersPerSigner > 0 { merged["ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER"] = strconv.Itoa(c.PodcertWorkersPerSigner) } + if c.EnvoyConcurrency > 0 { + merged["E2E_ATE_ENVOY_CONCURRENCY"] = strconv.Itoa(c.EnvoyConcurrency) + } // Resolved values again, so a flag overrides whatever the environment // carried rather than layering under it. delete(merged, "ATE_INSTALL_CLUSTER_SIZE") diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 240446ecfe..919181a4e9 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -67,6 +67,7 @@ func loadEnv(t *testing.T) { "CLUSTER_LOCATION", "CLUSTER_NAME", "DOCKER_BUILD_FLAGS", + "E2E_ATE_ENVOY_CONCURRENCY", "EXPECTED_JWT_ISSUER", "KIND_CLUSTER_NAME", "KO_DEFAULTPLATFORMS", @@ -799,6 +800,30 @@ func TestLoadRejectsInvalidPodcertWorkersEnv(t *testing.T) { } } +func TestLoadEnvoyConcurrency(t *testing.T) { + loadEnv(t) + cfg, err := Load(Options{EnvoyConcurrency: 4}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.EnvoyConcurrency != 4 { + t.Errorf("EnvoyConcurrency = %d, want 4", cfg.EnvoyConcurrency) + } + if got := scriptEnvMap(t, cfg)["E2E_ATE_ENVOY_CONCURRENCY"]; got != "4" { + t.Errorf("ScriptEnv()[E2E_ATE_ENVOY_CONCURRENCY] = %q, want 4", got) + } + + for _, val := range []string{"0", "-1", "abc"} { + t.Run("invalid_"+val, func(t *testing.T) { + loadEnv(t) + t.Setenv("E2E_ATE_ENVOY_CONCURRENCY", val) + if _, err := Load(Options{}); err == nil { + t.Fatalf("Load() with E2E_ATE_ENVOY_CONCURRENCY=%q succeeded, want an error", val) + } + }) + } +} + // --kind must reproduce the exports in the shell kind installer, including // clearing the GKE coordinates and isolating the snapshot bucket name. func TestLoadKindProfile(t *testing.T) { diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 81fe2aa2dc..5acc702b56 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -188,6 +188,12 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro return nil, &InvalidError{Value: v, Want: "a positive integer"} } + envoyConcurrency := r.Int("e2e.envoyConcurrency") + if r.Supplied("e2e.envoyConcurrency") && envoyConcurrency < 1 { + v, _ := r.Value("e2e.envoyConcurrency") + return nil, &InvalidError{Value: v, Want: "a positive integer"} + } + kubeconfig, kubeconfigEnv := splitKubeconfig(r) // One configured connection serves both pools; both empty selects the @@ -250,6 +256,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro RolloutTimeout: rolloutTimeout, rolloutTimeoutSet: r.Supplied("rolloutTimeout"), PodcertWorkersPerSigner: podcertWorkers, + EnvoyConcurrency: envoyConcurrency, ClusterSize: r.String("clusterSize"), CordonControlPlane: r.Bool("cordonControlPlane"), AdditionalEgressExtprocService: r.String("atenet.egress.additionalExtprocService"), diff --git a/cmd/ate-setup/internal/config/options_test.go b/cmd/ate-setup/internal/config/options_test.go index e84cc2817d..61621ed224 100644 --- a/cmd/ate-setup/internal/config/options_test.go +++ b/cmd/ate-setup/internal/config/options_test.go @@ -36,6 +36,7 @@ type Options struct { Router string RolloutTimeout string PodcertWorkersPerSigner int + EnvoyConcurrency int ClusterSize string CordonControlPlane bool AdditionalEgressExtprocService string @@ -114,5 +115,10 @@ func optionsFlagSet(opts Options) (*pflag.FlagSet, error) { return nil, err } } + if opts.EnvoyConcurrency != 0 { + if err := fs.Set("e2e-envoy-concurrency", strconv.Itoa(opts.EnvoyConcurrency)); err != nil { + return nil, err + } + } return fs, nil } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 0f803f2a11..b2527db22b 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -249,6 +249,11 @@ var Registry = []Setting{ Flag: "podcert-workers-per-signer", Kind: KindInt, Default: "0", Usage: "Worker goroutines per signer in podcertificate-controller", }, + { + Key: "e2e.envoyConcurrency", Env: "E2E_ATE_ENVOY_CONCURRENCY", + Flag: "e2e-envoy-concurrency", Kind: KindInt, Default: "0", + Usage: "Worker threads (--concurrency) for the Envoy container in atenet-egress", + }, { Key: "rolloutTimeout", Env: "ATE_INSTALL_ROLLOUT_TIMEOUT", Flag: "rollout-timeout", diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 390dd8a6ef..9f884d5ee6 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -19,6 +19,8 @@ import ( "context" "fmt" "os" + "path/filepath" + "strconv" "strings" "time" @@ -179,6 +181,12 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr return nil, err } raw = e.patchEnvoyDataplaneImage(raw, imageReference) + if e.Cfg.EnvoyConcurrency > 0 { + raw, err = patchAtenetEgressEnvoyConcurrency(raw, e.Cfg.EnvoyConcurrency) + if err != nil { + return nil, err + } + } rendered, err := e.renderBytes(raw) if err != nil { return nil, err @@ -186,6 +194,47 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr return e.ResolveManifestBytes(ctx, rendered) } +// patchAtenetEgressEnvoyConcurrency uses a kustomize JSON6902 patch to append +// --concurrency to the envoy container's args in the +// atenet-egress Deployment. +func patchAtenetEgressEnvoyConcurrency(raw []byte, concurrency int) ([]byte, error) { + dir, err := os.MkdirTemp("", "ate-setup-kustomize-") + if err != nil { + return nil, fmt.Errorf("while creating a scratch kustomization: %w", err) + } + defer func() { _ = os.RemoveAll(dir) }() + + if err := os.WriteFile(filepath.Join(dir, "manifest.yaml"), raw, 0o600); err != nil { + return nil, fmt.Errorf("while staging a manifest for kustomize: %w", err) + } + kustomization := fmt.Sprintf(`apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - manifest.yaml +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: atenet-egress + namespace: ate-system + patch: |- + - op: test + path: /spec/template/spec/containers/0/name + value: envoy + - op: add + path: /spec/template/spec/containers/0/args/- + value: --concurrency + - op: add + path: /spec/template/spec/containers/0/args/- + value: %q +`, strconv.Itoa(concurrency)) + if err := os.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte(kustomization), 0o600); err != nil { + return nil, fmt.Errorf("while writing a scratch kustomization: %w", err) + } + return kustomize.Build(dir) +} + // patchEnvoyDataplaneImage replaces the ${ENVOY_DATAPLANE_IMAGE} placeholder in // the manifest with imageRef. func (e *Env) patchEnvoyDataplaneImage(raw []byte, imageRef string) []byte { diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 3cfd8610e7..b213fbfa96 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -873,3 +873,75 @@ func TestPatchEnvoyDataplaneImage(t *testing.T) { t.Errorf("patchEnvoyDataplaneImage() = %q, want %q", got, want) } } + +func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) { + const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" + src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} + + envoyArgs := func(t *testing.T, manifest []byte) []string { + t.Helper() + for _, doc := range strings.Split(string(manifest), "\n---\n") { + var dep appsv1.Deployment + if err := yaml.Unmarshal([]byte(doc), &dep); err != nil || dep.Kind != "Deployment" || dep.Name != "atenet-egress" { + continue + } + for _, c := range dep.Spec.Template.Spec.Containers { + if c.Name == "envoy" { + return c.Args + } + } + t.Fatal("atenet-egress Deployment has no envoy container") + } + t.Fatal("rendered manifest has no atenet-egress Deployment") + return nil + } + + t.Run("unset leaves envoy args unchanged", func(t *testing.T) { + e := &Env{ + Cfg: &config.Config{Root: repoRoot(t), Router: config.RouterEnvoy, Images: src}, + resolver: images.NewPrebuilt(src, func(context.Context, string) (string, error) { + return digest, nil + }), + } + out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + if err != nil { + t.Fatalf("renderAtenetEgressManifest() error = %v", err) + } + args := envoyArgs(t, out) + if slices.Contains(args, "--concurrency") { + t.Errorf("envoy args unexpectedly contain --concurrency: %v", args) + } + }) + + t.Run("specified appends --concurrency to envoy container via kustomize", func(t *testing.T) { + e := &Env{ + Cfg: &config.Config{ + Root: repoRoot(t), + Router: config.RouterEnvoy, + Images: src, + EnvoyConcurrency: 4, + }, + resolver: images.NewPrebuilt(src, func(context.Context, string) (string, error) { + return digest, nil + }), + } + out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + if err != nil { + t.Fatalf("renderAtenetEgressManifest() error = %v", err) + } + args := envoyArgs(t, out) + want := []string{ + "-c", + "/etc/envoy/envoy.yaml", + "--service-node", + "atenet-egress", + "--service-cluster", + "atenet-egress", + "--concurrency", + "4", + } + if !slices.Equal(args, want) { + t.Errorf("envoy args = %v, want %v", args, want) + } + }) +} diff --git a/cmd/ate-setup/internal/steps/substrate_test.go b/cmd/ate-setup/internal/steps/substrate_test.go index 2f192e8122..6b2419461d 100644 --- a/cmd/ate-setup/internal/steps/substrate_test.go +++ b/cmd/ate-setup/internal/steps/substrate_test.go @@ -96,6 +96,10 @@ func TestActorTemplateFromManifestDemos(t *testing.T) { values: benchmarkValues, want: resources.ActorTemplateRef{Atespace: "benchmark-workloads", Name: "kernelmem"}, }, + { + relPath: "benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl", + want: resources.ActorTemplateRef{Atespace: "nighthawk", Name: "traffic-driver"}, + }, } for _, tc := range tests { t.Run(tc.relPath, func(t *testing.T) { diff --git a/cmd/kubectl-ate/internal/cmd/egress_policy_test.go b/cmd/kubectl-ate/internal/cmd/egress_policy_test.go index 455689a2fe..d736701d1d 100644 --- a/cmd/kubectl-ate/internal/cmd/egress_policy_test.go +++ b/cmd/kubectl-ate/internal/cmd/egress_policy_test.go @@ -17,6 +17,8 @@ package cmd import ( "bytes" "context" + "os" + "path/filepath" "strings" "testing" "time" @@ -1083,3 +1085,44 @@ func TestDeleteEgressPolicyRunner_Run(t *testing.T) { }) } } + +func TestEgressPolicyFromManifest_NighthawkEgress(t *testing.T) { + t.Parallel() + data, err := os.ReadFile(filepath.Join("../../../..", "benchmarking", "nighthawk-egress", "egress-policy.yaml")) + if err != nil { + t.Fatalf("reading nighthawk-egress policy manifest: %v", err) + } + got, err := egressPolicyFromManifest(data) + if err != nil { + t.Fatalf("egressPolicyFromManifest: %v", err) + } + want := &ateapipb.EgressPolicy{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: "nighthawk", + Name: "default", + }, + Rules: []*ateapipb.EgressRule{ + { + Http: &ateapipb.HTTPRule{ + Hostnames: []string{"*"}, + Ports: &ateapipb.Ports{Numbers: []int32{80}}, + }, + }, + { + Https: &ateapipb.HTTPSRule{ + Hostnames: []string{"*"}, + Ports: &ateapipb.Ports{Numbers: []int32{443}}, + }, + }, + { + TlsPassthrough: &ateapipb.TLSPassthroughRule{ + Hostnames: []string{"*"}, + Ports: &ateapipb.Ports{Numbers: []int32{8443}}, + }, + }, + }, + } + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("egressPolicyFromManifest mismatch (-want +got):\n%s", diff) + } +} diff --git a/docs/operator-install.md b/docs/operator-install.md index 531dbecf67..6f7bf5fb57 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -295,6 +295,7 @@ command. A setting with no default is unset unless something supplies it. | `demo.counter.storageClass` | `STORAGE_CLASS` | `--storage-class` | `standard` | `deploy demo counter` | | `demo.counter.withExternalVolume` | `ATE_DEMO_COUNTER_WITH_EXTERNAL_VOLUME` | `--with-external-volume` | `false` | `deploy demo counter` | | `docker.buildFlags` | `DOCKER_BUILD_FLAGS` | `--docker-build-flags` | — | (global) | +| `e2e.envoyConcurrency` | `E2E_ATE_ENVOY_CONCURRENCY` | `--e2e-envoy-concurrency` | `0` | (global) | | `gcp.clusterLocation` | `CLUSTER_LOCATION` | `--gcp-cluster-location` | — | (global) | | `gcp.clusterName` | `CLUSTER_NAME` | `--gcp-cluster-name` | — | (global) | | `gcp.projectID` | `PROJECT_ID` | `--gcp-project-id` | — | (global) | From 9e7d01ef266e210550a5c46550f0c6f8394f6a21 Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Mon, 12 Oct 2026 19:37:33 +0000 Subject: [PATCH 07/20] Make `DrainWorker` return `DrainWorkerResponse` (#2501) `DrainWorker` was the only custom method that returned its resource directly; every other custom method returns a `{MethodName}Response` wrapper. A wrapper keeps the API uniform and leaves room for method-specific output in the future. The API style guide now requires custom methods to return a `{MethodName}Response` and no longer lets them return the resource itself. Part of #2411 --- .../controlapi/functionaltest/worker_test.go | 7 +- cmd/ateapi/internal/controlapi/worker.go | 8 +- cmd/ateapi/internal/controlapi/worker_test.go | 12 +- .../internal/workersync/fakecontrol_test.go | 6 +- .../isolate/fake-workersync/controller.go | 2 +- .../fake-workersync/controller_test.go | 4 +- docs/api-style-guide.md | 8 +- pkg/proto/ateapipb/ateapi.pb.go | 544 ++++++++++-------- pkg/proto/ateapipb/ateapi.proto | 8 +- pkg/proto/ateapipb/ateapi_grpc.pb.go | 10 +- 10 files changed, 337 insertions(+), 272 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go index 908df6ca27..8b6a9fb483 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -263,9 +263,10 @@ func TestDrainWorker(t *testing.T) { t.Fatalf("DrainWorker failed: %v", err) } - want := newTestWorker(ns) - want.Metadata.Version = 2 - want.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING} + wantWorker := newTestWorker(ns) + wantWorker.Metadata.Version = 2 + wantWorker.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING} + want := &ateapipb.DrainWorkerResponse{Worker: wantWorker} if diff := cmp.Diff(want, drained, protocmp.Transform(), ignoreServerMetadata); diff != "" { t.Errorf("DrainWorker response mismatch (-want +got):\n%s", diff) } diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index b67253ee3a..46d6ab0393 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -252,7 +252,7 @@ func (s *ServiceImpl) DeleteWorker(ctx context.Context, name string, preconditio return s.store.DeleteWorker(ctx, name, precondition) } -func (s *RPCService) DrainWorker(ctx context.Context, req *ateapipb.DrainWorkerRequest) (*ateapipb.Worker, error) { +func (s *RPCService) DrainWorker(ctx context.Context, req *ateapipb.DrainWorkerRequest) (*ateapipb.DrainWorkerResponse, error) { if errs := apivalidation.ValidateDrainWorkerRequest(ctx, req); len(errs) > 0 { return nil, resources.ToAPIError(errs) } @@ -270,7 +270,7 @@ func (s *RPCService) DrainWorker(ctx context.Context, req *ateapipb.DrainWorkerR return nil, fmt.Errorf("while getting worker to drain: %w", err) } - return s.mutateWorker(ctx, name, store.PreconditionFrom(observed), func(toUpdate *ateapipb.Worker) error { + drained, err := s.mutateWorker(ctx, name, store.PreconditionFrom(observed), func(toUpdate *ateapipb.Worker) error { if toUpdate.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING { // already draining, do nothing return &workerUnchanged{worker: proto.Clone(toUpdate).(*ateapipb.Worker)} @@ -280,6 +280,10 @@ func (s *RPCService) DrainWorker(ctx context.Context, req *ateapipb.DrainWorkerR // until something releases them. Draining only stops new placements. return nil }) + if err != nil { + return nil, err + } + return &ateapipb.DrainWorkerResponse{Worker: drained}, nil } // mutateWorker runs mutate against the named Worker and translates what comes diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index 68c0f6d1fe..78e8cddd10 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -497,7 +497,7 @@ func TestUpdateWorker_DrainingWorkerKeepsLabelsMutable(t *testing.T) { } got, err := svc.UpdateWorker(ctx, &ateapipb.UpdateWorkerRequest{ - Worker: updateFrom(drained, func(w *ateapipb.Worker) { w.Labels = map[string]string{"tier": "batch"} }), + Worker: updateFrom(drained.GetWorker(), func(w *ateapipb.Worker) { w.Labels = map[string]string{"tier": "batch"} }), }) if err != nil { t.Fatalf("UpdateWorker() failed: %v", err) @@ -616,11 +616,11 @@ func TestDrainWorker(t *testing.T) { if err != nil { t.Fatalf("DrainWorker() failed: %v", err) } - if got.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_DRAINING { - t.Errorf("state = %v, want %v", got.GetStatus().GetState(), ateapipb.WorkerState_WORKER_STATE_DRAINING) + if got.GetWorker().GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_DRAINING { + t.Errorf("state = %v, want %v", got.GetWorker().GetStatus().GetState(), ateapipb.WorkerState_WORKER_STATE_DRAINING) } - if got.GetMetadata().GetVersion() != 2 { - t.Errorf("version = %d, want 2", got.GetMetadata().GetVersion()) + if got.GetWorker().GetMetadata().GetVersion() != 2 { + t.Errorf("version = %d, want 2", got.GetWorker().GetMetadata().GetVersion()) } // Draining again is a no-op, and specifically must not bump the version: @@ -646,7 +646,7 @@ func TestDrainWorker_KeepsAssignment(t *testing.T) { if err != nil { t.Fatalf("DrainWorker() failed: %v", err) } - if n := got.GetStatus().GetAllocated().GetActors(); n != 1 { + if n := got.GetWorker().GetStatus().GetAllocated().GetActors(); n != 1 { t.Errorf("drained worker hosts %d actors, want the 1 left in place", n) } } diff --git a/cmd/atecontroller/internal/workersync/fakecontrol_test.go b/cmd/atecontroller/internal/workersync/fakecontrol_test.go index 969b5ec3a2..10d59a5e83 100644 --- a/cmd/atecontroller/internal/workersync/fakecontrol_test.go +++ b/cmd/atecontroller/internal/workersync/fakecontrol_test.go @@ -227,7 +227,7 @@ func (f *fakeControl) DeleteWorker(_ context.Context, in *ateapipb.DeleteWorkerR return stored, nil } -func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { +func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerRequest, _ ...grpc.CallOption) (*ateapipb.DrainWorkerResponse, error) { f.mu.Lock() defer f.mu.Unlock() @@ -238,13 +238,13 @@ func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerReq } if stored.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING { // Already draining: no write, and specifically no version bump. - return proto.Clone(stored).(*ateapipb.Worker), nil + return &ateapipb.DrainWorkerResponse{Worker: proto.Clone(stored).(*ateapipb.Worker)}, nil } drained := proto.Clone(stored).(*ateapipb.Worker) drained.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING drained.Metadata.Version++ f.workers[name] = drained - return proto.Clone(drained).(*ateapipb.Worker), nil + return &ateapipb.DrainWorkerResponse{Worker: proto.Clone(drained).(*ateapipb.Worker)}, nil } func (f *fakeControl) ListWorkers(_ context.Context, in *ateapipb.ListWorkersRequest, _ ...grpc.CallOption) (*ateapipb.ListWorkersResponse, error) { diff --git a/cmd/benchmarking/isolate/fake-workersync/controller.go b/cmd/benchmarking/isolate/fake-workersync/controller.go index 7726105b07..b646fe1faf 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller.go @@ -47,7 +47,7 @@ import ( type workerClient interface { CreateWorker(ctx context.Context, in *ateapipb.CreateWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) DeleteWorker(ctx context.Context, in *ateapipb.DeleteWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) - DrainWorker(ctx context.Context, in *ateapipb.DrainWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) + DrainWorker(ctx context.Context, in *ateapipb.DrainWorkerRequest, opts ...grpc.CallOption) (*ateapipb.DrainWorkerResponse, error) GetWorker(ctx context.Context, in *ateapipb.GetWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) UpdateWorker(ctx context.Context, in *ateapipb.UpdateWorkerRequest, opts ...grpc.CallOption) (*ateapipb.Worker, error) ListWorkers(ctx context.Context, in *ateapipb.ListWorkersRequest, opts ...grpc.CallOption) (*ateapipb.ListWorkersResponse, error) diff --git a/cmd/benchmarking/isolate/fake-workersync/controller_test.go b/cmd/benchmarking/isolate/fake-workersync/controller_test.go index a28dff6f56..cf9493ed22 100644 --- a/cmd/benchmarking/isolate/fake-workersync/controller_test.go +++ b/cmd/benchmarking/isolate/fake-workersync/controller_test.go @@ -91,7 +91,7 @@ func (f *fakeControl) DeleteWorker(_ context.Context, in *ateapipb.DeleteWorkerR return w, nil } -func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { +func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerRequest, _ ...grpc.CallOption) (*ateapipb.DrainWorkerResponse, error) { f.mu.Lock() defer f.mu.Unlock() if f.drainErr != nil { @@ -102,7 +102,7 @@ func (f *fakeControl) DrainWorker(_ context.Context, in *ateapipb.DrainWorkerReq return nil, status.Error(codes.NotFound, "no such Worker") } w.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING - return w, nil + return &ateapipb.DrainWorkerResponse{Worker: w}, nil } func (f *fakeControl) GetWorker(_ context.Context, in *ateapipb.GetWorkerRequest, _ ...grpc.CallOption) (*ateapipb.Worker, error) { diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index c4af026579..6809490e11 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -326,18 +326,22 @@ allow external controllers and upstream systems / automations to react to resour Custom methods are for operations that don't map cleanly to CRUD: lifecycle transitions (Suspend, Resume, Pause), long-running actions, or commands with side effects that standard Update semantics would misrepresent. ```proto -rpc SuspendActor(SuspendActorRequest) returns (Actor) {} +rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} message SuspendActorRequest { ObjectRef actor = 1; } + +message SuspendActorResponse { + Actor actor = 1; +} ``` Rules: - RPC name **must** be a verb phrase: `{Verb}{Resource}` (e.g., `SuspendActor`, `ResumeActor`). - Request message name **must** match the RPC name with a `Request` suffix. - The request **must** identify the target resource using an `ObjectRef` field (for both atespace-scoped and global-scoped resources). -- Custom methods should return a response message matching the RPC name, with a Response suffix. When operating on a specific resource, a custom method may return the resource itself. +- Response message name **must** match the RPC name with a `Response` suffix. A custom method **must not** return the resource directly; put it in a field of the response, so method-specific output can be added later (e.g., `ResumeActorResponse.resumed`). --- diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index d92ef51c95..8d60d0f399 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -6835,6 +6835,52 @@ func (x *DrainWorkerRequest) GetWorker() *ObjectRef { return nil } +// DrainWorkerResponse holds the drained Worker. +type DrainWorkerResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The worker being drained. + Worker *Worker `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DrainWorkerResponse) Reset() { + *x = DrainWorkerResponse{} + mi := &file_ateapi_proto_msgTypes[94] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DrainWorkerResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DrainWorkerResponse) ProtoMessage() {} + +func (x *DrainWorkerResponse) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[94] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DrainWorkerResponse.ProtoReflect.Descriptor instead. +func (*DrainWorkerResponse) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{94} +} + +func (x *DrainWorkerResponse) GetWorker() *Worker { + if x != nil { + return x.Worker + } + return nil +} + // List actors with pagination. Provides "soft" guarantees: actors may be // missed or duplicated if the system state changes during pagination. type ListActorsRequest struct { @@ -6863,7 +6909,7 @@ type ListActorsRequest struct { func (x *ListActorsRequest) Reset() { *x = ListActorsRequest{} - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6875,7 +6921,7 @@ func (x *ListActorsRequest) String() string { func (*ListActorsRequest) ProtoMessage() {} func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[94] + mi := &file_ateapi_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6888,7 +6934,7 @@ func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsRequest.ProtoReflect.Descriptor instead. func (*ListActorsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{94} + return file_ateapi_proto_rawDescGZIP(), []int{95} } func (x *ListActorsRequest) GetAtespace() string { @@ -6924,7 +6970,7 @@ type ListActorsResponse struct { func (x *ListActorsResponse) Reset() { *x = ListActorsResponse{} - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6936,7 +6982,7 @@ func (x *ListActorsResponse) String() string { func (*ListActorsResponse) ProtoMessage() {} func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[95] + mi := &file_ateapi_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6949,7 +6995,7 @@ func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsResponse.ProtoReflect.Descriptor instead. func (*ListActorsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{95} + return file_ateapi_proto_rawDescGZIP(), []int{96} } func (x *ListActorsResponse) GetActors() []*Actor { @@ -7056,7 +7102,7 @@ type Worker struct { func (x *Worker) Reset() { *x = Worker{} - mi := &file_ateapi_proto_msgTypes[96] + mi := &file_ateapi_proto_msgTypes[97] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7068,7 +7114,7 @@ func (x *Worker) String() string { func (*Worker) ProtoMessage() {} func (x *Worker) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[96] + mi := &file_ateapi_proto_msgTypes[97] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7081,7 +7127,7 @@ func (x *Worker) ProtoReflect() protoreflect.Message { // Deprecated: Use Worker.ProtoReflect.Descriptor instead. func (*Worker) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{96} + return file_ateapi_proto_rawDescGZIP(), []int{97} } func (x *Worker) GetMetadata() *ResourceMetadata { @@ -7210,7 +7256,7 @@ type WorkerStatus struct { func (x *WorkerStatus) Reset() { *x = WorkerStatus{} - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7222,7 +7268,7 @@ func (x *WorkerStatus) String() string { func (*WorkerStatus) ProtoMessage() {} func (x *WorkerStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[97] + mi := &file_ateapi_proto_msgTypes[98] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7235,7 +7281,7 @@ func (x *WorkerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. func (*WorkerStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{97} + return file_ateapi_proto_rawDescGZIP(), []int{98} } func (x *WorkerStatus) GetState() WorkerState { @@ -7302,7 +7348,7 @@ type SandboxRuntime struct { func (x *SandboxRuntime) Reset() { *x = SandboxRuntime{} - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[99] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7314,7 +7360,7 @@ func (x *SandboxRuntime) String() string { func (*SandboxRuntime) ProtoMessage() {} func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[98] + mi := &file_ateapi_proto_msgTypes[99] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7327,7 +7373,7 @@ func (x *SandboxRuntime) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxRuntime.ProtoReflect.Descriptor instead. func (*SandboxRuntime) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{98} + return file_ateapi_proto_rawDescGZIP(), []int{99} } func (x *SandboxRuntime) GetSandboxClass() string { @@ -7375,7 +7421,7 @@ type VersionedSandboxCompat struct { func (x *VersionedSandboxCompat) Reset() { *x = VersionedSandboxCompat{} - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[100] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7387,7 +7433,7 @@ func (x *VersionedSandboxCompat) String() string { func (*VersionedSandboxCompat) ProtoMessage() {} func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[99] + mi := &file_ateapi_proto_msgTypes[100] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7400,7 +7446,7 @@ func (x *VersionedSandboxCompat) ProtoReflect() protoreflect.Message { // Deprecated: Use VersionedSandboxCompat.ProtoReflect.Descriptor instead. func (*VersionedSandboxCompat) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{99} + return file_ateapi_proto_rawDescGZIP(), []int{100} } func (x *VersionedSandboxCompat) GetSchemaVersion() string { @@ -7436,7 +7482,7 @@ type AttributeEntry struct { func (x *AttributeEntry) Reset() { *x = AttributeEntry{} - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7448,7 +7494,7 @@ func (x *AttributeEntry) String() string { func (*AttributeEntry) ProtoMessage() {} func (x *AttributeEntry) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[100] + mi := &file_ateapi_proto_msgTypes[101] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7461,7 +7507,7 @@ func (x *AttributeEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeEntry.ProtoReflect.Descriptor instead. func (*AttributeEntry) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{100} + return file_ateapi_proto_rawDescGZIP(), []int{101} } func (x *AttributeEntry) GetKey() string { @@ -7500,7 +7546,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7512,7 +7558,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[101] + mi := &file_ateapi_proto_msgTypes[102] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7525,7 +7571,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{101} + return file_ateapi_proto_rawDescGZIP(), []int{102} } func (x *WorkerResources) GetResources() *Resources { @@ -7578,7 +7624,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[103] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7590,7 +7636,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[102] + mi := &file_ateapi_proto_msgTypes[103] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7603,7 +7649,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{102} + return file_ateapi_proto_rawDescGZIP(), []int{103} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -7674,7 +7720,7 @@ type RegisterWorkerRequest struct { func (x *RegisterWorkerRequest) Reset() { *x = RegisterWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[104] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7686,7 +7732,7 @@ func (x *RegisterWorkerRequest) String() string { func (*RegisterWorkerRequest) ProtoMessage() {} func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[103] + mi := &file_ateapi_proto_msgTypes[104] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7699,7 +7745,7 @@ func (x *RegisterWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerRequest.ProtoReflect.Descriptor instead. func (*RegisterWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{103} + return file_ateapi_proto_rawDescGZIP(), []int{104} } func (x *RegisterWorkerRequest) GetWorker() *ObjectRef { @@ -7740,7 +7786,7 @@ type RegisterWorkerResponse struct { func (x *RegisterWorkerResponse) Reset() { *x = RegisterWorkerResponse{} - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[105] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7752,7 +7798,7 @@ func (x *RegisterWorkerResponse) String() string { func (*RegisterWorkerResponse) ProtoMessage() {} func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[104] + mi := &file_ateapi_proto_msgTypes[105] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7765,7 +7811,7 @@ func (x *RegisterWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterWorkerResponse.ProtoReflect.Descriptor instead. func (*RegisterWorkerResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{104} + return file_ateapi_proto_rawDescGZIP(), []int{105} } func (x *RegisterWorkerResponse) GetWorker() *Worker { @@ -7804,7 +7850,7 @@ type MintAteomActorCertificateRequest struct { func (x *MintAteomActorCertificateRequest) Reset() { *x = MintAteomActorCertificateRequest{} - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[106] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7816,7 +7862,7 @@ func (x *MintAteomActorCertificateRequest) String() string { func (*MintAteomActorCertificateRequest) ProtoMessage() {} func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[105] + mi := &file_ateapi_proto_msgTypes[106] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7829,7 +7875,7 @@ func (x *MintAteomActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintAteomActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{105} + return file_ateapi_proto_rawDescGZIP(), []int{106} } func (x *MintAteomActorCertificateRequest) GetActor() *ObjectRef { @@ -7866,7 +7912,7 @@ type MintAteomActorCertificateResponse struct { func (x *MintAteomActorCertificateResponse) Reset() { *x = MintAteomActorCertificateResponse{} - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[107] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7878,7 +7924,7 @@ func (x *MintAteomActorCertificateResponse) String() string { func (*MintAteomActorCertificateResponse) ProtoMessage() {} func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[106] + mi := &file_ateapi_proto_msgTypes[107] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7891,7 +7937,7 @@ func (x *MintAteomActorCertificateResponse) ProtoReflect() protoreflect.Message // Deprecated: Use MintAteomActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintAteomActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{106} + return file_ateapi_proto_rawDescGZIP(), []int{107} } func (x *MintAteomActorCertificateResponse) GetActorCertificates() [][]byte { @@ -7929,7 +7975,7 @@ type RequestActorSuspendRequest struct { func (x *RequestActorSuspendRequest) Reset() { *x = RequestActorSuspendRequest{} - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7941,7 +7987,7 @@ func (x *RequestActorSuspendRequest) String() string { func (*RequestActorSuspendRequest) ProtoMessage() {} func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[107] + mi := &file_ateapi_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7954,7 +8000,7 @@ func (x *RequestActorSuspendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendRequest.ProtoReflect.Descriptor instead. func (*RequestActorSuspendRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{107} + return file_ateapi_proto_rawDescGZIP(), []int{108} } func (x *RequestActorSuspendRequest) GetWorker() *ObjectRef { @@ -7988,7 +8034,7 @@ type RequestActorSuspendResponse struct { func (x *RequestActorSuspendResponse) Reset() { *x = RequestActorSuspendResponse{} - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8000,7 +8046,7 @@ func (x *RequestActorSuspendResponse) String() string { func (*RequestActorSuspendResponse) ProtoMessage() {} func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[108] + mi := &file_ateapi_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8013,7 +8059,7 @@ func (x *RequestActorSuspendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestActorSuspendResponse.ProtoReflect.Descriptor instead. func (*RequestActorSuspendResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{108} + return file_ateapi_proto_rawDescGZIP(), []int{109} } func (x *RequestActorSuspendResponse) GetActor() *Actor { @@ -8048,7 +8094,7 @@ type AccessPolicy struct { func (x *AccessPolicy) Reset() { *x = AccessPolicy{} - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8060,7 +8106,7 @@ func (x *AccessPolicy) String() string { func (*AccessPolicy) ProtoMessage() {} func (x *AccessPolicy) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[109] + mi := &file_ateapi_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8073,7 +8119,7 @@ func (x *AccessPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use AccessPolicy.ProtoReflect.Descriptor instead. func (*AccessPolicy) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{109} + return file_ateapi_proto_rawDescGZIP(), []int{110} } func (x *AccessPolicy) GetMetadata() *ResourceMetadata { @@ -8115,7 +8161,7 @@ type Binding struct { func (x *Binding) Reset() { *x = Binding{} - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8127,7 +8173,7 @@ func (x *Binding) String() string { func (*Binding) ProtoMessage() {} func (x *Binding) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[110] + mi := &file_ateapi_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8140,7 +8186,7 @@ func (x *Binding) ProtoReflect() protoreflect.Message { // Deprecated: Use Binding.ProtoReflect.Descriptor instead. func (*Binding) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{110} + return file_ateapi_proto_rawDescGZIP(), []int{111} } func (x *Binding) GetRole() string { @@ -8172,7 +8218,7 @@ type GetGlobalAccessPolicyRequest struct { func (x *GetGlobalAccessPolicyRequest) Reset() { *x = GetGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8184,7 +8230,7 @@ func (x *GetGlobalAccessPolicyRequest) String() string { func (*GetGlobalAccessPolicyRequest) ProtoMessage() {} func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[111] + mi := &file_ateapi_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8197,7 +8243,7 @@ func (x *GetGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{111} + return file_ateapi_proto_rawDescGZIP(), []int{112} } func (x *GetGlobalAccessPolicyRequest) GetName() string { @@ -8221,7 +8267,7 @@ type CreateGlobalAccessPolicyRequest struct { func (x *CreateGlobalAccessPolicyRequest) Reset() { *x = CreateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8233,7 +8279,7 @@ func (x *CreateGlobalAccessPolicyRequest) String() string { func (*CreateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[112] + mi := &file_ateapi_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8246,7 +8292,7 @@ func (x *CreateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{112} + return file_ateapi_proto_rawDescGZIP(), []int{113} } func (x *CreateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -8270,7 +8316,7 @@ type UpdateGlobalAccessPolicyRequest struct { func (x *UpdateGlobalAccessPolicyRequest) Reset() { *x = UpdateGlobalAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8282,7 +8328,7 @@ func (x *UpdateGlobalAccessPolicyRequest) String() string { func (*UpdateGlobalAccessPolicyRequest) ProtoMessage() {} func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[113] + mi := &file_ateapi_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8295,7 +8341,7 @@ func (x *UpdateGlobalAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateGlobalAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateGlobalAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{113} + return file_ateapi_proto_rawDescGZIP(), []int{114} } func (x *UpdateGlobalAccessPolicyRequest) GetAccessPolicy() *AccessPolicy { @@ -8325,7 +8371,7 @@ type GetAtespaceAccessPolicyRequest struct { func (x *GetAtespaceAccessPolicyRequest) Reset() { *x = GetAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8337,7 +8383,7 @@ func (x *GetAtespaceAccessPolicyRequest) String() string { func (*GetAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[114] + mi := &file_ateapi_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8350,7 +8396,7 @@ func (x *GetAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*GetAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{114} + return file_ateapi_proto_rawDescGZIP(), []int{115} } func (x *GetAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8386,7 +8432,7 @@ type CreateAtespaceAccessPolicyRequest struct { func (x *CreateAtespaceAccessPolicyRequest) Reset() { *x = CreateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8398,7 +8444,7 @@ func (x *CreateAtespaceAccessPolicyRequest) String() string { func (*CreateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[115] + mi := &file_ateapi_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8411,7 +8457,7 @@ func (x *CreateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use CreateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{115} + return file_ateapi_proto_rawDescGZIP(), []int{116} } func (x *CreateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8447,7 +8493,7 @@ type UpdateAtespaceAccessPolicyRequest struct { func (x *UpdateAtespaceAccessPolicyRequest) Reset() { *x = UpdateAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[116] + mi := &file_ateapi_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8459,7 +8505,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) String() string { func (*UpdateAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[116] + mi := &file_ateapi_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8472,7 +8518,7 @@ func (x *UpdateAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use UpdateAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{116} + return file_ateapi_proto_rawDescGZIP(), []int{117} } func (x *UpdateAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8513,7 +8559,7 @@ type DeleteAtespaceAccessPolicyRequest struct { func (x *DeleteAtespaceAccessPolicyRequest) Reset() { *x = DeleteAtespaceAccessPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[117] + mi := &file_ateapi_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8525,7 +8571,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) String() string { func (*DeleteAtespaceAccessPolicyRequest) ProtoMessage() {} func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[117] + mi := &file_ateapi_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8538,7 +8584,7 @@ func (x *DeleteAtespaceAccessPolicyRequest) ProtoReflect() protoreflect.Message // Deprecated: Use DeleteAtespaceAccessPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteAtespaceAccessPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{117} + return file_ateapi_proto_rawDescGZIP(), []int{118} } func (x *DeleteAtespaceAccessPolicyRequest) GetAtespace() *ObjectRef { @@ -8906,7 +8952,9 @@ const file_ateapi_proto_rawDesc = "" + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x12/\n" + "\aoptions\x18\x02 \x01(\v2\x15.ateapi.DeleteOptionsR\aoptions\"?\n" + "\x12DrainWorkerRequest\x12)\n" + - "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\"k\n" + + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\"=\n" + + "\x13DrainWorkerResponse\x12&\n" + + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"k\n" + "\x11ListActorsRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1b\n" + "\tpage_size\x18\x02 \x01(\x05R\bpageSize\x12\x1d\n" + @@ -9048,7 +9096,7 @@ const file_ateapi_proto_rawDesc = "" + "\vWorkerState\x12\x1c\n" + "\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n" + "\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n" + - "\x15WORKER_STATE_DRAINING\x10\x022\xfe\x18\n" + + "\x15WORKER_STATE_DRAINING\x10\x022\x8b\x19\n" + "\aControl\x124\n" + "\bGetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n" + "\vCreateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n" + @@ -9074,8 +9122,8 @@ const file_ateapi_proto_rawDesc = "" + "\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n" + "\fCreateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n" + "\fUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n" + - "\fDeleteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n" + - "\vDrainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n" + + "\fDeleteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12H\n" + + "\vDrainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x1b.ateapi.DrainWorkerResponse\"\x00\x12u\n" + "\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12E\n" + "\n" + "ListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12C\n" + @@ -9112,7 +9160,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 10) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 121) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 122) var file_ateapi_proto_goTypes = []any{ (SnapshotOwner)(0), // 0: ateapi.SnapshotOwner (SnapshotStorageStatus)(0), // 1: ateapi.SnapshotStorageStatus @@ -9218,34 +9266,35 @@ var file_ateapi_proto_goTypes = []any{ (*UpdateWorkerRequest)(nil), // 101: ateapi.UpdateWorkerRequest (*DeleteWorkerRequest)(nil), // 102: ateapi.DeleteWorkerRequest (*DrainWorkerRequest)(nil), // 103: ateapi.DrainWorkerRequest - (*ListActorsRequest)(nil), // 104: ateapi.ListActorsRequest - (*ListActorsResponse)(nil), // 105: ateapi.ListActorsResponse - (*Worker)(nil), // 106: ateapi.Worker - (*WorkerStatus)(nil), // 107: ateapi.WorkerStatus - (*SandboxRuntime)(nil), // 108: ateapi.SandboxRuntime - (*VersionedSandboxCompat)(nil), // 109: ateapi.VersionedSandboxCompat - (*AttributeEntry)(nil), // 110: ateapi.AttributeEntry - (*WorkerResources)(nil), // 111: ateapi.WorkerResources - (*ActorAssignment)(nil), // 112: ateapi.ActorAssignment - (*RegisterWorkerRequest)(nil), // 113: ateapi.RegisterWorkerRequest - (*RegisterWorkerResponse)(nil), // 114: ateapi.RegisterWorkerResponse - (*MintAteomActorCertificateRequest)(nil), // 115: ateapi.MintAteomActorCertificateRequest - (*MintAteomActorCertificateResponse)(nil), // 116: ateapi.MintAteomActorCertificateResponse - (*RequestActorSuspendRequest)(nil), // 117: ateapi.RequestActorSuspendRequest - (*RequestActorSuspendResponse)(nil), // 118: ateapi.RequestActorSuspendResponse - (*AccessPolicy)(nil), // 119: ateapi.AccessPolicy - (*Binding)(nil), // 120: ateapi.Binding - (*GetGlobalAccessPolicyRequest)(nil), // 121: ateapi.GetGlobalAccessPolicyRequest - (*CreateGlobalAccessPolicyRequest)(nil), // 122: ateapi.CreateGlobalAccessPolicyRequest - (*UpdateGlobalAccessPolicyRequest)(nil), // 123: ateapi.UpdateGlobalAccessPolicyRequest - (*GetAtespaceAccessPolicyRequest)(nil), // 124: ateapi.GetAtespaceAccessPolicyRequest - (*CreateAtespaceAccessPolicyRequest)(nil), // 125: ateapi.CreateAtespaceAccessPolicyRequest - (*UpdateAtespaceAccessPolicyRequest)(nil), // 126: ateapi.UpdateAtespaceAccessPolicyRequest - (*DeleteAtespaceAccessPolicyRequest)(nil), // 127: ateapi.DeleteAtespaceAccessPolicyRequest - nil, // 128: ateapi.Selector.MatchLabelsEntry - nil, // 129: ateapi.ExternalVolume.VolumeContextEntry - nil, // 130: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 131: google.protobuf.Timestamp + (*DrainWorkerResponse)(nil), // 104: ateapi.DrainWorkerResponse + (*ListActorsRequest)(nil), // 105: ateapi.ListActorsRequest + (*ListActorsResponse)(nil), // 106: ateapi.ListActorsResponse + (*Worker)(nil), // 107: ateapi.Worker + (*WorkerStatus)(nil), // 108: ateapi.WorkerStatus + (*SandboxRuntime)(nil), // 109: ateapi.SandboxRuntime + (*VersionedSandboxCompat)(nil), // 110: ateapi.VersionedSandboxCompat + (*AttributeEntry)(nil), // 111: ateapi.AttributeEntry + (*WorkerResources)(nil), // 112: ateapi.WorkerResources + (*ActorAssignment)(nil), // 113: ateapi.ActorAssignment + (*RegisterWorkerRequest)(nil), // 114: ateapi.RegisterWorkerRequest + (*RegisterWorkerResponse)(nil), // 115: ateapi.RegisterWorkerResponse + (*MintAteomActorCertificateRequest)(nil), // 116: ateapi.MintAteomActorCertificateRequest + (*MintAteomActorCertificateResponse)(nil), // 117: ateapi.MintAteomActorCertificateResponse + (*RequestActorSuspendRequest)(nil), // 118: ateapi.RequestActorSuspendRequest + (*RequestActorSuspendResponse)(nil), // 119: ateapi.RequestActorSuspendResponse + (*AccessPolicy)(nil), // 120: ateapi.AccessPolicy + (*Binding)(nil), // 121: ateapi.Binding + (*GetGlobalAccessPolicyRequest)(nil), // 122: ateapi.GetGlobalAccessPolicyRequest + (*CreateGlobalAccessPolicyRequest)(nil), // 123: ateapi.CreateGlobalAccessPolicyRequest + (*UpdateGlobalAccessPolicyRequest)(nil), // 124: ateapi.UpdateGlobalAccessPolicyRequest + (*GetAtespaceAccessPolicyRequest)(nil), // 125: ateapi.GetAtespaceAccessPolicyRequest + (*CreateAtespaceAccessPolicyRequest)(nil), // 126: ateapi.CreateAtespaceAccessPolicyRequest + (*UpdateAtespaceAccessPolicyRequest)(nil), // 127: ateapi.UpdateAtespaceAccessPolicyRequest + (*DeleteAtespaceAccessPolicyRequest)(nil), // 128: ateapi.DeleteAtespaceAccessPolicyRequest + nil, // 129: ateapi.Selector.MatchLabelsEntry + nil, // 130: ateapi.ExternalVolume.VolumeContextEntry + nil, // 131: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 132: google.protobuf.Timestamp } var file_ateapi_proto_depIdxs = []int32{ 1, // 0: ateapi.SnapshotStorage.status:type_name -> ateapi.SnapshotStorageStatus @@ -9253,11 +9302,11 @@ var file_ateapi_proto_depIdxs = []int32{ 0, // 2: ateapi.Snapshot.owner:type_name -> ateapi.SnapshotOwner 2, // 3: ateapi.Snapshot.fidelity:type_name -> ateapi.SnapshotFidelity 11, // 4: ateapi.Snapshot.durable_snapshot:type_name -> ateapi.SnapshotStorage - 128, // 5: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 131, // 6: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 131, // 7: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 129, // 5: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 132, // 6: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 132, // 7: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp 9, // 8: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 129, // 9: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 130, // 9: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry 14, // 10: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 33, // 11: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef 13, // 12: ateapi.Actor.worker_selector:type_name -> ateapi.Selector @@ -9281,7 +9330,7 @@ var file_ateapi_proto_depIdxs = []int32{ 15, // 30: ateapi.ActorStatus.external_volumes:type_name -> ateapi.ExternalVolume 28, // 31: ateapi.ActorStatus.crash:type_name -> ateapi.ActorCrash 12, // 32: ateapi.ActorStatus.snapshots:type_name -> ateapi.Snapshot - 131, // 33: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 132, // 33: ateapi.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp 33, // 34: ateapi.WorkerAssignment.worker:type_name -> ateapi.ObjectRef 12, // 35: ateapi.TagStatus.snapshot:type_name -> ateapi.Snapshot 14, // 36: ateapi.Tag.metadata:type_name -> ateapi.ResourceMetadata @@ -9299,7 +9348,7 @@ var file_ateapi_proto_depIdxs = []int32{ 38, // 48: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus 36, // 49: ateapi.Resources.limits:type_name -> ateapi.Limits 33, // 50: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef - 131, // 51: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp + 132, // 51: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp 37, // 52: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 5, // 53: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass 2, // 54: ateapi.SnapshotConfig.preferred_fidelity:type_name -> ateapi.SnapshotFidelity @@ -9353,7 +9402,7 @@ var file_ateapi_proto_depIdxs = []int32{ 94, // 102: ateapi.DeleteActorEgressPolicyRequest.options:type_name -> ateapi.DeleteOptions 33, // 103: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef 33, // 104: ateapi.MintActorJWTRequest.actor:type_name -> ateapi.ObjectRef - 131, // 105: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp + 132, // 105: ateapi.MintActorJWTResponse.expires_at:type_name -> google.protobuf.Timestamp 33, // 106: ateapi.MintActorCertificateRequest.actor:type_name -> ateapi.ObjectRef 31, // 107: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag 31, // 108: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag @@ -9361,144 +9410,145 @@ var file_ateapi_proto_depIdxs = []int32{ 33, // 110: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef 94, // 111: ateapi.DeleteTagRequest.options:type_name -> ateapi.DeleteOptions 33, // 112: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef - 112, // 113: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment - 106, // 114: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 113, // 113: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 107, // 114: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker 33, // 115: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef - 106, // 116: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker - 106, // 117: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker + 107, // 116: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker + 107, // 117: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker 33, // 118: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef 94, // 119: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions 33, // 120: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef - 16, // 121: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 14, // 122: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 130, // 123: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 107, // 124: ateapi.Worker.status:type_name -> ateapi.WorkerStatus - 8, // 125: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 111, // 126: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources - 111, // 127: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources - 108, // 128: ateapi.WorkerStatus.default_runtime:type_name -> ateapi.SandboxRuntime - 108, // 129: ateapi.WorkerStatus.restorable_runtimes:type_name -> ateapi.SandboxRuntime - 109, // 130: ateapi.SandboxRuntime.version:type_name -> ateapi.VersionedSandboxCompat - 110, // 131: ateapi.VersionedSandboxCompat.attributes:type_name -> ateapi.AttributeEntry - 35, // 132: ateapi.WorkerResources.resources:type_name -> ateapi.Resources - 14, // 133: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata - 33, // 134: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 35, // 135: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 33, // 136: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef - 111, // 137: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources - 108, // 138: ateapi.RegisterWorkerRequest.default_runtime:type_name -> ateapi.SandboxRuntime - 108, // 139: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime - 106, // 140: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker - 33, // 141: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef - 33, // 142: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef - 33, // 143: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef - 16, // 144: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor - 14, // 145: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata - 120, // 146: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding - 119, // 147: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 119, // 148: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 33, // 149: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 33, // 150: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 119, // 151: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 33, // 152: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 119, // 153: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy - 33, // 154: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef - 94, // 155: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions - 68, // 156: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 69, // 157: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 70, // 158: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 71, // 159: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 73, // 160: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 75, // 161: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 77, // 162: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest - 79, // 163: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 80, // 164: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 81, // 165: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 82, // 166: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 83, // 167: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 85, // 168: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest - 87, // 169: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest - 91, // 170: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 84, // 171: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 89, // 172: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 92, // 173: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 93, // 174: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 97, // 175: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 99, // 176: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 100, // 177: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 101, // 178: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 102, // 179: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 103, // 180: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 95, // 181: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 104, // 182: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 58, // 183: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 59, // 184: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 60, // 185: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 62, // 186: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 63, // 187: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 64, // 188: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 65, // 189: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 67, // 190: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 121, // 191: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest - 122, // 192: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest - 123, // 193: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest - 124, // 194: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest - 125, // 195: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest - 126, // 196: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest - 127, // 197: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest - 113, // 198: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest - 115, // 199: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest - 117, // 200: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest - 16, // 201: ateapi.Control.GetActor:output_type -> ateapi.Actor - 16, // 202: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 16, // 203: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 72, // 204: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 74, // 205: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 76, // 206: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 78, // 207: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse - 16, // 208: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 17, // 209: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 17, // 210: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 17, // 211: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 17, // 212: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 86, // 213: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse - 88, // 214: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse - 31, // 215: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 31, // 216: ateapi.Control.GetTag:output_type -> ateapi.Tag - 90, // 217: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 31, // 218: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 31, // 219: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 98, // 220: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 106, // 221: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 106, // 222: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 106, // 223: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 106, // 224: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 106, // 225: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 96, // 226: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 105, // 227: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 32, // 228: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 32, // 229: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 61, // 230: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 32, // 231: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 34, // 232: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 34, // 233: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 66, // 234: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 34, // 235: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 119, // 236: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 237: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 238: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 239: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 240: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 241: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 119, // 242: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy - 114, // 243: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse - 116, // 244: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse - 118, // 245: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse - 201, // [201:246] is the sub-list for method output_type - 156, // [156:201] is the sub-list for method input_type - 156, // [156:156] is the sub-list for extension type_name - 156, // [156:156] is the sub-list for extension extendee - 0, // [0:156] is the sub-list for field type_name + 107, // 121: ateapi.DrainWorkerResponse.worker:type_name -> ateapi.Worker + 16, // 122: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 14, // 123: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata + 131, // 124: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 108, // 125: ateapi.Worker.status:type_name -> ateapi.WorkerStatus + 8, // 126: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState + 112, // 127: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources + 112, // 128: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources + 109, // 129: ateapi.WorkerStatus.default_runtime:type_name -> ateapi.SandboxRuntime + 109, // 130: ateapi.WorkerStatus.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 110, // 131: ateapi.SandboxRuntime.version:type_name -> ateapi.VersionedSandboxCompat + 111, // 132: ateapi.VersionedSandboxCompat.attributes:type_name -> ateapi.AttributeEntry + 35, // 133: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 14, // 134: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 33, // 135: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 35, // 136: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 33, // 137: ateapi.RegisterWorkerRequest.worker:type_name -> ateapi.ObjectRef + 112, // 138: ateapi.RegisterWorkerRequest.capacity:type_name -> ateapi.WorkerResources + 109, // 139: ateapi.RegisterWorkerRequest.default_runtime:type_name -> ateapi.SandboxRuntime + 109, // 140: ateapi.RegisterWorkerRequest.restorable_runtimes:type_name -> ateapi.SandboxRuntime + 107, // 141: ateapi.RegisterWorkerResponse.worker:type_name -> ateapi.Worker + 33, // 142: ateapi.MintAteomActorCertificateRequest.actor:type_name -> ateapi.ObjectRef + 33, // 143: ateapi.RequestActorSuspendRequest.worker:type_name -> ateapi.ObjectRef + 33, // 144: ateapi.RequestActorSuspendRequest.actor:type_name -> ateapi.ObjectRef + 16, // 145: ateapi.RequestActorSuspendResponse.actor:type_name -> ateapi.Actor + 14, // 146: ateapi.AccessPolicy.metadata:type_name -> ateapi.ResourceMetadata + 121, // 147: ateapi.AccessPolicy.bindings:type_name -> ateapi.Binding + 120, // 148: ateapi.CreateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 120, // 149: ateapi.UpdateGlobalAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 33, // 150: ateapi.GetAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 33, // 151: ateapi.CreateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 120, // 152: ateapi.CreateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 33, // 153: ateapi.UpdateAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 120, // 154: ateapi.UpdateAtespaceAccessPolicyRequest.access_policy:type_name -> ateapi.AccessPolicy + 33, // 155: ateapi.DeleteAtespaceAccessPolicyRequest.atespace:type_name -> ateapi.ObjectRef + 94, // 156: ateapi.DeleteAtespaceAccessPolicyRequest.options:type_name -> ateapi.DeleteOptions + 68, // 157: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 69, // 158: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 70, // 159: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 71, // 160: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 73, // 161: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 75, // 162: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 77, // 163: ateapi.Control.RevertActor:input_type -> ateapi.RevertActorRequest + 79, // 164: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 80, // 165: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 81, // 166: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 82, // 167: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 83, // 168: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 85, // 169: ateapi.Control.MintActorJWT:input_type -> ateapi.MintActorJWTRequest + 87, // 170: ateapi.Control.MintActorCertificate:input_type -> ateapi.MintActorCertificateRequest + 91, // 171: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 84, // 172: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 89, // 173: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 92, // 174: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 93, // 175: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 97, // 176: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 99, // 177: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 100, // 178: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 101, // 179: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 102, // 180: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 103, // 181: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 95, // 182: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 105, // 183: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 58, // 184: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 59, // 185: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 60, // 186: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 62, // 187: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 63, // 188: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 64, // 189: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 65, // 190: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 67, // 191: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 122, // 192: ateapi.Control.GetGlobalAccessPolicy:input_type -> ateapi.GetGlobalAccessPolicyRequest + 123, // 193: ateapi.Control.CreateGlobalAccessPolicy:input_type -> ateapi.CreateGlobalAccessPolicyRequest + 124, // 194: ateapi.Control.UpdateGlobalAccessPolicy:input_type -> ateapi.UpdateGlobalAccessPolicyRequest + 125, // 195: ateapi.Control.GetAtespaceAccessPolicy:input_type -> ateapi.GetAtespaceAccessPolicyRequest + 126, // 196: ateapi.Control.CreateAtespaceAccessPolicy:input_type -> ateapi.CreateAtespaceAccessPolicyRequest + 127, // 197: ateapi.Control.UpdateAtespaceAccessPolicy:input_type -> ateapi.UpdateAtespaceAccessPolicyRequest + 128, // 198: ateapi.Control.DeleteAtespaceAccessPolicy:input_type -> ateapi.DeleteAtespaceAccessPolicyRequest + 114, // 199: ateapi.WorkerService.RegisterWorker:input_type -> ateapi.RegisterWorkerRequest + 116, // 200: ateapi.WorkerService.MintAteomActorCertificate:input_type -> ateapi.MintAteomActorCertificateRequest + 118, // 201: ateapi.WorkerService.RequestActorSuspend:input_type -> ateapi.RequestActorSuspendRequest + 16, // 202: ateapi.Control.GetActor:output_type -> ateapi.Actor + 16, // 203: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 16, // 204: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 72, // 205: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 74, // 206: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 76, // 207: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 78, // 208: ateapi.Control.RevertActor:output_type -> ateapi.RevertActorResponse + 16, // 209: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 17, // 210: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 17, // 211: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 17, // 212: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 17, // 213: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 86, // 214: ateapi.Control.MintActorJWT:output_type -> ateapi.MintActorJWTResponse + 88, // 215: ateapi.Control.MintActorCertificate:output_type -> ateapi.MintActorCertificateResponse + 31, // 216: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 31, // 217: ateapi.Control.GetTag:output_type -> ateapi.Tag + 90, // 218: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 31, // 219: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 31, // 220: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 98, // 221: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 107, // 222: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 107, // 223: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 107, // 224: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 107, // 225: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 104, // 226: ateapi.Control.DrainWorker:output_type -> ateapi.DrainWorkerResponse + 96, // 227: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 106, // 228: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 32, // 229: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 32, // 230: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 61, // 231: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 32, // 232: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 34, // 233: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 34, // 234: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 66, // 235: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 34, // 236: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 120, // 237: ateapi.Control.GetGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 120, // 238: ateapi.Control.CreateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 120, // 239: ateapi.Control.UpdateGlobalAccessPolicy:output_type -> ateapi.AccessPolicy + 120, // 240: ateapi.Control.GetAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 120, // 241: ateapi.Control.CreateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 120, // 242: ateapi.Control.UpdateAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 120, // 243: ateapi.Control.DeleteAtespaceAccessPolicy:output_type -> ateapi.AccessPolicy + 115, // 244: ateapi.WorkerService.RegisterWorker:output_type -> ateapi.RegisterWorkerResponse + 117, // 245: ateapi.WorkerService.MintAteomActorCertificate:output_type -> ateapi.MintAteomActorCertificateResponse + 119, // 246: ateapi.WorkerService.RequestActorSuspend:output_type -> ateapi.RequestActorSuspendResponse + 202, // [202:247] is the sub-list for method output_type + 157, // [157:202] is the sub-list for method input_type + 157, // [157:157] is the sub-list for extension type_name + 157, // [157:157] is the sub-list for extension extendee + 0, // [0:157] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -9512,7 +9562,7 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 10, - NumMessages: 121, + NumMessages: 122, NumExtensions: 0, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index a0269940fb..bd8ca8d230 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -113,7 +113,7 @@ service Control { // Mark a Worker as terminating so the scheduler stops routing new Actors to // it. Idempotent; one-way. Deliberately leaves any bound Actor alone. // Returns ABORTED if another write lands on the Worker first; retry. - rpc DrainWorker(DrainWorkerRequest) returns (Worker) {} + rpc DrainWorker(DrainWorkerRequest) returns (DrainWorkerResponse) {} // List the Actors hosted by a given Worker. rpc ListWorkerActorAssignments(ListWorkerActorAssignmentsRequest) returns (ListWorkerActorAssignmentsResponse) {} @@ -2072,6 +2072,12 @@ message DrainWorkerRequest { ObjectRef worker = 1; } +// DrainWorkerResponse holds the drained Worker. +message DrainWorkerResponse { + // The worker being drained. + Worker worker = 1; +} + // List actors with pagination. Provides "soft" guarantees: actors may be // missed or duplicated if the system state changes during pagination. message ListActorsRequest { diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 682ebb0e46..6242d56047 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -150,7 +150,7 @@ type ControlClient interface { // Mark a Worker as terminating so the scheduler stops routing new Actors to // it. Idempotent; one-way. Deliberately leaves any bound Actor alone. // Returns ABORTED if another write lands on the Worker first; retry. - DrainWorker(ctx context.Context, in *DrainWorkerRequest, opts ...grpc.CallOption) (*Worker, error) + DrainWorker(ctx context.Context, in *DrainWorkerRequest, opts ...grpc.CallOption) (*DrainWorkerResponse, error) // List the Actors hosted by a given Worker. ListWorkerActorAssignments(ctx context.Context, in *ListWorkerActorAssignmentsRequest, opts ...grpc.CallOption) (*ListWorkerActorAssignmentsResponse, error) // List Actors. @@ -439,9 +439,9 @@ func (c *controlClient) DeleteWorker(ctx context.Context, in *DeleteWorkerReques return out, nil } -func (c *controlClient) DrainWorker(ctx context.Context, in *DrainWorkerRequest, opts ...grpc.CallOption) (*Worker, error) { +func (c *controlClient) DrainWorker(ctx context.Context, in *DrainWorkerRequest, opts ...grpc.CallOption) (*DrainWorkerResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(Worker) + out := new(DrainWorkerResponse) err := c.cc.Invoke(ctx, Control_DrainWorker_FullMethodName, in, out, cOpts...) if err != nil { return nil, err @@ -692,7 +692,7 @@ type ControlServer interface { // Mark a Worker as terminating so the scheduler stops routing new Actors to // it. Idempotent; one-way. Deliberately leaves any bound Actor alone. // Returns ABORTED if another write lands on the Worker first; retry. - DrainWorker(context.Context, *DrainWorkerRequest) (*Worker, error) + DrainWorker(context.Context, *DrainWorkerRequest) (*DrainWorkerResponse, error) // List the Actors hosted by a given Worker. ListWorkerActorAssignments(context.Context, *ListWorkerActorAssignmentsRequest) (*ListWorkerActorAssignmentsResponse, error) // List Actors. @@ -813,7 +813,7 @@ func (UnimplementedControlServer) UpdateWorker(context.Context, *UpdateWorkerReq func (UnimplementedControlServer) DeleteWorker(context.Context, *DeleteWorkerRequest) (*Worker, error) { return nil, status.Error(codes.Unimplemented, "method DeleteWorker not implemented") } -func (UnimplementedControlServer) DrainWorker(context.Context, *DrainWorkerRequest) (*Worker, error) { +func (UnimplementedControlServer) DrainWorker(context.Context, *DrainWorkerRequest) (*DrainWorkerResponse, error) { return nil, status.Error(codes.Unimplemented, "method DrainWorker not implemented") } func (UnimplementedControlServer) ListWorkerActorAssignments(context.Context, *ListWorkerActorAssignmentsRequest) (*ListWorkerActorAssignmentsResponse, error) { From c2052e45bfd953ea2ed7a7568d84575cca90eb72 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Mon, 12 Oct 2026 20:06:44 +0000 Subject: [PATCH 08/20] gvisor: move the sandbox asset to release 20260907.0 (#2413) Addresses https://github.com/agent-substrate/substrate/issues/2377. ## What we saw Under CPU contention (14 sandboxes per worker pod, each opening about a thousand tunnels), 1 to 8% of gVisor sandboxes died mid-run. The actor stayed `RUNNING`, `SuspendActor` succeeded, and the next resume failed with `inconsistent private memory files on restore`. Sentry logs showed `Stub thread no longer exists; killing syscall thread`. Across the ladder runs on the nightly we lost 1/299, 17/968 and 79/999 actors, and 79 and 73 of 1000 in two seven-minute repros. ## Why the bump helps The 2026-09-02 nightly predates gVisor commit `80bb741691be` ("Close exposed race window in stub ThreadID setting"): a sentry that interrupts a stub thread mid context switch can read an invalid thread id and kill the whole subprocess. Release 20260907.0 is the first tagged release containing it (GitHub compare: 25 commits ahead of the fix, 0 behind). The same repro on 20260907.0 lost 0 of 1000 actors, as did the 34-minute 1000 x 1000 rung of the egress ladder. ## Change `manifests/ate-install/sandboxconfig-gvisor.yaml` gains version `v2` with release 20260907.0 as `defaultVersion`; the nightly entry is removed. `docs/api-guide.md` mirrors it and now states that a resume resolves the default version at restore time, so after this rolls out recut golden snapshots and expect snapshots captured under the nightly to fail on their next resume. ## Test signals - `go test ./pkg/api/v1alpha1/` applies the edited manifest against the CRD and the admission policy in envtest; `./cmd/ate-setup/...`, `./cmd/atelet/...` and the ateapi sandbox tests pass. - CI `e2e-test` installs this manifest through `ate-setup` and boots, suspends and restores gVisor actors, so a green run exercises the release. - Both sha256 values were recomputed from the downloaded tarballs and match the published `.sha512` files. - The loss counts above come from the issue's gist and the ladder run, not re-verified in this PR. Prepared with help from an AI coding agent; reviewed and verified by the author. --- docs/api-guide.md | 14 +++++++------- manifests/ate-install/sandboxconfig-gvisor.yaml | 15 +++++++++------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/api-guide.md b/docs/api-guide.md index cdf0766025..c28a038cd5 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -436,7 +436,7 @@ One consequence worth planning for: **a published snapshot is read from the ates `SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the default version of the config its `ActorTemplate` names via `sandboxConfig.configName`. -This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place. +This means a single, cluster-managed config pins the sandbox runtime version for many templates, and operators upgrade the runtime in one place. A resume also resolves the default version at restore time: the snapshot manifest records the assets it was captured with, but a restore runs the current default, so changing `defaultVersion` applies to every existing snapshot on its next resume. gVisor does not promise that a checkpoint restores under a different release, so after a version change recut the golden snapshots and expect snapshots captured under the old version to fail on their next resume. ### Specification (`SandboxConfigSpec`) @@ -492,20 +492,20 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v1 + defaultVersion: v2 versions: - - name: v1 + - name: v2 state: Enabled pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + url: "gs://gvisor/releases/release/20260907.0/x86_64/gvisor.tar.zstd" + sha256: "e32ed48a2ddc7c0ef5e922ff5d52e33b6fccc75b61fd4c40a0bd9bdfe90032e6" arm64: gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + url: "gs://gvisor/releases/release/20260907.0/aarch64/gvisor.tar.zstd" + sha256: "f3ba93a1a83dd4b6a322237030cdf81a278d9ef4e01ae6218ba69fbc479fdfb2" ``` ### Micro-VM SandboxConfig diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 77f3654bf9..96164f8c54 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -27,9 +27,12 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v1 + defaultVersion: v2 versions: - - name: v1 + # gVisor release 20260907.0, the first tagged release with commit + # 80bb741691be, which closes a systrap race that killed sandboxes under CPU + # contention. + - name: v2 state: Enabled # The root sandbox container's image. On GCP, prefer the in-project mirror # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. @@ -37,9 +40,9 @@ spec: assets: amd64: gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" - sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + url: "gs://gvisor/releases/release/20260907.0/x86_64/gvisor.tar.zstd" + sha256: "e32ed48a2ddc7c0ef5e922ff5d52e33b6fccc75b61fd4c40a0bd9bdfe90032e6" arm64: gvisor: - url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" - sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" + url: "gs://gvisor/releases/release/20260907.0/aarch64/gvisor.tar.zstd" + sha256: "f3ba93a1a83dd4b6a322237030cdf81a278d9ef4e01ae6218ba69fbc479fdfb2" From 579cfcb64300876f564f3e3b4684e2a79b47601d Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Mon, 12 Oct 2026 20:43:29 +0000 Subject: [PATCH 09/20] `ate-setup delete ate-system` now deletes the ClusterTrustBundles Substrate published (#2418) ClusterTrustBundles are cluster-scoped, so deleting the ate-system and podcertificate-controller namespaces left Substrate's four bundles behind. - Once both namespaces are gone (so the controllers can't republish), teardown deletes exactly these bundles, each only when its signer also matches: - `podidentity.podcert.ate.dev:identity:primary-bundle` (signer `podidentity.podcert.ate.dev/identity`) - `servicedns.podcert.ate.dev:identity:primary-bundle` (signer `servicedns.podcert.ate.dev/identity`) - `postgres.podcert.ate.dev:identity:primary-bundle` (signer `postgres.podcert.ate.dev/identity`) - `egress-mitm.ate.dev:mitm:primary-bundle` (signer `egress-mitm.ate.dev/mitm`) - A foreign bundle under one of these signers with another name is kept. Tested with unit tests and a kind install plus teardown: the four bundles were deleted; the kube-apiserver bundle and foreign bundles, including two under our signers, were kept. --- cmd/ate-setup/internal/steps/delete.go | 63 +++++- cmd/ate-setup/internal/steps/delete_test.go | 228 ++++++++++++++++++++ internal/clustertrustbundle/client.go | 13 +- internal/clustertrustbundle/client_test.go | 12 +- 4 files changed, 306 insertions(+), 10 deletions(-) diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 92c59f3c1c..74568bf8b4 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -16,6 +16,7 @@ package steps import ( "context" + "errors" "fmt" "slices" @@ -26,6 +27,7 @@ import ( "k8s.io/client-go/util/retry" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" + "github.com/agent-substrate/substrate/internal/clustertrustbundle" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -36,7 +38,8 @@ import ( // bundle: which of them the install created depends on the router and // credential provider that were selected, and teardown must not depend on // remembering that. The provider goes first so that a later failure cannot -// leave its ClusterRole and binding behind. +// leave its ClusterRole and binding behind. The cluster-scoped trust bundles +// go last, once the controllers that publish them are gone. func (e *Env) DeleteAteSystem(ctx context.Context) error { log.Step("delete_ate_system") @@ -76,7 +79,8 @@ func (e *Env) DeleteAteSystem(ctx context.Context) error { return err } } - if err := e.Kube.WaitDeleted(ctx, schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}, "", e.Namespace(), e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil { + namespace := schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} + if err := e.Kube.WaitDeleted(ctx, namespace, "", e.Namespace(), e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil { return err } // Removing the protection finalizer by hand is safe here. atecontroller, @@ -85,6 +89,14 @@ func (e *Env) DeleteAteSystem(ctx context.Context) error { if err := releaseSandboxConfigs(ctx, e.Kube.Dynamic.Resource(atev1alpha1.GroupVersion.WithResource("sandboxconfigs"))); err != nil { return err } + // The podcertificate controller republishes its bundles every few + // seconds, so they can only be deleted once it is gone. + if err := e.Kube.WaitDeleted(ctx, namespace, "", NamespacePodCert, e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil { + return err + } + if err := e.DeleteTrustBundles(ctx); err != nil { + return err + } return e.UnlabelNodesSubstrateVersion(ctx) } @@ -124,6 +136,53 @@ func releaseSandboxConfigs(ctx context.Context, res dynamic.ResourceInterface) e return nil } +// substrateTrustBundles maps the name of each ClusterTrustBundle Substrate +// publishes to its signer: the podcertificate controller's (see +// trustBundleNames) and atecontroller's egress MITM bundle +// (egressMITMTrustBundleName and egressMITMSignerName in +// cmd/atecontroller/internal/controllers, which ate-setup cannot import). +var substrateTrustBundles = map[string]string{ + "podidentity.podcert.ate.dev:identity:primary-bundle": "podidentity.podcert.ate.dev/identity", + "servicedns.podcert.ate.dev:identity:primary-bundle": "servicedns.podcert.ate.dev/identity", + "postgres.podcert.ate.dev:identity:primary-bundle": "postgres.podcert.ate.dev/identity", + "egress-mitm.ate.dev:mitm:primary-bundle": "egress-mitm.ate.dev/mitm", +} + +// DeleteTrustBundles deletes the ClusterTrustBundles in substrateTrustBundles. +// They are cluster-scoped, so deleting the namespaces of the controllers that +// publish them leaves them behind. +// +// A bundle is deleted only when both its name and its signer match. Anyone +// allowed to attest for one of these signers may publish a bundle under a +// different name, so the signer alone does not show that Substrate created it. +func (e *Env) DeleteTrustBundles(ctx context.Context) error { + bundles, err := clustertrustbundle.NewClient(e.Kube.Typed, nil) + if errors.Is(err, clustertrustbundle.ErrNotServed) { + return nil + } + if err != nil { + return fmt.Errorf("while discovering the ClusterTrustBundle API: %w", err) + } + list, err := bundles.List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("while listing ClusterTrustBundles: %w", err) + } + for _, bundle := range list.Items { + if signer, ok := substrateTrustBundles[bundle.Name]; !ok || bundle.Spec.SignerName != signer { + continue + } + opts := metav1.DeleteOptions{Preconditions: metav1.NewUIDPreconditions(string(bundle.UID))} + if err := bundles.Delete(ctx, bundle.Name, opts); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return fmt.Errorf("while deleting ClusterTrustBundle %s: %w", bundle.Name, err) + } + log.Infof("Deleted ClusterTrustBundle %s", bundle.Name) + } + return nil +} + // DeleteAtenet removes the atenet dataplane and the bundled credential // provider, deleting the provider first so that a later failure cannot leave // its ClusterRole and binding behind. The provider's policy ConfigMap stays: diff --git a/cmd/ate-setup/internal/steps/delete_test.go b/cmd/ate-setup/internal/steps/delete_test.go index f516ed111b..0ff7eb6ee8 100644 --- a/cmd/ate-setup/internal/steps/delete_test.go +++ b/cmd/ate-setup/internal/steps/delete_test.go @@ -24,13 +24,21 @@ import ( "strings" "testing" + certsv1 "k8s.io/api/certificates/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" "k8s.io/client-go/util/retry" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" + "github.com/agent-substrate/substrate/internal/clustertrustbundle" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -270,3 +278,223 @@ func TestDeleteDemos(t *testing.T) { }) } } + +// trustBundle returns a bundle for signer the way the API server requires it +// to be named: the signer name with "/" replaced by ":", then a suffix. +func trustBundle(signer, suffix string) *certsv1.ClusterTrustBundle { + name := suffix + if signer != "" { + name = strings.ReplaceAll(signer, "/", ":") + ":" + suffix + } + return &certsv1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID("uid-" + name), + Labels: map[string]string{"podcert.ate.dev/canarying": "live"}, + }, + Spec: certsv1.ClusterTrustBundleSpec{SignerName: signer, TrustBundle: "pem"}, + } +} + +// trustBundleKube serves ClusterTrustBundles at version, seeded with bundles. +// An empty version serves no ClusterTrustBundle API at all. +func trustBundleKube(t *testing.T, version string, bundles ...*certsv1.ClusterTrustBundle) (*fake.Clientset, *clustertrustbundle.Client) { + t.Helper() + kc := fake.NewSimpleClientset() + if version == "" { + return kc, nil + } + kc.Resources = []*metav1.APIResourceList{{ + GroupVersion: "certificates.k8s.io/" + version, + APIResources: []metav1.APIResource{{Name: "clustertrustbundles"}}, + }} + c, err := clustertrustbundle.NewClient(kc, nil) + if err != nil { + t.Fatal(err) + } + for _, b := range bundles { + if _, err := c.Create(t.Context(), b, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + kc.ClearActions() + return kc, c +} + +func remainingBundles(t *testing.T, c *clustertrustbundle.Client) []string { + t.Helper() + list, err := c.List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + var names []string + for _, b := range list.Items { + names = append(names, b.Name) + } + slices.Sort(names) + return names +} + +func TestDeleteTrustBundles(t *testing.T) { + for _, version := range []string{"v1", "v1beta1"} { + t.Run(version, func(t *testing.T) { + var ours []*certsv1.ClusterTrustBundle + for name, signer := range substrateTrustBundles { + b := trustBundle(signer, "primary-bundle") + if b.Name != name { + t.Fatalf("test bundle for %s is named %s, want %s", signer, b.Name, name) + } + ours = append(ours, b) + } + // Others' bundles stay: ones under our signers with another + // name, as a CA rotation or another publisher might create, one + // that shares a domain suffix with ours, one that carries the + // label our consumers select on, and one with no signer. + others := []*certsv1.ClusterTrustBundle{ + trustBundle("podidentity.podcert.ate.dev/identity", "next-bundle"), + trustBundle("egress-mitm.ate.dev/mitm", "foreign-bundle"), + trustBundle("example.com/signer", "primary-bundle"), + trustBundle("other.podcert.ate.dev/identity", "primary-bundle"), + trustBundle("podidentity.podcert.ate.dev/other", "primary-bundle"), + trustBundle("", "unsigned-bundle"), + } + kc, c := trustBundleKube(t, version, append(slices.Clone(ours), others...)...) + e := &Env{Kube: &kube.Client{Typed: kc}} + + if err := e.DeleteTrustBundles(t.Context()); err != nil { + t.Fatalf("DeleteTrustBundles() = %v", err) + } + var want []string + for _, b := range others { + want = append(want, b.Name) + } + slices.Sort(want) + if got := remainingBundles(t, c); !slices.Equal(got, want) { + t.Errorf("remaining bundles = %q, want %q", got, want) + } + + // Each delete is pinned to the UID that was listed, and goes to + // the served version. + var deleted []string + for _, a := range kc.Actions() { + del, ok := a.(ktesting.DeleteActionImpl) + if !ok { + continue + } + if got := del.GetResource().Version; got != version { + t.Errorf("delete of %s went to %s, want %s", del.Name, got, version) + } + pre := del.DeleteOptions.Preconditions + if pre == nil || pre.UID == nil || *pre.UID != types.UID("uid-"+del.Name) { + t.Errorf("delete of %s has preconditions %+v, want UID uid-%s", del.Name, pre, del.Name) + } + deleted = append(deleted, del.Name) + } + if len(deleted) != len(ours) { + t.Errorf("deleted %q, want %d bundles", deleted, len(ours)) + } + + // A rerun finds nothing left of ours and changes nothing. + kc.ClearActions() + if err := e.DeleteTrustBundles(t.Context()); err != nil { + t.Fatalf("second DeleteTrustBundles() = %v", err) + } + for _, a := range kc.Actions() { + if a.GetVerb() == "delete" { + t.Errorf("second run issued %v", a) + } + } + if got := remainingBundles(t, c); !slices.Equal(got, want) { + t.Errorf("after rerun, remaining bundles = %q, want %q", got, want) + } + }) + } +} + +// A bundle with one of our names but another signer is not ours. The API +// server does not admit one, so this guards the check itself. +func TestDeleteTrustBundlesNameWithWrongSigner(t *testing.T) { + foreign := trustBundle("example.com/signer", "primary-bundle") + foreign.Name = "postgres.podcert.ate.dev:identity:primary-bundle" + kc, c := trustBundleKube(t, "v1", foreign) + e := &Env{Kube: &kube.Client{Typed: kc}} + if err := e.DeleteTrustBundles(t.Context()); err != nil { + t.Fatalf("DeleteTrustBundles() = %v", err) + } + if got := remainingBundles(t, c); !slices.Equal(got, []string{foreign.Name}) { + t.Errorf("remaining bundles = %q, want %q", got, foreign.Name) + } +} + +func TestDeleteTrustBundlesNoneOfOurs(t *testing.T) { + foreign := trustBundle("example.com/signer", "primary-bundle") + kc, c := trustBundleKube(t, "v1", foreign) + e := &Env{Kube: &kube.Client{Typed: kc}} + if err := e.DeleteTrustBundles(t.Context()); err != nil { + t.Fatalf("DeleteTrustBundles() = %v", err) + } + for _, a := range kc.Actions() { + if a.GetVerb() == "delete" { + t.Errorf("issued %v with none of ours present", a) + } + } + if got := remainingBundles(t, c); !slices.Equal(got, []string{foreign.Name}) { + t.Errorf("remaining bundles = %q, want %q", got, foreign.Name) + } +} + +// A cluster that serves no ClusterTrustBundle API cannot hold any of ours. +func TestDeleteTrustBundlesAPINotServed(t *testing.T) { + kc, _ := trustBundleKube(t, "") + e := &Env{Kube: &kube.Client{Typed: kc}} + if err := e.DeleteTrustBundles(t.Context()); err != nil { + t.Fatalf("DeleteTrustBundles() = %v, want nil", err) + } + for _, a := range kc.Actions() { + if a.GetResource().Resource == "clustertrustbundles" { + t.Errorf("issued %v without a ClusterTrustBundle API", a) + } + } +} + +func TestDeleteTrustBundlesErrors(t *testing.T) { + denied := apierrors.NewForbidden(schema.GroupResource{Group: "certificates.k8s.io", Resource: "clustertrustbundles"}, "", fmt.Errorf("denied")) + for _, tc := range []struct { + name string + verb string + err error + wantErr bool + }{ + // Deleted by someone else between the list and the delete. + {name: "already gone", verb: "delete", err: apierrors.NewNotFound(schema.GroupResource{}, ""), wantErr: false}, + {name: "delete denied", verb: "delete", err: denied, wantErr: true}, + {name: "list denied", verb: "list", err: denied, wantErr: true}, + {name: "discovery fails", verb: "get", err: apierrors.NewServiceUnavailable("unavailable"), wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + kc, _ := trustBundleKube(t, "v1", trustBundle("podidentity.podcert.ate.dev/identity", "primary-bundle")) + resource := "clustertrustbundles" + if tc.verb == "get" { + // Discovery goes through the fake's "get resource" action. + resource = "resource" + } + kc.PrependReactor(tc.verb, resource, func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, tc.err + }) + e := &Env{Kube: &kube.Client{Typed: kc}} + err := e.DeleteTrustBundles(t.Context()) + if (err != nil) != tc.wantErr { + t.Fatalf("DeleteTrustBundles() = %v, want error %t", err, tc.wantErr) + } + }) + } +} + +// Every bundle install waits for must be one teardown deletes. +func TestSubstrateTrustBundlesCoverInstalledBundles(t *testing.T) { + for _, name := range trustBundleNames { + if _, ok := substrateTrustBundles[name]; !ok { + t.Errorf("teardown does not delete ClusterTrustBundle %s", name) + } + } +} diff --git a/internal/clustertrustbundle/client.go b/internal/clustertrustbundle/client.go index bdd1a5930c..104c75478e 100644 --- a/internal/clustertrustbundle/client.go +++ b/internal/clustertrustbundle/client.go @@ -17,6 +17,7 @@ package clustertrustbundle import ( "context" + "errors" "fmt" "sync" "time" @@ -35,6 +36,9 @@ import ( "k8s.io/client-go/tools/cache" ) +// ErrNotServed reports that the cluster serves no ClusterTrustBundle API. +var ErrNotServed = errors.New("neither v1 nor v1beta1 ClusterTrustBundle is served") + // Discover prefers v1 and falls back only when discovery reports it absent. func Discover(d discovery.DiscoveryInterface) (schema.GroupVersion, error) { for _, gv := range []schema.GroupVersion{certsv1.SchemeGroupVersion, certsv1beta1.SchemeGroupVersion} { @@ -51,7 +55,7 @@ func Discover(d discovery.DiscoveryInterface) (schema.GroupVersion, error) { } } } - return schema.GroupVersion{}, fmt.Errorf("neither v1 nor v1beta1 ClusterTrustBundle is served") + return schema.GroupVersion{}, ErrNotServed } // Client uses one API version for live operations and its native informer cache. @@ -156,3 +160,10 @@ func (c *Client) Update(ctx context.Context, bundle *certsv1.ClusterTrustBundle, result, err := c.kc.CertificatesV1beta1().ClusterTrustBundles().Update(ctx, ToBeta(bundle), opts) return ToV1(result), err } + +func (c *Client) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + if c.v1 { + return c.kc.CertificatesV1().ClusterTrustBundles().Delete(ctx, name, opts) + } + return c.kc.CertificatesV1beta1().ClusterTrustBundles().Delete(ctx, name, opts) +} diff --git a/internal/clustertrustbundle/client_test.go b/internal/clustertrustbundle/client_test.go index 44807bf5ee..ab9864067c 100644 --- a/internal/clustertrustbundle/client_test.go +++ b/internal/clustertrustbundle/client_test.go @@ -16,6 +16,7 @@ package clustertrustbundle import ( "context" + "errors" "fmt" "reflect" "testing" @@ -69,6 +70,9 @@ func TestDiscovery(t *testing.T) { if tc.err != nil && calls != 1 { t.Fatal("fell back after discovery error") } + if notServed := errors.Is(err, ErrNotServed); notServed != (tc.err == nil) { + t.Fatalf("errors.Is(%v, ErrNotServed) = %t", err, notServed) + } return } if err != nil || gv.Version != tc.want { @@ -150,13 +154,7 @@ func TestClient(t *testing.T) { check(c.Update(ctx, want, metav1.UpdateOptions{FieldManager: "test"})) receive() check(c.GetCached(want.Name)) - // Delete through the selected native client to exercise watch deletion. - if version == "v1" { - err = kc.CertificatesV1().ClusterTrustBundles().Delete(ctx, want.Name, metav1.DeleteOptions{}) - } else { - err = kc.CertificatesV1beta1().ClusterTrustBundles().Delete(ctx, want.Name, metav1.DeleteOptions{}) - } - if err != nil { + if err := c.Delete(ctx, want.Name, metav1.DeleteOptions{}); err != nil { t.Fatal(err) } receive() From d259e5a64a7d9c2435e1dead802cf01a7b3de244 Mon Sep 17 00:00:00 2001 From: nybidari <59618317+nybidari@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:45:06 +0000 Subject: [PATCH 10/20] Update gVisor to 2026-10-10 nightly and use fs/ split checkpoint for durable dirs (#2455) Use gVisor's split filesystem checkpointing from the 2026-10-10 nightly release instead of host-side tar archiving for durable-dir volumes: - Mount durable-dir volumes as container-shared tmpfs inside the gVisor sandbox in ShapeGVisor. - On MEMORY checkpoints, pass -fs-checkpoint-paths to runsc checkpoint so durable-dir volumes are checkpointed into the fs/ subdirectory alongside the process and rootfs checkpoint. - On VOLUMES checkpoints, run runsc fscheckpoint with -image-path /fs and -path : for each durable-dir mount. - On RestoreWorkload, pass --fs-restore-image-path /fs to runsc create on the pause container when fs/fscheckpoint.pb is present, then either restore the full sandbox (MEMORY) or start fresh containers with only durable-dir volumes restored (VOLUMES). - Allow fs/ relative paths in snapshot file validation, local pause staging, and object store plugin transfers. Fixes # > It's a good idea to open an issue first for discussion. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- cmd/atelet/main.go | 16 +- cmd/atelet/main_test.go | 1 + cmd/ateom-gvisor/durable.go | 110 +++-------- cmd/ateom-gvisor/durable_test.go | 183 ++++++++++-------- cmd/ateom-gvisor/main.go | 80 +++----- cmd/ateom-gvisor/phaselog.go | 20 +- cmd/ateom-gvisor/phaselog_test.go | 34 ++-- cmd/ateom-gvisor/runsc.go | 100 +++------- cmd/ateom-gvisor/runsc_test.go | 23 ++- docs/api-guide.md | 12 +- internal/objectstoreplugin/node.go | 5 + internal/objectstoreplugin/plugin_test.go | 17 +- internal/ocispec/gvisor.go | 8 + internal/ocispec/parity_test.go | 11 +- internal/resources/snapshotfiles.go | 12 +- internal/resources/snapshotfiles_test.go | 4 +- .../ate-install/sandboxconfig-gvisor.yaml | 18 +- 17 files changed, 304 insertions(+), 350 deletions(-) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 9d05477df1..971feef65d 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -841,6 +841,9 @@ func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Che } recordSnapshotSize(ctx, fileName, allocatedBytes(info), req.GetActorTemplateAtespace(), req.GetActorTemplateName()) + if err := root.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return fmt.Errorf("while creating parent directory for %s: %w", dst, err) + } if err := root.Rename(src, dst); err != nil { return fmt.Errorf("failed to move %s to %s: %w", src, dst, err) } @@ -1336,10 +1339,16 @@ func (s *AteomHerder) Terminate(ctx context.Context, req *ateletpb.TerminateRequ } // checkLocalSnapshotFiles verifies each snapshot file exists in dir as a -// regular file. Lstat, so a symlink cannot point ateom outside the snapshot. +// regular file. Lstat via os.Root, so a symlink cannot point ateom outside the +// snapshot. func checkLocalSnapshotFiles(dir string, files []string) error { + root, err := os.OpenRoot(dir) + if err != nil { + return wrapFileSystemErr("while opening local checkpoint directory", err) + } + defer root.Close() for _, name := range files { - info, err := os.Lstat(filepath.Join(dir, name)) + info, err := root.Lstat(name) if err != nil { return wrapFileSystemErr("while checking local checkpoint file", err) } @@ -1381,6 +1390,9 @@ func (s *AteomHerder) copyLocalCheckpoint(ctx context.Context, actorDir, snapsho if !info.Mode().IsRegular() { return fmt.Errorf("%s is not a regular file", src) } + if err := root.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return fmt.Errorf("while creating parent directory for %s: %w", dst, err) + } // Link rather than copy. The local checkpoint lives under the same actor dir // as the restore staging area, so this stages the memory image in constant // time instead of re-writing its whole working set. Nothing rewrites the diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 757de4581f..1cbb89367a 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -349,6 +349,7 @@ func TestCheckpointSnapshotFiles(t *testing.T) { }{ {name: "required and present", files: []string{"checkpoint.img"}, required: true}, {name: "data subset", files: []string{"checkpoint.img", "data.tar"}, data: []string{"data.tar"}, required: true}, + {name: "fs subdir data subset", files: []string{"checkpoint.img", "fs/fscheckpoint.pb", "fs/multitar.img"}, data: []string{"fs/fscheckpoint.pb", "fs/multitar.img"}, required: true}, {name: "data not a snapshot file", files: []string{"checkpoint.img"}, data: []string{"data.tar"}, required: true, wantErr: true}, {name: "optional and empty", required: false}, {name: "required and empty", required: true, wantErr: true}, diff --git a/cmd/ateom-gvisor/durable.go b/cmd/ateom-gvisor/durable.go index f5fe1e0950..0058a416b0 100644 --- a/cmd/ateom-gvisor/durable.go +++ b/cmd/ateom-gvisor/durable.go @@ -17,104 +17,56 @@ package main import ( - "context" "errors" "fmt" "io/fs" "os" "path/filepath" - "strings" "github.com/agent-substrate/substrate/internal/proto/ateompb" - "github.com/agent-substrate/substrate/internal/tarutil" + "github.com/agent-substrate/substrate/internal/resources" ) -// durableTarFile is the snapshot file holding the tar of one durable-dir -// volume: the contents of under -// ActorDirs.durable_dir_volume_mounts_dir, plus a root entry for the volume -// directory's own metadata. The volume directory itself comes from atelet, -// never from the snapshot: runsc bind-mounts it by path, so a symlink planted -// there would expose whatever it points at to the sandbox. -func durableTarFile(volumeName string) (string, error) { - if volumeName == "" || volumeName == "." || strings.Contains(volumeName, "/") || !filepath.IsLocal(volumeName) { - return "", fmt.Errorf("invalid durable-dir volume name %q", volumeName) - } - return "durable-dir-" + volumeName + ".tar", nil -} +const fsCheckpointManifest = "fscheckpoint.pb" -// hasDurableVolumes reports whether any container mounts a durable-dir volume. -func hasDurableVolumes(containers []*ateompb.Container) bool { +// durableFSCheckpointPaths returns the runsc filesystem checkpoint target +// specs (:) for all durable-dir volume mounts +// across containers. Every mount is emitted even when a volume is shared across +// multiple mounts: runsc creates one shared tmpfs under whichever mount is +// set up first and deduplicates by vfs.Filesystem when saving. +func durableFSCheckpointPaths(containers []*ateompb.Container) []string { + var paths []string for _, c := range containers { - if len(c.GetDurableDirVolumeMounts()) > 0 { - return true + for _, m := range c.GetDurableDirVolumeMounts() { + paths = append(paths, fmt.Sprintf("%s:%s", c.GetName(), m.GetMountPath())) } } - return false + return paths } -// tarDurableVolumes archives each durable-dir volume under dir into the -// checkpoint directory, one tar per volume, and returns the file names. The -// caller must have paused the guest first. -// -// Sockets the workload left behind and gVisor internal files (.gvisor.*) are -// skipped rather than archived. -func tarDurableVolumes(ctx context.Context, dir, checkpointDir string, volumes []string) ([]string, error) { - skip := func(rel string) bool { - base := filepath.Base(rel) - return strings.HasPrefix(base, ".gvisor.") - } - var files []string - for _, vol := range volumes { - name, err := durableTarFile(vol) - if err != nil { - return nil, err - } - if err := tarutil.CreateFilteredWithRoot(ctx, filepath.Join(checkpointDir, name), filepath.Join(dir, vol), skip); err != nil { - return nil, fmt.Errorf("while archiving durable-dir volume %q: %w", vol, err) +// durableSnapshotFiles returns the subset of snapshotFiles that belong to the +// filesystem checkpoint in the fs/ subdirectory. +func durableSnapshotFiles(snapshotFiles []string) []string { + var out []string + for _, f := range snapshotFiles { + if filepath.Dir(f) == resources.FSCheckpointSubdir { + out = append(out, f) } - files = append(files, name) } - return files, nil + return out } -// untarDurableVolumes restores each durable-dir volume from the snapshot into -// its directory under dir, which atelet has already created, empty. A volume -// with no tar in the snapshot (added to the template since) stays empty. -func untarDurableVolumes(dir, snapshotDir string, volumes []string) error { - for _, vol := range volumes { - name, err := durableTarFile(vol) - if err != nil { - return err - } - tarPath := filepath.Join(snapshotDir, name) - if _, err := os.Stat(tarPath); errors.Is(err, fs.ErrNotExist) { - continue +// fsRestoreArgs returns `--fs-restore-image-path /fs` if a +// filesystem checkpoint manifest exists in /fs, or nil if no +// filesystem checkpoint is present in the snapshot. +func fsRestoreArgs(checkpointDir string) ([]string, error) { + fsDir := filepath.Join(checkpointDir, resources.FSCheckpointSubdir) + manifestPath := filepath.Join(fsDir, fsCheckpointManifest) + if _, err := os.Stat(manifestPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil } - volDir := filepath.Join(dir, vol) - if err := os.MkdirAll(volDir, 0o700); err != nil { - return fmt.Errorf("while creating durable-dir volume dir %q: %w", volDir, err) - } - if err := tarutil.Extract(tarPath, volDir); err != nil { - return fmt.Errorf("while restoring durable-dir volume %q: %w", vol, err) - } - removeGVisorFiles(volDir) - } - return nil -} - -// removeGVisorFiles deletes gVisor internal files (.gvisor.*) under dir, -// best-effort, through an os.Root so the restored tree's symlinks are never -// followed. -func removeGVisorFiles(dir string) { - root, err := os.OpenRoot(dir) - if err != nil { - return + return nil, fmt.Errorf("while checking filesystem checkpoint manifest %q: %w", manifestPath, err) } - defer root.Close() - _ = fs.WalkDir(root.FS(), ".", func(rel string, d fs.DirEntry, err error) error { - if err == nil && !d.IsDir() && strings.HasPrefix(d.Name(), ".gvisor.") { - _ = root.Remove(rel) - } - return nil - }) + return []string{"--fs-restore-image-path", fsDir}, nil } diff --git a/cmd/ateom-gvisor/durable_test.go b/cmd/ateom-gvisor/durable_test.go index cf0483542c..33e14c56b5 100644 --- a/cmd/ateom-gvisor/durable_test.go +++ b/cmd/ateom-gvisor/durable_test.go @@ -22,108 +22,129 @@ import ( "slices" "testing" - "github.com/agent-substrate/substrate/internal/tarutil" + "github.com/agent-substrate/substrate/internal/proto/ateompb" ) -// mkVolumeDirs creates the per-volume directories atelet prepares. -func mkVolumeDirs(t *testing.T, dir string, vols ...string) { - t.Helper() - for _, v := range vols { - if err := os.MkdirAll(filepath.Join(dir, v), 0o700); err != nil { - t.Fatal(err) - } +func TestDurableFSCheckpointPaths(t *testing.T) { + containers := []*ateompb.Container{ + { + Name: "app", + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/var/data"}, + {VolumeName: "cache", MountPath: "/var/cache"}, + }, + }, + { + Name: "sidecar", + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{ + {VolumeName: "logs", MountPath: "/var/log/app"}, + }, + }, } -} -func TestDurableVolumesRoundTrip(t *testing.T) { - src := t.TempDir() - mkVolumeDirs(t, src, "data", "cache") - for rel, content := range map[string]string{ - "data/notes.txt": "kept", - "data/.gvisor.filestat": "internal", - "cache/c.bin": "cached", - } { - if err := os.WriteFile(filepath.Join(src, rel), []byte(content), 0o644); err != nil { - t.Fatal(err) - } + got := durableFSCheckpointPaths(containers) + want := []string{ + "app:/var/data", + "app:/var/cache", + "sidecar:/var/log/app", } - // The volume root's own mode is the mount point's, so it must survive. - if err := os.Chmod(filepath.Join(src, "data"), 0o750); err != nil { - t.Fatal(err) - } - checkpointDir := t.TempDir() - files, err := tarDurableVolumes(t.Context(), src, checkpointDir, []string{"cache", "data"}) - if err != nil { - t.Fatalf("tarDurableVolumes: %v", err) - } - if want := []string{"durable-dir-cache.tar", "durable-dir-data.tar"}; !slices.Equal(files, want) { - t.Errorf("tarDurableVolumes files = %v, want %v", files, want) + if !slices.Equal(got, want) { + t.Errorf("durableFSCheckpointPaths() = %v, want %v", got, want) } +} - dst := t.TempDir() - mkVolumeDirs(t, dst, "data", "cache") - if err := untarDurableVolumes(dst, checkpointDir, []string{"cache", "data"}); err != nil { - t.Fatalf("untarDurableVolumes: %v", err) - } - for rel, want := range map[string]string{"data/notes.txt": "kept", "cache/c.bin": "cached"} { - if got, err := os.ReadFile(filepath.Join(dst, rel)); err != nil || string(got) != want { - t.Errorf("restored %q = %q, %v; want %q", rel, got, err, want) - } +func TestDurableSnapshotFiles(t *testing.T) { + snapshotFiles := []string{ + "checkpoint.img", + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + "pages.img", + "pages_meta.img", } - if _, err := os.Lstat(filepath.Join(dst, "data/.gvisor.filestat")); !os.IsNotExist(err) { - t.Errorf(".gvisor.filestat: Lstat = %v; want it skipped", err) + + got := durableSnapshotFiles(snapshotFiles) + want := []string{ + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", } - if fi, err := os.Stat(filepath.Join(dst, "data")); err != nil || fi.Mode().Perm() != 0o750 { - t.Errorf("restored volume dir: Stat = %v, %v; want mode 0750", fi, err) + if !slices.Equal(got, want) { + t.Errorf("durableSnapshotFiles() = %v, want %v", got, want) } } -// runsc bind-mounts / by path, so the snapshot must not be able -// to replace that directory with a symlink out of the actor's tree. -func TestUntarDurableVolumesCannotPlantVolumeDir(t *testing.T) { - victim := t.TempDir() - planted := t.TempDir() - if err := os.Symlink(victim, filepath.Join(planted, "data")); err != nil { +func TestListSnapshotFilesWithFSSubdir(t *testing.T) { + checkpointDir := t.TempDir() + fsDir := filepath.Join(checkpointDir, "fs") + if err := os.MkdirAll(fsDir, 0o700); err != nil { t.Fatal(err) } - snapshotDir := t.TempDir() - // An entry named after the volume, as the old single-tar layout had. - if err := tarutil.Create(t.Context(), filepath.Join(snapshotDir, "durable-dir-data.tar"), planted); err != nil { - t.Fatal(err) + for _, rel := range []string{ + "checkpoint.img", + "pages.img", + "pages_meta.img", + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + } { + if err := os.WriteFile(filepath.Join(checkpointDir, rel), []byte("data"), 0o600); err != nil { + t.Fatal(err) + } } - dst := t.TempDir() - mkVolumeDirs(t, dst, "data") - if err := untarDurableVolumes(dst, snapshotDir, []string{"data"}); err != nil { - t.Fatalf("untarDurableVolumes: %v", err) + got, err := listSnapshotFiles(checkpointDir) + if err != nil { + t.Fatalf("listSnapshotFiles() = %v", err) } - if fi, err := os.Lstat(filepath.Join(dst, "data")); err != nil || !fi.IsDir() { - t.Errorf("data: Lstat = %v, %v; want a real directory", fi, err) + want := []string{ + "checkpoint.img", + "fs/fscheckpoint.pb", + "fs/multitar.img", + "fs/pages.img", + "fs/pages_meta.img", + "pages.img", + "pages_meta.img", } - entries, err := os.ReadDir(victim) - if err != nil || len(entries) != 0 { - t.Errorf("victim: ReadDir = %v, %v; want it untouched", entries, err) + if !slices.Equal(got, want) { + t.Errorf("listSnapshotFiles() = %v, want %v", got, want) } } -// A volume added to the template after the snapshot has no tar: it restores -// empty rather than failing. -func TestUntarDurableVolumesMissingTarIsEmpty(t *testing.T) { - dst := t.TempDir() - mkVolumeDirs(t, dst, "new") - if err := untarDurableVolumes(dst, t.TempDir(), []string{"new"}); err != nil { - t.Fatalf("untarDurableVolumes: %v", err) - } - entries, err := os.ReadDir(filepath.Join(dst, "new")) - if err != nil || len(entries) != 0 { - t.Errorf("new: ReadDir = %v, %v; want an empty directory", entries, err) - } -} +func TestFSRestoreArgs(t *testing.T) { + t.Run("no fs checkpoint present", func(t *testing.T) { + checkpointDir := t.TempDir() + got, err := fsRestoreArgs(checkpointDir) + if err != nil { + t.Fatalf("fsRestoreArgs() = %v", err) + } + if len(got) != 0 { + t.Errorf("fsRestoreArgs() = %v, want empty", got) + } + }) -func TestDurableTarFileRejectsBadNames(t *testing.T) { - for _, v := range []string{"", ".", "..", "a/b", "/abs", "../x"} { - if _, err := durableTarFile(v); err == nil { - t.Errorf("durableTarFile(%q) = nil error, want one", v) + t.Run("fs checkpoint present in fs subdir", func(t *testing.T) { + checkpointDir := t.TempDir() + fsDir := filepath.Join(checkpointDir, "fs") + if err := os.MkdirAll(fsDir, 0o700); err != nil { + t.Fatal(err) } - } + if err := os.WriteFile(filepath.Join(fsDir, "fscheckpoint.pb"), []byte("manifest"), 0o600); err != nil { + t.Fatal(err) + } + + got, err := fsRestoreArgs(checkpointDir) + if err != nil { + t.Fatalf("fsRestoreArgs() = %v", err) + } + want := []string{ + "--fs-restore-image-path", fsDir, + } + if !slices.Equal(got, want) { + t.Errorf("fsRestoreArgs() = %v, want %v", got, want) + } + }) } diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 531e65ed47..c7215fbeb3 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -24,6 +24,7 @@ import ( "net" "os" "os/signal" + "path/filepath" "slices" "sort" "sync" @@ -91,9 +92,6 @@ const minUsageSampleInterval = 50 * time.Second // kubelet SIGKILL of ateom itself. const workloadGracePeriod = 30 * time.Minute -// resumeTimeout is the conservative ceiling for unpausing a paused sandbox. -const resumeTimeout = 30 * time.Second - func main() { pflag.Parse() if *showVersion { @@ -755,52 +753,26 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, fmt.Errorf("while creating checkpoint directory: %w", err) } - // durableFiles are the durable-dir tars written below: the VOLUMES subset. - var durableFiles []string - // Always take durable-dir snapshot if at least one container has a durable-dir volume mount. - // TODO(dberkov): this is a temporary workaround until gVisor supports taking durable-dir snapshots in a single request with the process snapshot. + containers := req.GetSpec().GetContainers() + durablePaths := durableFSCheckpointPaths(containers) switch fidelity { case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: - if !hasDurableVolumes(req.GetSpec().GetContainers()) { + if len(durablePaths) == 0 { return nil, fmt.Errorf("no durable-dir volumes found for VOLUMES snapshot") } - err := rcmd.cmdPause(ctx, ocispec.PauseContainer) - timing.pause = lap(&tLast) - if err != nil { - return nil, fmt.Errorf("while pausing pause container: %w", err) - } - var tarErr error - durableFiles, tarErr = tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath, durableVolumeNames(req.GetSpec())) + err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) timing.durableDir = lap(&tLast) - // Undoing our own pause must not depend on the caller's context: - // tarutil does not check ctx, so a deadline expiring mid-tar would - // fail the resume instantly and leave the sandbox paused forever. - resumeCtx, cancelResume := context.WithTimeout(context.WithoutCancel(ctx), resumeTimeout) - defer cancelResume() - err = rcmd.cmdResume(resumeCtx, ocispec.PauseContainer) - timing.resume = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while resuming pause container: %w", err) - } - if tarErr != nil { - return nil, fmt.Errorf("while archiving durable-dir volumes: %w", tarErr) + return nil, fmt.Errorf("while checkpointing durable-dir volumes: %w", err) } case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Checkpoint pause container (root of the sandbox) - // TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling. - err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath) + // Checkpoint pause container (root of the sandbox), splitting durable-dir + // volumes into /fs when present. + err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) timing.checkpoint = lap(&tLast) if err != nil { return nil, fmt.Errorf("while checkpointing pause: %w", err) } - if hasDurableVolumes(req.GetSpec().GetContainers()) { - var err error - durableFiles, err = tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath, durableVolumeNames(req.GetSpec())) - timing.durableDir = lap(&tLast) - if err != nil { - return nil, fmt.Errorf("while archiving durable-dir volumes: %w", err) - } - } default: return nil, fmt.Errorf("unsupported snapshot fidelity: %v", fidelity) } @@ -818,11 +790,13 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec s.recordFinalIfEnded(ctx, hosted) // Report exactly the files runsc wrote so atelet ships precisely this set - // (checkpoint.img plus any pages images), rather than a hardcoded list. + // (checkpoint.img plus any pages images and fs/ durable-dir files), rather + // than a hardcoded list. snapshotFiles, err := listSnapshotFiles(checkpointPath) if err != nil { return nil, fmt.Errorf("while listing checkpoint files: %w", err) } + durableFiles := durableSnapshotFiles(snapshotFiles) s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointed", attribution) @@ -830,7 +804,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec } // listSnapshotFiles returns the (relative) names of regular files directly under -// dir, which atelet ships to object storage as the snapshot. +// dir and under dir/fs, which atelet ships to object storage as the snapshot. func listSnapshotFiles(dir string) ([]string, error) { entries, err := os.ReadDir(dir) if err != nil { @@ -842,6 +816,15 @@ func listSnapshotFiles(dir string) ([]string, error) { files = append(files, e.Name()) } } + fsEntries, err := os.ReadDir(filepath.Join(dir, resources.FSCheckpointSubdir)) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + for _, e := range fsEntries { + if e.Type().IsRegular() { + files = append(files, filepath.Join(resources.FSCheckpointSubdir, e.Name())) + } + } sort.Strings(files) return files, nil } @@ -999,14 +982,11 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } }() checkpointDir := req.GetActorDirs().GetRestoreDir() - - if hasDurableVolumes(containers) { - err := untarDurableVolumes(req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec())) - timing.durableDir = lap(&tLast) - if err != nil { - return nil, fmt.Errorf("while restoring durable-dir volumes: %w", err) - } + restoreArgs, err := fsRestoreArgs(checkpointDir) + if err != nil { + return nil, err } + // Compose the pause rootfs before create (see RunWorkload). runsc restore // only needs the rootfs to hold the correct content; whether it came from // an untar or an overlay of cached layers is transparent to it. @@ -1018,9 +998,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore switch fidelity { case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES: - // Create and start pause container (cold boot with durable-dir volumes restored) + // Create and start pause container (cold boot with durable-dir volumes restored from fs/) containersToDelete = append(containersToDelete, ocispec.PauseContainer) - err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) + err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, restoreArgs) timing.pauseCreate = lap(&tLast) if err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) @@ -1031,9 +1011,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, fmt.Errorf("while starting pause container: %w", err) } case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Create and restore pause container + // Create and restore pause container (restores both process/rootfs and fs/ durable-dir volumes) containersToDelete = append(containersToDelete, ocispec.PauseContainer) - err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil) + err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, restoreArgs) timing.pauseCreate = lap(&tLast) if err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) diff --git a/cmd/ateom-gvisor/phaselog.go b/cmd/ateom-gvisor/phaselog.go index 1509545447..6c73cca5c0 100644 --- a/cmd/ateom-gvisor/phaselog.go +++ b/cmd/ateom-gvisor/phaselog.go @@ -36,7 +36,6 @@ const ( phasePrep = "prep" phaseEgressPrepare = "egress_prepare" phaseNetSetup = "net_setup" - phaseDurableDir = "durable_dir" phasePauseRootfs = "pause_rootfs" phasePauseCreate = "pause_create" phasePauseRestore = "pause_restore" @@ -46,9 +45,8 @@ const ( phaseWakeupProbe = "wakeup_probe" phaseActivate = "activate" - phasePause = "pause" phaseCheckpoint = "checkpoint" - phaseResume = "resume" + phaseDurableDir = "durable_dir" phaseTeardown = "teardown" phaseTotal = ateattr.SnapshotPhaseTotal @@ -62,10 +60,10 @@ const containerCountKey = "ate.actor.container.count" // left at zero never ran. Under a Data scope pauseRestore and appRestore time // the cold start that stands in for the restore. type restoreTiming struct { - prep, egressPrepare, netSetup, durableDir time.Duration - pauseRootfs, pauseCreate, pauseRestore time.Duration - appRootfs, appCreate, appRestore time.Duration - wakeupProbe, activate, total time.Duration + prep, egressPrepare, netSetup time.Duration + pauseRootfs, pauseCreate, pauseRestore time.Duration + appRootfs, appCreate, appRestore time.Duration + wakeupProbe, activate, total time.Duration } func (t restoreTiming) phases() []ateomphaselog.Phase { @@ -73,7 +71,6 @@ func (t restoreTiming) phases() []ateomphaselog.Phase { {Name: phasePrep, D: t.prep}, {Name: phaseEgressPrepare, D: t.egressPrepare}, {Name: phaseNetSetup, D: t.netSetup}, - {Name: phaseDurableDir, D: t.durableDir}, {Name: phasePauseRootfs, D: t.pauseRootfs}, {Name: phasePauseCreate, D: t.pauseCreate}, {Name: phasePauseRestore, D: t.pauseRestore}, @@ -87,19 +84,16 @@ func (t restoreTiming) phases() []ateomphaselog.Phase { } // checkpointTiming is the elapsed time of each CheckpointWorkload phase. A -// Full scope runs checkpoint; a Data scope runs pause and resume around the -// durable_dir capture instead. +// Full scope runs checkpoint; a Data scope runs durable_dir capture instead. type checkpointTiming struct { - prep, pause, checkpoint, durableDir, resume, teardown, total time.Duration + prep, checkpoint, durableDir, teardown, total time.Duration } func (t checkpointTiming) phases() []ateomphaselog.Phase { return []ateomphaselog.Phase{ {Name: phasePrep, D: t.prep}, - {Name: phasePause, D: t.pause}, {Name: phaseCheckpoint, D: t.checkpoint}, {Name: phaseDurableDir, D: t.durableDir}, - {Name: phaseResume, D: t.resume}, {Name: phaseTeardown, D: t.teardown}, {Name: phaseTotal, D: t.total}, } diff --git a/cmd/ateom-gvisor/phaselog_test.go b/cmd/ateom-gvisor/phaselog_test.go index a90f81cdd3..e8de62c51f 100644 --- a/cmd/ateom-gvisor/phaselog_test.go +++ b/cmd/ateom-gvisor/phaselog_test.go @@ -43,8 +43,7 @@ func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any { } // TestRestoreTimingPhases pins the keys a Full restore record carries, in -// the layout benchmarking tooling joins with atelet's record on, and that a -// phase which never ran stays off it. +// the layout benchmarking tooling joins with atelet's record on. func TestRestoreTimingPhases(t *testing.T) { t.Parallel() @@ -52,17 +51,15 @@ func TestRestoreTimingPhases(t *testing.T) { prep: 10 * time.Millisecond, egressPrepare: 120 * time.Millisecond, netSetup: 30 * time.Millisecond, - // No durable volumes: the phase never ran. - durableDir: 0, - pauseRootfs: 5 * time.Millisecond, - pauseCreate: 200 * time.Millisecond, - pauseRestore: 900 * time.Millisecond, - appRootfs: 15 * time.Millisecond, - appCreate: 50 * time.Millisecond, - appRestore: 400 * time.Millisecond, - wakeupProbe: 250 * time.Millisecond, - activate: 2 * time.Millisecond, - total: 2 * time.Second, + pauseRootfs: 5 * time.Millisecond, + pauseCreate: 200 * time.Millisecond, + pauseRestore: 900 * time.Millisecond, + appRootfs: 15 * time.Millisecond, + appCreate: 50 * time.Millisecond, + appRestore: 400 * time.Millisecond, + wakeupProbe: 250 * time.Millisecond, + activate: 2 * time.Millisecond, + total: 2 * time.Second, } attrs := ateomphaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateomphaselog.RestoreDurationKey, nil, timing.phases()) @@ -99,9 +96,6 @@ func TestRestoreTimingPhases(t *testing.T) { t.Errorf("%s = %v, want %v", k, got, want) } } - if v, ok := rec["ateom.actor.restore.duration.durable_dir"]; ok { - t.Errorf("durable_dir present with %v, want absent", v) - } if rec["ate.actor.uid"] != "uid-abc" { t.Errorf("ate.actor.uid = %v, want uid-abc", rec["ate.actor.uid"]) } @@ -110,16 +104,14 @@ func TestRestoreTimingPhases(t *testing.T) { } } -// TestCheckpointTimingPhases: a Data checkpoint pauses and resumes around the -// durable capture and never runs the runsc checkpoint, and the record says so. +// TestCheckpointTimingPhases: a Data checkpoint runs the durable capture and +// never runs the full runsc checkpoint, and the record says so. func TestCheckpointTimingPhases(t *testing.T) { t.Parallel() timing := checkpointTiming{ prep: 20 * time.Millisecond, - pause: 3 * time.Millisecond, durableDir: 300 * time.Millisecond, - resume: 4 * time.Millisecond, teardown: 150 * time.Millisecond, total: 500 * time.Millisecond, } @@ -128,9 +120,7 @@ func TestCheckpointTimingPhases(t *testing.T) { for k, want := range map[string]float64{ "ateom.actor.checkpoint.duration.prep": 0.02, - "ateom.actor.checkpoint.duration.pause": 0.003, "ateom.actor.checkpoint.duration.durable_dir": 0.3, - "ateom.actor.checkpoint.duration.resume": 0.004, "ateom.actor.checkpoint.duration.teardown": 0.15, "ateom.actor.checkpoint.duration.total": 0.5, } { diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 1745603333..8caf5157f5 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -25,12 +25,14 @@ import ( "log/slog" "os" "os/exec" + "path/filepath" "slices" "strings" "syscall" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/sizing" ) @@ -83,6 +85,7 @@ func (r *runsc) writeSpec(containerName string) error { spec := ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: container}) ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, + ActorDirs: r.actorDirs, ContainerName: containerName, DurableVolumes: r.durableVolumes, Size: r.size, @@ -163,12 +166,10 @@ func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName strin return nil } -func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath string) error { - slog.InfoContext(ctx, "About to run runsc checkpoint", slog.String("container", containerName)) - - cmd := exec.CommandContext( - ctx, - r.path, +// checkpointArgs builds the argv for `runsc checkpoint `. Factored +// out so the argument construction can be unit-tested without executing runsc. +func (r *runsc) checkpointArgs(containerName, checkpointPath string, fsCheckpointPaths []string) []string { + args := []string{ "-log-format", "json", "--alsologtostderr", // "-debug", @@ -179,8 +180,18 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath "-root", runscStateDir(r.actorDirs), "checkpoint", "-image-path", checkpointPath, - containerName, // Name of the container - ) + } + if len(fsCheckpointPaths) > 0 { + args = append(args, "-fs-checkpoint-paths", strings.Join(fsCheckpointPaths, ",")) + } + args = append(args, containerName) + return args +} + +func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath string, fsCheckpointPaths []string) error { + slog.InfoContext(ctx, "About to run runsc checkpoint", slog.String("container", containerName)) + + cmd := exec.CommandContext(ctx, r.path, r.checkpointArgs(containerName, checkpointPath, fsCheckpointPaths)...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := reaper.RunCommand(cmd) @@ -190,10 +201,10 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath return nil } -//nolint:unused -func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPath string, durableDirMounts []string) error { - slog.InfoContext(ctx, "About to run runsc fscheckpoint", slog.String("container", containerName)) - +// fsCheckpointArgs builds the argv for `runsc fscheckpoint `. +// Factored out so the argument construction can be unit-tested without +// executing runsc. +func (r *runsc) fsCheckpointArgs(containerName, checkpointPath string, fsCheckpointPaths []string) []string { args := []string{ "-log-format", "json", "--alsologtostderr", @@ -204,20 +215,21 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa // "-strace", "-root", runscStateDir(r.actorDirs), "fscheckpoint", - "-image-path", checkpointPath, + "-image-path", filepath.Join(checkpointPath, resources.FSCheckpointSubdir), } - for _, ddv := range durableDirMounts { + for _, ddv := range fsCheckpointPaths { args = append(args, "-path", ddv) } // name of the container must be the last parameter. args = append(args, containerName) + return args +} - cmd := exec.CommandContext( - ctx, - r.path, - args..., - ) +func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPath string, fsCheckpointPaths []string) error { + slog.InfoContext(ctx, "About to run runsc fscheckpoint", slog.String("container", containerName)) + + cmd := exec.CommandContext(ctx, r.path, r.fsCheckpointArgs(containerName, checkpointPath, fsCheckpointPaths)...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := reaper.RunCommand(cmd) @@ -227,56 +239,6 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa return nil } -// pauseArgs builds the argv for `runsc pause `. Factored out so the -// argument construction can be unit-tested without executing runsc. -func (r *runsc) pauseArgs(containerName string) []string { - return []string{ - "-log-format", "json", - "--alsologtostderr", - "-root", runscStateDir(r.actorDirs), - "pause", - containerName, - } -} - -// cmdPause pauses all processes in the container (or sandbox, if pause). -func (r *runsc) cmdPause(ctx context.Context, containerName string) error { - slog.InfoContext(ctx, "About to run runsc pause", slog.String("container", containerName)) - - cmd := exec.CommandContext(ctx, r.path, r.pauseArgs(containerName)...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := reaper.RunCommand(cmd); err != nil { - return fmt.Errorf("while running `runsc pause`: %w", err) - } - return nil -} - -// resumeArgs builds the argv for `runsc resume `. Factored out so the -// argument construction can be unit-tested without executing runsc. -func (r *runsc) resumeArgs(containerName string) []string { - return []string{ - "-log-format", "json", - "--alsologtostderr", - "-root", runscStateDir(r.actorDirs), - "resume", - containerName, - } -} - -// cmdResume unpauses a paused container (or sandbox, if pause). -func (r *runsc) cmdResume(ctx context.Context, containerName string) error { - slog.InfoContext(ctx, "About to run runsc resume", slog.String("container", containerName)) - - cmd := exec.CommandContext(ctx, r.path, r.resumeArgs(containerName)...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := reaper.RunCommand(cmd); err != nil { - return fmt.Errorf("while running `runsc resume`: %w", err) - } - return nil -} - // restoreArgs builds the argv for `runsc restore `. Factored out so // the argument construction can be unit-tested without executing runsc. func (r *runsc) restoreArgs(containerName, checkpointPath string) []string { diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index f9a8bcc709..1588366cfa 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -126,45 +126,52 @@ func TestWaitArgs(t *testing.T) { } } -func TestPauseArgs(t *testing.T) { +func TestCheckpointArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", actorUID: "test-actor-123", actorDirs: testActorDirs, } + checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.pauseArgs(ocispec.PauseContainer) + got := r.checkpointArgs(ocispec.PauseContainer, checkpointDir, []string{"app:/var/data", "app:/var/cache"}) want := []string{ "-log-format", "json", "--alsologtostderr", "-root", "/node/actors/test-actor-123/runsc-state", - "pause", + "checkpoint", + "-image-path", checkpointDir, + "-fs-checkpoint-paths", "app:/var/data,app:/var/cache", ocispec.PauseContainer, } if !reflect.DeepEqual(got, want) { - t.Errorf("pauseArgs() = %v, want %v", got, want) + t.Errorf("checkpointArgs() = %v, want %v", got, want) } } -func TestResumeArgs(t *testing.T) { +func TestFsCheckpointArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", actorUID: "test-actor-123", actorDirs: testActorDirs, } + checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.resumeArgs(ocispec.PauseContainer) + got := r.fsCheckpointArgs(ocispec.PauseContainer, checkpointDir, []string{"app:/var/data", "app:/var/cache"}) want := []string{ "-log-format", "json", "--alsologtostderr", "-root", "/node/actors/test-actor-123/runsc-state", - "resume", + "fscheckpoint", + "-image-path", filepath.Join(checkpointDir, "fs"), + "-path", "app:/var/data", + "-path", "app:/var/cache", ocispec.PauseContainer, } if !reflect.DeepEqual(got, want) { - t.Errorf("resumeArgs() = %v, want %v", got, want) + t.Errorf("fsCheckpointArgs() = %v, want %v", got, want) } } diff --git a/docs/api-guide.md b/docs/api-guide.md index c28a038cd5..754ace24da 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -492,20 +492,20 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v2 + defaultVersion: v3 versions: - - name: v2 + - name: v3 state: Enabled pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20260907.0/x86_64/gvisor.tar.zstd" - sha256: "e32ed48a2ddc7c0ef5e922ff5d52e33b6fccc75b61fd4c40a0bd9bdfe90032e6" + url: "gs://gvisor/releases/nightly/2026-10-10/x86_64/gvisor.tar.zstd" + sha256: "4ae7fab988c3647d2a835d6c3c70c6a99021c562db97827592562b6c51c3e88c" arm64: gvisor: - url: "gs://gvisor/releases/release/20260907.0/aarch64/gvisor.tar.zstd" - sha256: "f3ba93a1a83dd4b6a322237030cdf81a278d9ef4e01ae6218ba69fbc479fdfb2" + url: "gs://gvisor/releases/nightly/2026-10-10/aarch64/gvisor.tar.zstd" + sha256: "aaf414f06eea7960238e43165c691f0a36cefc0593fb141e011e96211c7b6f57" ``` ### Micro-VM SandboxConfig diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go index 6c7f8fa35d..bef2923ca7 100644 --- a/internal/objectstoreplugin/node.go +++ b/internal/objectstoreplugin/node.go @@ -83,6 +83,11 @@ func (p *NodePlugin) FetchSnapshot(ctx context.Context, req *objectstorev1.Fetch if err != nil { return fmt.Errorf("while addressing %s in GCS: %w", fileName, err) } + if dir := filepath.Dir(fileName); dir != "." { + if err := root.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("while creating %s in restore directory: %w", dir, err) + } + } local, err := root.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return fmt.Errorf("while opening %s in restore directory: %w", fileName, err) diff --git a/internal/objectstoreplugin/plugin_test.go b/internal/objectstoreplugin/plugin_test.go index 88eb1389d3..d9b0b12121 100644 --- a/internal/objectstoreplugin/plugin_test.go +++ b/internal/objectstoreplugin/plugin_test.go @@ -164,10 +164,14 @@ func TestUploadFetchRoundTrip(t *testing.T) { t.Fatal(err) } } + if err := os.Mkdir(filepath.Join(src, "fs"), 0o755); err != nil { + t.Fatal(err) + } files := map[string][]byte{ - "memory.img": bytes.Repeat([]byte("abcdefgh"), 1<<16), - "state.bin": []byte("vm state"), - manifestFile: []byte(`{"snapshotFiles":["memory.img","state.bin"]}`), + "memory.img": bytes.Repeat([]byte("abcdefgh"), 1<<16), + "state.bin": []byte("vm state"), + "fs/fscheckpoint.pb": []byte("fs manifest"), + manifestFile: []byte(`{"snapshotFiles":["memory.img","state.bin","fs/fscheckpoint.pb"]}`), } for name, content := range files { if err := os.WriteFile(filepath.Join(src, name), content, 0o644); err != nil { @@ -176,7 +180,7 @@ func TestUploadFetchRoundTrip(t *testing.T) { } if _, err := client.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ - SnapshotUri: testURI, LocalPath: src, Files: []string{"memory.img", "state.bin"}, + SnapshotUri: testURI, LocalPath: src, Files: []string{"memory.img", "state.bin", "fs/fscheckpoint.pb"}, }); err != nil { t.Fatalf("UploadSnapshot(data) = %v", err) } @@ -188,6 +192,7 @@ func TestUploadFetchRoundTrip(t *testing.T) { // The stored layout is the one atelet has always written. want := []string{ + testBucket + "/" + testPrefix + "/fs/fscheckpoint.pb.zstd", testBucket + "/" + testPrefix + "/manifest.json", testBucket + "/" + testPrefix + "/memory.img.zstd", testBucket + "/" + testPrefix + "/state.bin.zstd", @@ -195,12 +200,12 @@ func TestUploadFetchRoundTrip(t *testing.T) { if got := backend.keys(); !equal(got, want) { t.Fatalf("stored objects = %v, want %v", got, want) } - if got := backend.m[want[0]]; !bytes.Equal(got, files[manifestFile]) { + if got := backend.m[want[1]]; !bytes.Equal(got, files[manifestFile]) { t.Errorf("stored manifest = %q, want it uncompressed", got) } if _, err := client.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ - SnapshotUri: testURI, WritePath: dst, Files: []string{manifestFile, "memory.img", "state.bin"}, + SnapshotUri: testURI, WritePath: dst, Files: []string{manifestFile, "memory.img", "state.bin", "fs/fscheckpoint.pb"}, }); err != nil { t.Fatalf("FetchSnapshot = %v", err) } diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index ae47f5ad2a..f73de3b182 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -15,8 +15,10 @@ package ocispec import ( + "path/filepath" "slices" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/sizing" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -32,6 +34,7 @@ const resolvConf = "/etc/resolv.conf" // GVisorOptions describes the gVisor-specific context of one actor container. type GVisorOptions struct { ActorUID string + ActorDirs *ateompb.ActorDirs ContainerName string // DurableVolumes are declared on the sandbox (pause) spec only. DurableVolumes []string @@ -56,6 +59,11 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { spec.Annotations["io.kubernetes.cri.container-name"] = o.ContainerName if o.ContainerName == PauseContainer { spec.Annotations["io.kubernetes.cri.container-type"] = "sandbox" + for _, v := range o.DurableVolumes { + spec.Annotations["dev.gvisor.spec.mount."+v+".type"] = "tmpfs" + spec.Annotations["dev.gvisor.spec.mount."+v+".share"] = "container" + spec.Annotations["dev.gvisor.spec.mount."+v+".source"] = filepath.Join(o.ActorDirs.GetDurableDirVolumeMountsDir(), v) + } } else { spec.Annotations["io.kubernetes.cri.container-type"] = "container" spec.Annotations["io.kubernetes.cri.sandbox-id"] = PauseContainer diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 468c511721..293c26a35b 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -153,7 +153,7 @@ func TestShapeGVisor_ResolvConfPrecedesVolumes(t *testing.T) { // Shaping a spec twice does not accumulate mounts. func TestShapeGVisor_Idempotent(t *testing.T) { spec := Build(parityOptions) - o := GVisorOptions{ActorUID: testActorUID, ContainerName: PauseContainer, DurableVolumes: []string{"data"}, Size: paritySize} + o := GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: PauseContainer, DurableVolumes: []string{"data"}, Size: paritySize} ShapeGVisor(spec, o) first := len(spec.Mounts) ShapeGVisor(spec, o) @@ -163,6 +163,15 @@ func TestShapeGVisor_Idempotent(t *testing.T) { if got := spec.Annotations["io.kubernetes.cri.container-type"]; got != "sandbox" { t.Errorf("pause container-type = %q, want sandbox", got) } + if got := spec.Annotations["dev.gvisor.spec.mount.data.type"]; got != "tmpfs" { + t.Errorf("durable-dir mount type annotation = %q, want tmpfs", got) + } + if got := spec.Annotations["dev.gvisor.spec.mount.data.share"]; got != "container" { + t.Errorf("durable-dir mount share annotation = %q, want container", got) + } + if got, want := spec.Annotations["dev.gvisor.spec.mount.data.source"], "/node/actors/a/durable-dir/data"; got != want { + t.Errorf("durable-dir mount source annotation = %q, want %q", got, want) + } } // gVisor sizes the container's cgroup leaf from the actor-level limits: one diff --git a/internal/resources/snapshotfiles.go b/internal/resources/snapshotfiles.go index 4487344999..fcacb2cdda 100644 --- a/internal/resources/snapshotfiles.go +++ b/internal/resources/snapshotfiles.go @@ -19,14 +19,20 @@ import ( "path/filepath" ) +// FSCheckpointSubdir is the subdirectory under the checkpoint directory where +// filesystem checkpoint files are stored. +const FSCheckpointSubdir = "fs" + // ValidateSnapshotFileNames requires each name to be a distinct plain file name -// in the checkpoint directory. Actual file access must still use os.Root so -// symlinks cannot escape that directory. +// in the checkpoint directory or its "fs" subdirectory. Actual file access must +// still use os.Root so symlinks cannot escape that directory. func ValidateSnapshotFileNames(files []string) error { seen := make(map[string]bool, len(files)) for i, name := range files { + dir := filepath.Dir(name) + base := filepath.Base(name) switch { - case name != filepath.Base(name) || !filepath.IsLocal(name) || name == ".": + case !filepath.IsLocal(name) || name != filepath.Clean(name) || base == "." || (dir != "." && dir != FSCheckpointSubdir): return fmt.Errorf("snapshotFiles[%d] %q is not a file name in the checkpoint directory", i, name) case seen[name]: return fmt.Errorf("snapshotFiles[%d] %q is duplicated", i, name) diff --git a/internal/resources/snapshotfiles_test.go b/internal/resources/snapshotfiles_test.go index cba3bb3d5c..e5667f6429 100644 --- a/internal/resources/snapshotfiles_test.go +++ b/internal/resources/snapshotfiles_test.go @@ -24,11 +24,13 @@ func TestValidateSnapshotFileNames(t *testing.T) { }{ {name: "empty list", files: nil}, {name: "plain names", files: []string{"manifest.json", "memory.img", "durable-dir.tar"}}, + {name: "fs subdir names", files: []string{"checkpoint.img", "fs/fscheckpoint.pb", "fs/multitar.img", "fs/pages.img", "fs/pages_meta.img"}}, {name: "empty name", files: []string{""}, wantErr: true}, {name: "dot", files: []string{"."}, wantErr: true}, {name: "dot dot", files: []string{".."}, wantErr: true}, {name: "absolute", files: []string{"/etc/passwd"}, wantErr: true}, - {name: "nested", files: []string{"a/b"}, wantErr: true}, + {name: "nested arbitrary subdir", files: []string{"a/b"}, wantErr: true}, + {name: "deeply nested under fs", files: []string{"fs/a/b"}, wantErr: true}, {name: "escaping", files: []string{"../x"}, wantErr: true}, {name: "duplicate", files: []string{"a", "b", "a"}, wantErr: true}, } { diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 96164f8c54..800bfc608c 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -27,12 +27,12 @@ metadata: sandboxconfig.ate.dev/is-class-default: "true" spec: sandboxClass: gvisor - defaultVersion: v2 + defaultVersion: v3 versions: - # gVisor release 20260907.0, the first tagged release with commit - # 80bb741691be, which closes a systrap race that killed sandboxes under CPU - # contention. - - name: v2 + # gVisor nightly 2026-10-10, for the split filesystem checkpoint + # (runsc fscheckpoint, -fs-checkpoint-paths and -fs-restore-image-path) that + # ateom-gvisor uses to save and restore durable-dir volumes. + - name: v3 state: Enabled # The root sandbox container's image. On GCP, prefer the in-project mirror # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. @@ -40,9 +40,9 @@ spec: assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20260907.0/x86_64/gvisor.tar.zstd" - sha256: "e32ed48a2ddc7c0ef5e922ff5d52e33b6fccc75b61fd4c40a0bd9bdfe90032e6" + url: "gs://gvisor/releases/nightly/2026-10-10/x86_64/gvisor.tar.zstd" + sha256: "4ae7fab988c3647d2a835d6c3c70c6a99021c562db97827592562b6c51c3e88c" arm64: gvisor: - url: "gs://gvisor/releases/release/20260907.0/aarch64/gvisor.tar.zstd" - sha256: "f3ba93a1a83dd4b6a322237030cdf81a278d9ef4e01ae6218ba69fbc479fdfb2" + url: "gs://gvisor/releases/nightly/2026-10-10/aarch64/gvisor.tar.zstd" + sha256: "aaf414f06eea7960238e43165c691f0a36cefc0593fb141e011e96211c7b6f57" From 3367a4ecbc20083367cd2e122115269b444da182 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Mon, 12 Oct 2026 21:53:36 +0000 Subject: [PATCH 11/20] ate-setup: roll ate-api-server once when its environment changes (#2371) The digest that turns an environment change into a rollout was patched onto the running Deployment before the deploy applied the new manifest, and a fresh install skipped it. So the first `deploy apiserver` after an install actually **rolled twice** whenever it brought a new image: once for the new annotation on the old image, then again for the new image. When the environment did change, the old image came up on the new values before the new image landed. A deploy now stamps the digest on the Deployment it applies, so a new image and a new environment are one rollout and a fresh install starts out stamped. A deploy whose manifest has no such Deployment now fails rather than applying it without the digest. `create api-server-env-vars`, which applies no Deployment, still patches the running one. Made with help from Claude Code Part of https://github.com/agent-substrate/substrate/issues/1887 and it unblocks https://github.com/agent-substrate/substrate/issues/1686 --- cmd/ate-setup/internal/cmd/create.go | 2 +- cmd/ate-setup/internal/steps/apienv.go | 77 +++++++++++++++--- cmd/ate-setup/internal/steps/apienv_test.go | 90 +++++++++++++++++++++ cmd/ate-setup/internal/steps/deploy.go | 8 +- 4 files changed, 163 insertions(+), 14 deletions(-) diff --git a/cmd/ate-setup/internal/cmd/create.go b/cmd/ate-setup/internal/cmd/create.go index 7dde06aa92..a0b4da52e6 100644 --- a/cmd/ate-setup/internal/cmd/create.go +++ b/cmd/ate-setup/internal/cmd/create.go @@ -74,7 +74,7 @@ func init() { if err := e.EnsureEnvVarsSafeStandalone(ctx); err != nil { return err } - return e.CreateAPIServerEnvVars(ctx) + return e.UpdateAPIServerEnvVars(ctx) }, }, { diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 2669a533d2..43e2e26dea 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -23,6 +23,9 @@ import ( "os" "slices" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" ) @@ -41,6 +44,10 @@ const envHashAnnotation = "ate.dev/env-hash" // resolves the PostgreSQL connection, role, and schema flags from // the result. It lists the secretRef last, so the Secret wins over a DSN a // previous installer left in the ConfigMap. +// +// It does not restart ate-api-server. A deploy stamps the new digest into the +// Deployment it applies; UpdateAPIServerEnvVars patches it for a run that +// applies no Deployment. func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { log.Step("create_api_server_env_vars") if err := e.Kube.EnsureNamespace(ctx, e.Namespace()); err != nil { @@ -121,7 +128,14 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { buildAPIServerEnvVars(readWriteDSN, ownerDSN, schema)); err != nil { return err } - if err := e.applyPostgresServerCA(ctx); err != nil { + return e.applyPostgresServerCA(ctx) +} + +// UpdateAPIServerEnvVars is CreateAPIServerEnvVars for an install that is not +// being redeployed: it also stamps the new digest on the running Deployment, +// which rolls ate-api-server onto the new values. +func (e *Env) UpdateAPIServerEnvVars(ctx context.Context) error { + if err := e.CreateAPIServerEnvVars(ctx); err != nil { return err } return e.annotateAPIServerEnvHash(ctx) @@ -203,27 +217,71 @@ func (e *Env) EnsureEnvVarsSafeStandalone(ctx context.Context) error { "Run `ate-setup deploy ate-apiserver` instead, which also updates the Deployment", SecretAPIEnvVars) } -// annotateAPIServerEnvHash stamps the pod template with a digest of the -// apiserver's environment, so that a changed DSN starts a rollout. Kubernetes -// does not restart pods when an envFrom ConfigMap or Secret changes. +// annotateAPIServerEnvHash stamps the running Deployment's pod template with +// the digest, so that a changed DSN starts a rollout. Kubernetes does not +// restart pods when an envFrom ConfigMap or Secret changes. func (e *Env) annotateAPIServerEnvHash(ctx context.Context) error { dep, err := e.Kube.GetDeployment(ctx, e.Namespace(), "ate-api-server") if err != nil { return err } if dep == nil { - // Fresh install: the first rollout starts with the new values. + // Nothing runs yet; the deploy that creates it stamps the digest. return nil } + hash, err := e.apiServerEnvHash(ctx) + if err != nil { + return err + } + patch := fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{%q:%q}}}}}`, + envHashAnnotation, hash) + return e.Kube.PatchDeployment(ctx, e.Namespace(), "ate-api-server", []byte(patch)) +} - cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars) +// applyWithAPIServerEnvHash applies a manifest that carries the ate-api-server +// Deployment, with the digest of its current environment on the pod template. +// A new image and a new environment are then one rollout, and a fresh install +// starts out stamped. +func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) error { + objs, err := kube.DecodeManifestBytes(manifest) if err != nil { return err } - secret, err := e.Kube.GetSecret(ctx, e.Namespace(), SecretAPIEnvVars) + hash, err := e.apiServerEnvHash(ctx) if err != nil { return err } + if err := stampAPIServerEnvHash(objs, hash); err != nil { + return err + } + return e.Kube.Apply(ctx, objs) +} + +// stampAPIServerEnvHash sets the digest on the ate-api-server Deployment's pod +// template and leaves every other object alone. A manifest without that +// Deployment is an error: applying it unstamped would stop a changed +// environment from rolling. +func stampAPIServerEnvHash(objs []*unstructured.Unstructured, hash string) error { + for _, obj := range objs { + if obj.GetKind() != "Deployment" || obj.GetName() != "ate-api-server" { + continue + } + return unstructured.SetNestedField(obj.Object, hash, + "spec", "template", "metadata", "annotations", envHashAnnotation) + } + return fmt.Errorf("the manifest has no deployment/ate-api-server to stamp with %s", envHashAnnotation) +} + +// apiServerEnvHash digests the apiserver's environment as it is in the cluster. +func (e *Env) apiServerEnvHash(ctx context.Context) (string, error) { + cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars) + if err != nil { + return "", err + } + secret, err := e.Kube.GetSecret(ctx, e.Namespace(), SecretAPIEnvVars) + if err != nil { + return "", err + } var cmData map[string]string if cm != nil { cmData = cm.Data @@ -232,10 +290,7 @@ func (e *Env) annotateAPIServerEnvHash(ctx context.Context) error { if secret != nil { secretData = secret.Data } - - patch := fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{%q:%q}}}}}`, - envHashAnnotation, envHash(cmData, secretData)) - return e.Kube.PatchDeployment(ctx, e.Namespace(), "ate-api-server", []byte(patch)) + return envHash(cmData, secretData), nil } // envHash digests the apiserver's environment sources. Only changes matter, so diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index f47616e465..16b28d78c3 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -21,8 +21,10 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" "github.com/agent-substrate/substrate/pkg/postgressetup" ) @@ -412,3 +414,91 @@ func TestAnnotateAPIServerEnvHash(t *testing.T) { } }) } + +// CreateAPIServerEnvVars only writes the environment. The deploy that runs +// it then applies the Deployment with the new digest, so ate-api-server rolls +// once, onto the new image. Only create api-server-env-vars, which applies no +// Deployment, patches the running one. +func TestOnlyAStandaloneEnvUpdateRollsTheAPIServer(t *testing.T) { + newEnv := func(t *testing.T) *Env { + return &Env{Cfg: &config.Config{PostgresReadWriteConnectionString: "postgres://runtime@postgres/atepg"}, + Kube: fakeKube(t, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: NamespaceAteSystem}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: ConfigMapAPIEnvVars, Namespace: NamespaceAteSystem}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: SecretAPIEnvVars, Namespace: NamespaceAteSystem}}, + apiServerDeployment(SecretAPIEnvVars), + )} + } + stamped := func(t *testing.T, e *Env) bool { + dep, err := e.Kube.GetDeployment(t.Context(), NamespaceAteSystem, "ate-api-server") + if err != nil { + t.Fatalf("GetDeployment() error = %v", err) + } + _, ok := dep.Spec.Template.Annotations[envHashAnnotation] + return ok + } + + t.Run("deploy", func(t *testing.T) { + e := newEnv(t) + if err := e.CreateAPIServerEnvVars(t.Context()); err != nil { + t.Fatalf("CreateAPIServerEnvVars() error = %v", err) + } + if stamped(t, e) { + t.Errorf("CreateAPIServerEnvVars() patched the running Deployment; the deploy's apply should carry the digest") + } + }) + t.Run("standalone", func(t *testing.T) { + e := newEnv(t) + if err := e.UpdateAPIServerEnvVars(t.Context()); err != nil { + t.Fatalf("UpdateAPIServerEnvVars() error = %v", err) + } + if !stamped(t, e) { + t.Errorf("UpdateAPIServerEnvVars() left the running Deployment unstamped, so it would not roll") + } + }) +} + +// The applied Deployment carries the digest, so one apply rolls a new image +// and a new environment together, and a fresh install starts out stamped. A +// manifest without it fails instead of applying unstamped. +func TestStampAPIServerEnvHash(t *testing.T) { + objs, err := kube.DecodeManifestBytes([]byte(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-api-server + namespace: ate-system +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-controller + namespace: ate-system +spec: + template: + metadata: {} +`)) + if err != nil { + t.Fatalf("DecodeManifestBytes() error = %v", err) + } + if err := stampAPIServerEnvHash(objs, "digest"); err != nil { + t.Fatalf("stampAPIServerEnvHash() error = %v", err) + } + annotations := func(obj *unstructured.Unstructured) map[string]string { + a, _, _ := unstructured.NestedStringMap(obj.Object, "spec", "template", "metadata", "annotations") + return a + } + if got := annotations(objs[0]); got[envHashAnnotation] != "digest" || got["prometheus.io/scrape"] != "true" { + t.Errorf("ate-api-server pod template annotations = %v, want the digest added to the existing ones", got) + } + if got := annotations(objs[1]); got[envHashAnnotation] != "" { + t.Errorf("ate-controller pod template annotations = %v, want no digest", got) + } + if err := stampAPIServerEnvHash(objs[1:], "digest"); err == nil { + t.Errorf("stampAPIServerEnvHash() without ate-api-server error = nil, want an error") + } +} diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 0d3e981a9c..77a9f98a2e 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -145,7 +145,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err != nil { return err } - if err := e.Kube.ApplyBytes(ctx, manifests); err != nil { + if err := e.applyWithAPIServerEnvHash(ctx, manifests); err != nil { return err } @@ -311,7 +311,11 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { return err } } - if err := e.renderResolveApply(ctx, e.Cfg.Manifest("ate-api-server.yaml")); err != nil { + manifest, err := e.renderResolve(ctx, e.Cfg.Manifest("ate-api-server.yaml")) + if err != nil { + return err + } + if err := e.applyWithAPIServerEnvHash(ctx, manifest); err != nil { return err } // After the manifest, which resets the pod template to the sidecar-free base. From 29bebd3a07eef7e22e261ded31bd74d49b7cc833 Mon Sep 17 00:00:00 2001 From: Yufan Su Date: Mon, 12 Oct 2026 22:17:11 +0000 Subject: [PATCH 12/20] e2e: update credential injection e2e to call an in-cluster echo server (#2535) #2513 ### Changes - Re-enable the credential injection e2e which was skipped in #2518 - `testserver http` gains `/echoheaders`, which returns the request headers it received as `{"headers": {...}}`. - The suite deploys two echo origins in its own namespace: - `echo-https` on 443, serving a certificate from the cluster's service-DNS signer through a projected `podCertificate` volume. The egress gateway trusts that signer once `hack/setup-e2e-egress-tls-kind.sh` has run, and CI already runs it. - `echo-http` on 80. - The fail-closed hosts are `.invalid` names that exist only in the policy. - Coverage is unchanged: injection and no-header checks over HTTPS and plain HTTP, plus the three fail-closed cases (unfetchable secret, unserved provider, unauthorized namespace) on both schemes. ### Testing - [x] Tests pass - [ ] Appropriate changes to documentation are included in the PR --- internal/e2e/fixtures/testserver/http.go | 24 ++- internal/e2e/fixtures/testserver/http_test.go | 48 ++++++ internal/e2e/fixtures/testserver/main.go | 2 +- .../egresscredinject/egresscredinject_test.go | 141 ++++++++++++------ 4 files changed, 168 insertions(+), 47 deletions(-) diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 3cf941d8ca..9f1a885470 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -15,20 +15,30 @@ package main import ( + "encoding/json" "io" "log" "net/http" + "strings" "time" "github.com/spf13/cobra" ) +// echoedHeaders is the /echoheaders response body. +type echoedHeaders struct { + // Headers holds the request headers the origin received, keyed by + // canonical name, with repeated values joined by ",". + Headers map[string]string `json:"headers"` +} + // newHTTPHandler is the HTTP/1.1 origin an Actor's egress lands on. It exists so // a test can assert the destination port is recovered from SO_ORIGINAL_DST // rather than defaulted from the URL scheme: the actor fetches its /healthz on a // non-standard port, and the gateway's access log is expected to carry that // port. The optional body makes the same fixture useful for assertions about a -// response inside the egress tunnel. +// response inside the egress tunnel. /echoheaders returns the request headers as +// they arrived, which is how a suite sees what the gateway put on the wire. func newHTTPHandler(body string) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { @@ -37,6 +47,16 @@ func newHTTPHandler(body string) http.Handler { _, _ = io.WriteString(w, body) } }) + mux.HandleFunc("/echoheaders", func(w http.ResponseWriter, r *http.Request) { + echoed := echoedHeaders{Headers: make(map[string]string, len(r.Header))} + for name, values := range r.Header { + echoed.Headers[name] = strings.Join(values, ",") + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(echoed); err != nil { + log.Printf("testserver http: encoding /echoheaders response: %v", err) + } + }) return mux } @@ -49,7 +69,7 @@ func newHTTPCmd() *cobra.Command { ) cmd := &cobra.Command{ Use: "http", - Short: "Serve an HTTP/1.1 origin answering /healthz.", + Short: "Serve an HTTP/1.1 origin answering /healthz and /echoheaders.", Args: cobra.NoArgs, RunE: func(_ *cobra.Command, _ []string) error { server := &http.Server{ diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go index 45ab8f3713..78b6add9de 100644 --- a/internal/e2e/fixtures/testserver/http_test.go +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -17,8 +17,10 @@ package main import ( "crypto/tls" "crypto/x509" + "encoding/json" "io" "net/http" + "strings" "testing" "time" ) @@ -45,6 +47,52 @@ func TestHTTPHandlerDefaultsToEmptyHealthz(t *testing.T) { } } +func TestHTTPHandlerEchoesRequestHeaders(t *testing.T) { + server := startOriginServer(t, newHTTPHandler(""), "", "") + + // Long enough to catch a truncated value, the size of a JWT. + token := strings.Repeat("eyJhbGciOiJFUzI1NiJ9.", 64) + request, err := http.NewRequest(http.MethodGet, server.httpURL+"/echoheaders", nil) + if err != nil { + t.Fatalf("building request: %v", err) + } + // Sent lowercase on the wire; the origin reports it canonicalized. + request.Header["authorization"] = []string{"Bearer placeholder"} + request.Header.Set("X-Actor-Token", token) + request.Header.Add("X-Repeated", "one") + request.Header.Add("X-Repeated", "two") + + client := &http.Client{Timeout: 2 * time.Second} + response, err := client.Do(request) + if err != nil { + t.Fatalf("GET /echoheaders: %v", err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Fatalf("GET /echoheaders status = %d, want %d", response.StatusCode, http.StatusOK) + } + if got := response.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("GET /echoheaders Content-Type = %q, want application/json", got) + } + var echoed echoedHeaders + if err := json.NewDecoder(response.Body).Decode(&echoed); err != nil { + t.Fatalf("decoding /echoheaders: %v", err) + } + for name, want := range map[string]string{ + "Authorization": "Bearer placeholder", + "X-Actor-Token": token, + "X-Repeated": "one,two", + } { + if got := echoed.Headers[name]; got != want { + t.Errorf("echoed %s = %q, want %q", name, got, want) + } + } + if got, ok := echoed.Headers["X-Absent"]; ok { + t.Errorf("echoed X-Absent = %q for a header the request did not carry, want it missing", got) + } +} + func TestHTTPSTestResponseUsesHTTP11AndVerifiedIdentity(t *testing.T) { cert := writeOriginCertificate(t) server := startOriginServer(t, newHTTPHandler("fixture response"), cert.certFile, cert.keyFile) diff --git a/internal/e2e/fixtures/testserver/main.go b/internal/e2e/fixtures/testserver/main.go index 6393712f61..addd7b57d5 100644 --- a/internal/e2e/fixtures/testserver/main.go +++ b/internal/e2e/fixtures/testserver/main.go @@ -18,7 +18,7 @@ // than a new image to build and push: // // testserver grpc --listen=:50051 a cleartext HTTP/2 gRPC echo origin -// testserver http --listen=:8080 a plain HTTP origin serving /healthz +// testserver http --listen=:8080 an HTTP(S) origin serving /healthz and /echoheaders // testserver egressprobe --listen=:8080 a client that drives the egress gateway // testserver websocket --listen=:8080 a websocket server that responds to PINGs // diff --git a/internal/e2e/suites/egresscredinject/egresscredinject_test.go b/internal/e2e/suites/egresscredinject/egresscredinject_test.go index d15d84a455..919974f40f 100644 --- a/internal/e2e/suites/egresscredinject/egresscredinject_test.go +++ b/internal/e2e/suites/egresscredinject/egresscredinject_test.go @@ -26,6 +26,7 @@ import ( "io" "net/http" "net/url" + "path" "slices" "strings" "testing" @@ -36,26 +37,55 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" ) const probeTemplate = "probe" var probeNamespace string -// The suite's hostnames, one per injection outcome: each is covered by exactly -// one https rule, so each selects exactly one CredentialHeader. echoHost is the -// only one whose response matters — it echoes the request headers it -// received back as JSON, which is what proves the header was on the wire. +// servingBundle is the echo-https pod's certificate chain and key, in one file. +const servingBundle = "/run/servicedns.podcert.ate.dev/credential-bundle.pem" + +// The origins the injection fetches reach, one per scheme. +var ( + // echoHTTPSServer serves the certificate kubelet requests for it from the + // cluster's service-DNS signer, which names the Services selecting the pod + // as ..svc. The egress gateway trusts that signer once + // hack/setup-e2e-egress-tls-kind.sh has run. + echoHTTPSServer = e2e.ServerPod{ + Name: "echo-https", + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--tls-cert=" + servingBundle, "--tls-key=" + servingBundle}, + Port: 443, + TargetPort: 8443, + HealthScheme: corev1.URISchemeHTTPS, + Volumes: []corev1.Volume{{ + Name: "servicedns", + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{PodCertificate: &corev1.PodCertificateProjection{ + SignerName: "servicedns.podcert.ate.dev/identity", + KeyType: "ECDSAP256", + CredentialBundlePath: path.Base(servingBundle), + }}}, + }}, + }}, + VolumeMounts: []corev1.VolumeMount{{Name: "servicedns", MountPath: path.Dir(servingBundle), ReadOnly: true}}, + } + echoHTTPServer = e2e.ServerPod{ + Name: "echo-http", + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http"}, + Port: 80, + TargetPort: 8080, + } +) + +// The hosts of the fail-closed cases. const ( - echoHost = "httpbin.org" - unfetchableHost = "example.com" - unservedHost = "example.org" - unauthorizedHost = "example.net" - echoOrigin = "https://" + echoHost + "/headers" - echoOriginPlain = "http://" + echoHost + "/headers" - unfetchableOrigin = "https://" + unfetchableHost + "/" - unservedOrigin = "https://" + unservedHost + "/" - unauthorizedOrigin = "https://" + unauthorizedHost + "/" + unfetchableHost = "unfetchable.egresscredinject.invalid" + unservedHost = "unserved.egresscredinject.invalid" + unauthorizedHost = "unauthorized.egresscredinject.invalid" ) // placeholder is the Authorization value the actor sends. replace_headers @@ -83,17 +113,18 @@ var withPlaceholder = []string{"header=" + url.QueryEscape("Authorization:"+plac // not or cannot produce denies the request — 403 for an unfetchable // secret and for a namespace outside the atespace's authorization // (default-deny), 403 or 500 for a URI naming a provider this gateway -// does not serve. +// does not serve — over HTTPS and plain HTTP alike. // -// This needs the install made with the bundled provider, which -// deploys the k8s-credential-provider and points the egress gateway at it. -// Locally: +// The echo origins run in the cluster, so the HTTPS one needs the gateway to +// trust the service-DNS signer its certificate comes from, which +// hack/setup-e2e-egress-tls-kind.sh adds after the install. The install must +// be made with the bundled provider, which deploys the k8s-credential-provider +// and points the egress gateway at it. Locally: // // hack/install-ate-kind.sh --deploy-atenet --credential-provider='{"name":"k8s.io"}' +// hack/setup-e2e-egress-tls-kind.sh // hack/run-e2e-kind.sh ./internal/e2e/suites/egresscredinject -v -args --no-color func TestActorEgressCredentialInjection(t *testing.T) { - t.Skip("disabled until the echo origin is in-cluster: httpbin.org outages fail CI for every PR (issue 2513)") - env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") if err != nil { t.Fatalf("CheckEnv failed: %v", err) @@ -101,12 +132,20 @@ func TestActorEgressCredentialInjection(t *testing.T) { ctx := context.Background() clients := e2e.GetClients() + echoHTTPS := e2e.DeployServerPod(t, ctx, echoHTTPSServer) + httpSpec := echoHTTPServer + httpSpec.Namespace = echoHTTPS.Namespace + echoHTTP := e2e.DeployServerPod(t, ctx, httpSpec) + // The names the service-DNS signer certifies. + httpsHost := echoHTTPSServer.Name + "." + echoHTTPS.Namespace + ".svc" + httpHost := echoHTTPServer.Name + "." + echoHTTP.Namespace + ".svc" + e2e.ConfigureCredentialProvider(t) probeNamespace, _ = e2e.DeployProbe(t, env["BUCKET_NAME"], "egresscredinject", e2e.WithTrustBundle()) const id = "probe-credinject" - createAndResumeActor(t, ctx, clients, id) + createAndResumeActor(t, ctx, clients, id, httpsHost, httpHost) waitForActorState(t, ctx, clients, id, ateapipb.ActorState_ACTOR_STATE_RUNNING) rc, err := e2e.NewRouterClient(ctx) @@ -115,10 +154,22 @@ func TestActorEgressCredentialInjection(t *testing.T) { } defer rc.Close() - for _, origin := range []string{echoOrigin, echoOriginPlain} { - t.Run(origin, func(t *testing.T) { + // HTTPS is injected on the gateway's MITM leg and plain HTTP on its + // cleartext leg, each by a rule with the same effect. + legs := []struct { + scheme string + host string + echo e2e.Server + }{ + {"https", httpsHost, echoHTTPS}, + {"http", httpHost, echoHTTP}, + } + for _, leg := range legs { + t.Run(leg.scheme, func(t *testing.T) { + origin := leg.scheme + "://" + leg.host + "/echoheaders" + // The upstream must receive the policy's credential instead of the - // actor's placeholder, on both HTTP and HTTPS. + // actor's placeholder. wantHeader := "Bearer " + e2e.CredentialInjectionToken replaced := fetchEcho(t, ctx, rc, id, origin, withPlaceholder) if got := assertEchoedAuthorization(t, "injection fetch", replaced); got != wantHeader { @@ -139,26 +190,28 @@ func TestActorEgressCredentialInjection(t *testing.T) { // itself. tests := []struct { name string - origin string + host string wantStatuses []string }{{ name: "unfetchable secret", - origin: unfetchableOrigin, + host: unfetchableHost, wantStatuses: []string{"403"}, }, { // Agentgateway denies an unserved provider as 403; Envoy uses 500. name: "unserved provider", - origin: unservedOrigin, + host: unservedHost, wantStatuses: []string{"403", "500"}, }, { name: "unauthorized namespace", - origin: unauthorizedOrigin, + host: unauthorizedHost, wantStatuses: []string{"403"}, }} for _, tt := range tests { - for _, origin := range []string{tt.origin, strings.Replace(tt.origin, "https://", "http://", 1)} { - t.Run(tt.name+"/"+origin, func(t *testing.T) { - got := probeFetch(t, ctx, rc, id, origin, withPlaceholder) + for _, leg := range legs { + t.Run(tt.name+"/"+leg.scheme, func(t *testing.T) { + origin := leg.scheme + "://" + tt.host + "/" + withDial := slices.Concat(withPlaceholder, []string{"dial=" + url.QueryEscape(leg.echo.Address())}) + got := probeFetch(t, ctx, rc, id, origin, withDial) if got.Error != "" { t.Fatalf("fetch of %s failed at the transport (%s), want an HTTP denial %v from the gateway", origin, got.Error, tt.wantStatuses) } @@ -176,8 +229,7 @@ const echoRetryWindow = 2 * time.Minute // transient failures for up to echoRetryWindow: // // - certificate errors, until the gateway's signing pool propagates; -// - 503, while the gateway reconnects to a redeployed provider; -// - 502/503/504 from httpbin.org itself. +// - 503, while the gateway reconnects to a redeployed provider. func fetchEcho(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id, origin string, extraParams []string) fetchResponse { t.Helper() deadline := time.Now().Add(echoRetryWindow) @@ -196,15 +248,11 @@ func transientEchoFailure(resp fetchResponse) bool { if resp.Error != "" { return strings.Contains(resp.Error, "certificate") || strings.Contains(resp.Error, "x509") } - switch resp.Status { - case "502", "503", "504": - return true - } - return false + return resp.Status == "503" } // echoedHeaders is the echo origin's response shape: the request headers it -// received, echoed back. (httpbin.org/headers returns {"headers": {...}}.) +// received, echoed back by testserver's /echoheaders. type echoedHeaders struct { Headers map[string]string `json:"headers"` } @@ -228,7 +276,7 @@ func assertEchoedAuthorization(t *testing.T, step string, resp fetchResponse) st t.Fatalf("%s: TLS through the MITM egress gateway failed: %s", step, resp.Error) } if resp.Status != "200" { - t.Fatalf("%s: status %s, want 200 (an injection failure would deny with 403/500/503; is the provider deployed and the gateway installed with --credential-provider='{\"name\":\"k8s.io\"}'?) body %q", step, resp.Status, resp.Body) + t.Fatalf("%s: status %s, want 200 (an injection failure would deny with 403/500/503; is the provider deployed, the gateway installed with --credential-provider='{\"name\":\"k8s.io\"}', and hack/setup-e2e-egress-tls-kind.sh run so the gateway trusts the echo origin's certificate?) body %q", step, resp.Status, resp.Body) } return decodeEchoedHeaders(t, step, resp.Body)["Authorization"] } @@ -279,8 +327,10 @@ func probeFetch(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id, ori } // createAndResumeActor first deletes any actor left by an earlier run, since -// actor records outlive the fixture namespace. -func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, id string) { +// actor records outlive the fixture namespace. echoHTTPSHost and echoHTTPHost +// are the echo origins' names, whose rules inject a credential the provider +// resolves. +func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, id, echoHTTPSHost, echoHTTPHost string) { t.Helper() ref := &ateapipb.ObjectRef{Atespace: probeNamespace, Name: id} // NotFound is the normal case on a fresh run. Other errors don't stop the @@ -305,13 +355,16 @@ func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Client t.Logf("cleanup: DeleteActor %q failed, actor leaked (remove with: kubectl ate delete actor %s -a %s): %v", id, id, probeNamespace, err) } }) - // Exercise each credential outcome on both HTTP and HTTPS. - var rules []*ateapipb.EgressRule + // Exercise each credential outcome on both HTTP and HTTPS: the echo + // origins' credential resolves, and each fail-closed host's cannot. + rules := []*ateapipb.EgressRule{ + e2e.EgressInjectHeader("Authorization", "Bearer ", e2e.CredentialInjectionURI, echoHTTPSHost), + e2e.EgressInjectHeaderHTTP("Authorization", "Bearer ", e2e.CredentialInjectionURI, echoHTTPHost), + } for _, credential := range []struct { host string uri string }{ - {echoHost, e2e.CredentialInjectionURI}, {unfetchableHost, "ate-secret://k8s.io/default/" + e2e.CredentialSecretsNamespace + "/no-such-secret/token"}, {unservedHost, "ate-secret://other.io/default/" + e2e.CredentialSecretsNamespace + "/api-token/token"}, {unauthorizedHost, "ate-secret://k8s.io/default/kube-system/api-token/token"}, From 16ccb1304b2a438fee7c9d1337f778c770517d45 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Mon, 12 Oct 2026 23:03:45 +0000 Subject: [PATCH 13/20] Fetch private sandbox assets through the object-store plugin (#2439) atelet held its own storage credential to download sandbox assets, and private micro-VM assets only worked on GCS or S3. Assets now go through the node's object-store plugin, which already has the credentials. - Add `AssetProvider.FetchAsset` to the plugin API. - The plugin refuses snapshot locations, stages downloads where atelet can't read them, and replaces the target only once the sha256 matches; `pkg/assetfetch` shares these checks with other plugins. - atelet asks the plugin first for every asset. Only for a `gs://` URL, and only when the plugin returns NotFound, InvalidArgument, Unimplemented or Unavailable, does it fall back to an anonymous read (which serves public buckets such as `gs://gvisor`). It never falls back when the plugin refuses a snapshot location or the size or digest is wrong. - atelet re-checks the size and sha256 of whatever it caches, since the plugin is a separate, possibly third-party process and atelet executes these bytes. Tested with the kind gVisor and micro-VM lanes before the plugin-first change; only the NFS external-volume tests failed, same as on main. Unit tests cover each fallback and non-fallback code. Related #2063: assets in the cluster's own store no longer wait on an anonymous GCS lookup first. #### Breaking change Plugins must implement `AssetProvider` to serve sandbox assets; public `gs://` assets still load anonymously if the plugin can't serve them. atelet exits if `ATE_STORAGE_BACKEND` is set on it; set storage settings on the snapshot-plugin sidecar. Please add the `breaking-change` label. Fixes #2385 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/atelet/main.go | 63 +- cmd/atelet/rpc_logging_test.go | 116 +++ cmd/atelet/sandbox_assets.go | 267 +++-- cmd/atelet/sandbox_assets_plugin_test.go | 666 +++++++++++++ cmd/atelet/snapshot_transfer.go | 30 +- cmd/atelet/snapshot_transfer_test.go | 81 ++ cmd/snapshot-plugin/main.go | 49 +- cmd/snapshot-plugin/main_test.go | 99 ++ docs/api-guide.md | 2 +- docs/architecture.md | 2 +- docs/threat-model.md | 2 +- docs/threats.json | 2 +- hack/install-microvm-deps.sh | 16 +- hack/microvm-assets/README.md | 4 + hack/microvm-assets/stage-to-gcs.sh | 7 +- hack/microvm-assets/stage-to-rustfs.sh | 5 +- internal/objectstoreplugin/asset.go | 107 ++ internal/objectstoreplugin/asset_test.go | 495 ++++++++++ .../objectstoreplugintest.go | 13 + internal/proto/ateletpb/atelet.pb.go | 12 +- internal/proto/ateletpb/atelet.proto | 8 +- internal/protoredact/protoredact_test.go | 14 +- manifests/ate-install/atelet.yaml | 31 +- .../generated/ate.dev_sandboxconfigs.yaml | 11 +- .../kind/atelet/kustomization.yaml | 20 +- .../microvm/sandboxconfig-microvm.yaml.tmpl | 26 +- pkg/api/v1alpha1/sandboxconfig_types.go | 11 +- .../v1alpha1/sandboxconfig_validation_test.go | 2 +- pkg/assetfetch/assetfetch.go | 446 +++++++++ pkg/assetfetch/assetfetch_test.go | 916 ++++++++++++++++++ pkg/assetfetch/exchange_linux.go | 37 + pkg/assetfetch/exchange_other.go | 34 + pkg/objectstorage/gcsranged.go | 3 + pkg/objectstorage/objects.go | 21 + pkg/objectstorage/objectsize_test.go | 126 +++ pkg/objectstorage/rangedget.go | 5 + pkg/objectstorage/s3ranged.go | 23 +- pkg/proto/objectstorepb/v1/objectstore.pb.go | 153 ++- pkg/proto/objectstorepb/v1/objectstore.proto | 47 + .../objectstorepb/v1/objectstore_grpc.pb.go | 140 +++ 40 files changed, 3950 insertions(+), 162 deletions(-) create mode 100644 cmd/atelet/rpc_logging_test.go create mode 100644 cmd/atelet/sandbox_assets_plugin_test.go create mode 100644 cmd/snapshot-plugin/main_test.go create mode 100644 internal/objectstoreplugin/asset.go create mode 100644 internal/objectstoreplugin/asset_test.go create mode 100644 pkg/assetfetch/assetfetch.go create mode 100644 pkg/assetfetch/assetfetch_test.go create mode 100644 pkg/assetfetch/exchange_linux.go create mode 100644 pkg/assetfetch/exchange_other.go create mode 100644 pkg/objectstorage/objectsize_test.go diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 971feef65d..a829fff68a 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -114,7 +114,7 @@ var ( drainDelay = pflag.Duration("drain-delay", 0, "How long to keep accepting new RPCs after SIGTERM before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 5*time.Minute, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") - snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/node.sock", "Unix socket of the node snapshot plugin that external snapshots are fetched and uploaded through.") + snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/node.sock", "Unix socket of the node object-store plugin that external snapshots are fetched and uploaded through, and every sandbox asset is fetched through. A gs:// asset the plugin cannot serve is then read anonymously.") ) var _ imagecache.CandidateKeychain = (*credentialprovider.Keychain)(nil) @@ -131,6 +131,9 @@ func main() { serverboot.Fatal(ctx, "Invalid --log-level", err) } slog.InfoContext(ctx, "atelet starting", slog.String("version", version.Version)) + if err := rejectStorageEnv(); err != nil { + serverboot.Fatal(ctx, "Invalid environment", err) + } // Kept separate from ctx so in-flight work (e.g. a Checkpoint/Restore // streaming a multi-GiB snapshot) is not cancelled the moment SIGTERM @@ -228,16 +231,14 @@ func main() { go newImageCacheGC(imageCache, *imageCacheDir).Run(ctx) } + // Sandbox assets go through the object-store plugin; a gs:// asset it + // cannot serve, such as one in the public gs://gvisor bucket, is read + // anonymously. wrappedAnonGCS, err := objectstorage.NewGCSClient(ctx, option.WithoutAuthentication()) if err != nil { serverboot.Fatal(ctx, "Failed to create anonymous GCS client", err) } - wrappedGCS, err := objectstorage.NewFromEnv(ctx) - if err != nil { - serverboot.Fatal(ctx, "Failed to set up the object storage backend", err) - } - volPlugins := make(map[string]volume.VolumePluginWorkerPlane) k8sClient, ateClient, err := newKubeClients() if err != nil { @@ -301,8 +302,9 @@ func main() { ctx, ateomDialer, wrappedAnonGCS, - wrappedGCS, objectstorev1.NewNodeProviderClient(snapshotPluginConn), + objectstorev1.NewAssetProviderClient(snapshotPluginConn), + *snapshotPluginSocket, imageCache, instruments, volPlugins, @@ -388,11 +390,7 @@ func main() { } }() - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + svr := grpc.NewServer(ateomHerderServerOptions(credentials.NewTLS(tlsCfg))...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) @@ -405,6 +403,28 @@ func main() { slog.InfoContext(ctx, "Shutdown complete") } +// ateomHerderServerOptions are the options of the gRPC server that serves +// AteomHerder to ate-api-server. Its interceptor logs every request; fields +// marked debug_redact (env values, asset URLs) are masked in that log. +func ateomHerderServerOptions(creds credentials.TransportCredentials) []grpc.ServerOption { + return []grpc.ServerOption{ + grpc.Creds(creds), + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), + } +} + +// rejectStorageEnv fails if ATE_STORAGE_BACKEND is set on atelet, which no +// longer reads it: object storage settings belong on the snapshot-plugin +// sidecar. Only this variable is a signal: EKS pod identity sets AWS_* in +// every container. +func rejectStorageEnv() error { + if v, ok := os.LookupEnv("ATE_STORAGE_BACKEND"); ok { + return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on atelet, which no longer reads it; set the storage backend on its snapshot-plugin sidecar", v) + } + return nil +} + // drainOnShutdown drives graceful shutdown when ctx is cancelled (SIGTERM or // interrupt): it marks the process not-ready, waits drain-delay while still // accepting work, then GracefulStop()s the gRPC server so in-flight RPCs finish. @@ -451,14 +471,17 @@ func (g *directCSIDriverConfigGetter) Get(name string) (*atev1alpha1.CSIDriverCo type AteomHerder struct { ateletpb.UnimplementedAteomHerderServer - ateomDialer *AteomDialer - imageCache *imagecache.Store + ateomDialer *AteomDialer + imageCache *imagecache.Store + // anonGCSClient reads gs:// assets the plugin cannot serve, without credentials. anonGCSClient objectstorage.ObjectStorage - // gcsClient downloads sandbox assets. Snapshot files never use it: they move - // through snapshotPlugin. - gcsClient objectstorage.ObjectStorage // snapshotPlugin moves external snapshot files between the node and storage. snapshotPlugin objectstorev1.NodeProviderClient + // assetPlugin fetches every sandbox asset first; anonGCSClient reads a gs:// + // asset it cannot serve. + assetPlugin objectstorev1.AssetProviderClient + // pluginSocket is the object-store plugin's socket, for error messages. + pluginSocket string // snapshotScratchDir holds short-lived manifest directories; it must be // inside the node plugin's root. snapshotScratchDir string @@ -476,8 +499,9 @@ func NewService( ctx context.Context, ateomDialer *AteomDialer, anonGCSClient objectstorage.ObjectStorage, - gcsClient objectstorage.ObjectStorage, snapshotPlugin objectstorev1.NodeProviderClient, + assetPlugin objectstorev1.AssetProviderClient, + pluginSocket string, imageCache *imagecache.Store, instruments *Instruments, volumePlugins map[string]volume.VolumePluginWorkerPlane, @@ -488,8 +512,9 @@ func NewService( ateomDialer: ateomDialer, imageCache: imageCache, anonGCSClient: anonGCSClient, - gcsClient: gcsClient, snapshotPlugin: snapshotPlugin, + assetPlugin: assetPlugin, + pluginSocket: pluginSocket, snapshotScratchDir: ateletpath.SnapshotScratchDir, instruments: instruments, volumePlugins: volumePlugins, diff --git a/cmd/atelet/rpc_logging_test.go b/cmd/atelet/rpc_logging_test.go new file mode 100644 index 0000000000..e9005599c2 --- /dev/null +++ b/cmd/atelet/rpc_logging_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "log/slog" + "net" + "strings" + "testing" + + "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/protoredact" + "github.com/agent-substrate/substrate/internal/serverboot" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" +) + +// requestLogHerder answers Run and fails Restore, so the test covers the +// request log of a handled and of a failed RPC. +type requestLogHerder struct { + ateletpb.UnimplementedAteomHerderServer +} + +func (requestLogHerder) Run(context.Context, *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { + return &ateletpb.RunResponse{}, nil +} + +func (requestLogHerder) Restore(context.Context, *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { + return nil, apierror.Unavailable("asset fetch failed") +} + +// TestAteomHerderRequestLogMasksSignedAssetURL sends Run and Restore with a +// signed https asset URL through a server built with atelet's own options and +// logger, and checks the request log carries neither the URL's user info nor +// its query. +func TestAteomHerderRequestLogMasksSignedAssetURL(t *testing.T) { + const ( + signedURL = "https://asset-user:userinfo-secret@storage.example.com/bucket/gvisor.tar.zstd?X-Goog-Signature=query-secret&X-Goog-Expires=900" + sum = "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + ) + var logBuf bytes.Buffer + origLogger := slog.Default() + t.Cleanup(func() { slog.SetDefault(origLogger) }) + serverboot.InitLoggerWithWriter(&logBuf) + + lis := bufconn.Listen(1 << 20) + srv := grpc.NewServer(ateomHerderServerOptions(insecure.NewCredentials())...) + ateletpb.RegisterAteomHerderServer(srv, requestLogHerder{}) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + conn, err := grpc.NewClient("passthrough:///bufnet", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + client := ateletpb.NewAteomHerderClient(conn) + + assets := &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + Assets: map[string]*ateletpb.ArchAssets{ + "amd64": {Files: map[string]*ateletpb.AssetFile{"gvisor": {Url: signedURL, Sha256: sum}}}, + }, + } + ctx := t.Context() + if _, err := client.Run(ctx, &ateletpb.RunRequest{ActorUid: "uid-run", SandboxAssets: assets}); err != nil { + t.Fatalf("Run: %v", err) + } + if _, err := client.Restore(ctx, &ateletpb.RestoreRequest{ActorUid: "uid-restore", SandboxAssets: assets}); status.Code(err) != codes.Unavailable { + t.Fatalf("Restore err = %v, want Unavailable", err) + } + srv.Stop() + + gotLog := logBuf.String() + for _, leak := range []string{"userinfo-secret", "query-secret", "asset-user", "X-Goog-Signature"} { + if strings.Contains(gotLog, leak) { + t.Fatalf("request log contains %q from the signed asset URL:\n%s", leak, gotLog) + } + } + for _, method := range []string{ateletpb.AteomHerder_Run_FullMethodName, ateletpb.AteomHerder_Restore_FullMethodName} { + var line string + for l := range strings.Lines(gotLog) { + if strings.Contains(l, `"method":"`+method+`"`) { + line = l + } + } + if line == "" { + t.Fatalf("no request log line for %s:\n%s", method, gotLog) + } + // The sha256 still names the asset. + for _, want := range []string{`"url":"` + protoredact.Placeholder + `"`, `"sha256":"` + sum + `"`} { + if !strings.Contains(line, want) { + t.Fatalf("%s request log missing %s:\n%s", method, want, line) + } + } + } +} diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 0deb76889f..305c2f667e 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -30,6 +30,7 @@ import ( "io" "io/fs" "log/slog" + "net/url" "os" "path/filepath" "runtime" @@ -39,11 +40,15 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" + "github.com/agent-substrate/substrate/internal/apierror" "github.com/agent-substrate/substrate/internal/nodepath" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // sandboxManifestName is the object/file name of the per-snapshot manifest that @@ -72,6 +77,12 @@ type assetEntry struct { SHA256 string `json:"sha256"` } +// displayURL returns the asset URL as scheme://host/path, for logs and +// errors. +func (e assetEntry) displayURL() string { + return redactURLString(e.URL) +} + // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto // the local node's architecture: the sandbox class and pause image plus the // asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball @@ -198,7 +209,7 @@ func (s *AteomHerder) fetchAsset(ctx context.Context, entry assetEntry) (string, return "", wrapFileSystemErr("while stat-ing local file", err) } - slog.InfoContext(ctx, "Sandbox asset cache miss; downloading", slog.String("url", entry.URL), slog.String("sha256", entry.SHA256)) + slog.InfoContext(ctx, "Sandbox asset cache miss; downloading", slog.String("url", entry.displayURL()), slog.String("sha256", entry.SHA256)) t := time.Now() tmpName, err := s.downloadVerified(ctx, entry, filepath.Base(localPath)+"-download-") if err != nil { @@ -236,14 +247,14 @@ func (s *AteomHerder) fetchGVisorRelease(ctx context.Context, entry assetEntry) return "", wrapFileSystemErr("while stat-ing extracted release dir", err) } - slog.InfoContext(ctx, "gVisor release cache miss; downloading", slog.String("url", entry.URL), slog.String("sha256", entry.SHA256)) + slog.InfoContext(ctx, "gVisor release cache miss; downloading", slog.String("url", entry.displayURL()), slog.String("sha256", entry.SHA256)) tDownload := time.Now() tarball, err := s.downloadVerified(ctx, entry, filepath.Base(releaseDir)+"-download-") if err != nil { return "", err } defer os.Remove(tarball) - slog.InfoContext(ctx, "gVisor release download complete", slog.String("url", entry.URL), slog.Duration("duration", time.Since(tDownload))) + slog.InfoContext(ctx, "gVisor release download complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(tDownload))) tmpDir, err := os.MkdirTemp(nodepath.StaticFilesDir, filepath.Base(releaseDir)+"-extract-") if err != nil { @@ -251,14 +262,16 @@ func (s *AteomHerder) fetchGVisorRelease(ctx context.Context, entry assetEntry) } defer func() { _ = os.RemoveAll(tmpDir) }() // no-op after rename - slog.InfoContext(ctx, "Extracting gVisor archive", slog.String("path", tarball), slog.String("url", entry.URL)) + slog.InfoContext(ctx, "Extracting gVisor archive", slog.String("path", tarball), slog.String("url", entry.displayURL())) tExtract := time.Now() - if err := extractTarArchive(ctx, tarball, entry.URL, tmpDir); err != nil { + // The redacted form ends in the URL path, so a query does not hide the + // archive suffix. + if err := extractTarArchive(ctx, tarball, entry.displayURL(), tmpDir); err != nil { return "", err } - slog.InfoContext(ctx, "gVisor archive extraction complete", slog.String("url", entry.URL), slog.Duration("duration", time.Since(tExtract))) + slog.InfoContext(ctx, "gVisor archive extraction complete", slog.String("url", entry.displayURL()), slog.Duration("duration", time.Since(tExtract))) if fi, err := os.Stat(filepath.Join(tmpDir, "runsc")); err != nil || !fi.Mode().IsRegular() { - return "", fmt.Errorf("gvisor tarball %v contains no runsc binary (stat: %v)", entry.URL, err) + return "", fmt.Errorf("gvisor tarball %v contains no runsc binary (stat: %v)", entry.displayURL(), err) } if err := os.Chmod(tmpDir, 0o755); err != nil { // MkdirTemp created it 0700 return "", wrapFileSystemErr("while setting extraction dir mode", err) @@ -273,63 +286,225 @@ func (s *AteomHerder) fetchGVisorRelease(ctx context.Context, entry assetEntry) return runscPath, nil } -// downloadVerified streams entry.URL into a temp file in the static-files -// cache, hashing as it goes, and returns the temp path once the size cap and -// sha256 both check out. On error the temp file is removed; on success the -// caller owns it (rename it into place or extract from it, then remove it). +// downloadVerified fetches entry.URL into a temp file in the static-files +// cache and returns the temp path once the size cap and sha256 both check out. +// On error the temp file is removed; on success the caller owns it (rename it +// into place or extract from it, then remove it). +// +// Every asset is fetched through the object-store plugin's AssetProvider +// first, with the plugin's credentials; atelet holds none of its own and does +// not know which bucket holds snapshots. A gs:// asset is then read +// anonymously, which serves public buckets such as gs://gvisor, but only when +// no plugin is configured or the plugin reports it has no such object or +// cannot serve it, as a plugin on a backend other than GCS does for every +// gs:// URI (see anonymousFallbackCodes). A refusal on policy or content, and a canceled +// request, are final. atelet verifies the bytes it caches whichever path +// fetched them, since it executes them. func (s *AteomHerder) downloadVerified(ctx context.Context, entry assetEntry, tmpPrefix string) (string, error) { - // Assets live in one of two places: public buckets (gVisor's releases in - // gs://gvisor — read anonymously) or the cluster's own object store (micro-VM - // kata/CH assets staged into the snapshot bucket — read with the main client, - // which is rustfs/S3 in kind and authenticated GCS on GKE). Auth is an - // atelet-level decision, not per-asset: try the anonymous client first so the - // common public-gVisor path stays fast, then fall back to the main client. The - // asset is streamed (not buffered) to disk below. - slog.DebugContext(ctx, "Streaming download from storage", slog.String("url", entry.URL)) - rc, err := s.openAsset(ctx, entry.URL) - if err != nil { - return "", fmt.Errorf("while fetching %v: %w", entry.URL, err) - } - defer rc.Close() - wantSum, err := hex.DecodeString(entry.SHA256) if err != nil { return "", fmt.Errorf("while parsing sha256 hash: %w", err) } + u, err := url.Parse(entry.URL) + if err != nil { + return "", apierror.InvalidArgument("sandbox asset URL is not a valid URL: %v", urlErrorCause(err)) + } + shown := redactURL(u) tmpFile, err := os.CreateTemp(nodepath.StaticFilesDir, tmpPrefix) if err != nil { return "", wrapFileSystemErr("while creating temp file", err) } tmpName := tmpFile.Name() - defer tmpFile.Close() ok := false defer func() { if !ok { os.Remove(tmpName) } }() + // The plugin writes the file by path. + if err := tmpFile.Close(); err != nil { + return "", wrapFileSystemErr("while closing temp file", err) + } + + // atelet's only client of its own is an anonymous GCS client, for public + // buckets such as gs://gvisor, so only a gs:// asset has a fallback. + anonFallback := u.Scheme == "gs" + var pluginCode codes.Code + if s.assetPlugin != nil { + pluginCode = s.fetchFromPlugin(ctx, entry, shown, tmpName) + if pluginCode == codes.OK { + if err := verifyAssetFile(tmpName, shown, wantSum); err != nil { + return "", err + } + ok = true + return tmpName, nil + } + if !anonFallback || !slices.Contains(anonymousFallbackCodes, pluginCode) { + slog.WarnContext(ctx, "Object-store plugin failed to fetch sandbox asset", + slog.String("url", shown), slog.String("code", pluginCode.String())) + return "", s.pluginError(ctx, entry, shown, pluginCode, nil) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return "", fmt.Errorf("sandbox asset %s: %w", shown, ctxErr) + } + slog.InfoContext(ctx, "Object-store plugin did not serve sandbox asset; downloading it anonymously", + slog.String("url", shown), slog.String("code", pluginCode.String())) + } else if !anonFallback { + return "", apierror.FailedPrecondition("sandbox asset %s: no object-store plugin is configured to fetch it", shown) + } + + if err := s.downloadAnonymous(ctx, entry, shown, tmpName, wantSum); err != nil { + if s.assetPlugin == nil { + return "", err + } + return "", s.pluginError(ctx, entry, shown, pluginCode, err) + } + ok = true + return tmpName, nil +} + +// anonymousFallbackCodes are the plugin codes after which atelet reads a gs:// +// asset anonymously: the plugin has no such object, cannot serve the URL, does +// not implement AssetProvider, or cannot be reached. PermissionDenied (the URL +// is under a snapshot location) and FailedPrecondition (wrong size or sha256) +// are refusals another source must not override. +var anonymousFallbackCodes = []codes.Code{codes.NotFound, codes.InvalidArgument, codes.Unimplemented, codes.Unavailable} + +// downloadAnonymous streams the gs:// asset entry.URL into the existing file +// at path with no credentials, replacing anything a failed plugin fetch left +// there, hashing as it goes, and checks the size cap and sha256. +func (s *AteomHerder) downloadAnonymous(ctx context.Context, entry assetEntry, shown, path string, wantSum []byte) error { + slog.DebugContext(ctx, "Streaming anonymous download from storage", slog.String("url", shown)) + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return wrapFileSystemErr("while opening temp file", err) + } + defer out.Close() + rc, err := objectstorage.Open(ctx, s.anonGCSClient, entry.URL) + if err != nil { + return fmt.Errorf("while fetching %v: %w", shown, err) + } + defer rc.Close() + // +1 lets an over-cap asset trip n > cap. Verify-after-copy keeps a bad + // download at the temp path, never the cache. + hasher := sha256.New() + n, err := io.Copy(io.MultiWriter(out, hasher), io.LimitReader(rc, maxAssetBytes+1)) + if err != nil { + return wrapFileSystemErr(fmt.Sprintf("while downloading %v", shown), err) + } + if n > maxAssetBytes { + return fmt.Errorf("asset %v exceeds %d-byte cap", shown, maxAssetBytes) + } + if got := hasher.Sum(nil); !bytes.Equal(got, wantSum) { + return fmt.Errorf("asset %v sha256 mismatch; got=%x want=%x", shown, got, wantSum) + } + if err := out.Close(); err != nil { // flush before the caller reads/renames + return wrapFileSystemErr("while closing temp file", err) + } + return nil +} - // Stream to disk, hashing as we go; +1 lets an over-cap asset trip n > cap. - // Verify-after-copy keeps a bad download at the temp path, never the cache. +// fetchFromPlugin asks the object-store plugin to write entry into the +// existing file at path, and returns the plugin's gRPC code: codes.OK once the +// plugin reports the file written. pluginError turns any other code into +// atelet's error. +// +// The plugin's status message is dropped here and never logged or returned: +// atelet's errors become actor crash messages, and a plugin's text may quote a +// signed URL or other credential in a form no scrubber can recognize. +func (s *AteomHerder) fetchFromPlugin(ctx context.Context, entry assetEntry, shown, path string) codes.Code { + slog.DebugContext(ctx, "Fetching sandbox asset through the object-store plugin", slog.String("url", shown)) + _, err := s.assetPlugin.FetchAsset(ctx, &objectstorev1.FetchAssetRequest{ + AssetUri: entry.URL, + Sha256: entry.SHA256, + WritePath: path, + MaxBytes: maxAssetBytes, + }) + return status.Code(err) +} + +// pluginError is atelet's own description of the plugin failing with code. +// anonErr, if set, is why the anonymous read that followed failed; it is +// atelet's own text and is kept in the message the caller receives. +func (s *AteomHerder) pluginError(ctx context.Context, entry assetEntry, shown string, code codes.Code, anonErr error) error { + newErr := apierror.FailedPrecondition + var msg string + switch code { + case codes.Unimplemented: + msg = fmt.Sprintf("the object-store plugin at %s does not serve %s; private sandbox assets need a plugin that implements it", + s.pluginSocket, objectstorev1.AssetProvider_ServiceDesc.ServiceName) + case codes.Unavailable: + newErr, msg = apierror.Unavailable, fmt.Sprintf("the object-store plugin at %s is unavailable", s.pluginSocket) + case codes.Canceled, codes.DeadlineExceeded: + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("sandbox asset %s: %w", shown, ctxErr) + } + newErr, msg = apierror.Unavailable, fmt.Sprintf("the object-store plugin returned %s", code) + case codes.PermissionDenied: + msg = fmt.Sprintf("the object-store plugin refused it (%s); assets must not live under a snapshot location", code) + case codes.NotFound: + msg = fmt.Sprintf("the object-store plugin did not find it (%s)", code) + case codes.FailedPrecondition: + msg = fmt.Sprintf("the object-store plugin rejected its content (%s): larger than %d bytes or sha256 is not %s", code, maxAssetBytes, entry.SHA256) + case codes.InvalidArgument: + msg = fmt.Sprintf("the object-store plugin rejected the request (%s)", code) + default: + newErr, msg = apierror.Internal, fmt.Sprintf("the object-store plugin failed (%s)", code) + } + if anonErr != nil { + msg += fmt.Sprintf("; anonymous download failed: %v", anonErr) + } + return newErr("sandbox asset %s: %s", shown, msg) +} + +// verifyAssetFile checks the size cap and sha256 of the file at path. +// The plugin is out of process and may be third-party, so atelet does not +// rely on pkg/assetfetch and verifies the bytes it executes. +func verifyAssetFile(path, shown string, wantSum []byte) error { + f, err := os.Open(path) + if err != nil { + return wrapFileSystemErr("while opening fetched asset", err) + } + defer f.Close() hasher := sha256.New() - n, err := io.Copy(io.MultiWriter(tmpFile, hasher), io.LimitReader(rc, maxAssetBytes+1)) + n, err := io.Copy(hasher, io.LimitReader(f, maxAssetBytes+1)) if err != nil { - return "", wrapFileSystemErr(fmt.Sprintf("while downloading %v", entry.URL), err) + return wrapFileSystemErr("while hashing fetched asset", err) } if n > maxAssetBytes { - return "", fmt.Errorf("asset %v exceeds %d-byte cap", entry.URL, maxAssetBytes) + return fmt.Errorf("asset %v exceeds %d-byte cap", shown, maxAssetBytes) } if got := hasher.Sum(nil); !bytes.Equal(got, wantSum) { - return "", fmt.Errorf("sha256 mismatch; got=%x want=%s", got, entry.SHA256) + return fmt.Errorf("asset %v sha256 mismatch; got=%x want=%x", shown, got, wantSum) } + return nil +} - if err := tmpFile.Close(); err != nil { // flush before the caller reads/renames - return "", wrapFileSystemErr("while closing temp file", err) +// redactURL returns scheme://host/path of u, for logs and errors: user info +// or a query may carry credentials, and atelet errors become actor crash +// messages. +func redactURL(u *url.URL) string { + return (&url.URL{Scheme: u.Scheme, Host: u.Host, Path: u.Path}).String() +} + +// redactURLString is redactURL for a URL that has not been parsed yet. +func redactURLString(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" } + return redactURL(u) +} - ok = true - return tmpName, nil +// urlErrorCause strips the *url.Error wrapper, whose message repeats the full +// URL, query included. +func urlErrorCause(err error) error { + var ue *url.Error + if errors.As(err, &ue) { + return ue.Err + } + return err } // extractTarArchive decompresses and extracts the tarball file at tarPath into @@ -445,26 +620,6 @@ func writeTarFile(dest string, r io.Reader, mode fs.FileMode) error { return nil } -// openAsset streams url, trying the anonymous client first (public buckets like -// gs://gvisor) then the main object-storage client (the cluster's own bucket, e.g. -// micro-VM assets in rustfs/S3 or an authenticated GCS bucket). The caller closes -// the returned reader. Streaming (rather than buffering the whole asset) keeps a -// multi-hundred-MiB guest image off the heap. -func (s *AteomHerder) openAsset(ctx context.Context, url string) (io.ReadCloser, error) { - rc, anonErr := objectstorage.Open(ctx, s.anonGCSClient, url) - if anonErr == nil { - return rc, nil - } - if s.gcsClient == nil { - return nil, anonErr - } - rc, mainErr := objectstorage.Open(ctx, s.gcsClient, url) - if mainErr != nil { - return nil, fmt.Errorf("anonymous open failed (%v); main client open failed: %w", anonErr, mainErr) - } - return rc, nil -} - // writeSandboxRecord persists the actor's running sandbox assets on-node so a // later Checkpoint (whose request no longer carries the sandbox config) can // re-fetch the same binaries and pin them into the snapshot manifest. diff --git a/cmd/atelet/sandbox_assets_plugin_test.go b/cmd/atelet/sandbox_assets_plugin_test.go new file mode 100644 index 0000000000..d26993e3ab --- /dev/null +++ b/cmd/atelet/sandbox_assets_plugin_test.go @@ -0,0 +1,666 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/atelet/internal/ateletpath" + "github.com/agent-substrate/substrate/internal/apierror" + "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/nodepath" + "github.com/agent-substrate/substrate/internal/objectstoreplugin" + "github.com/agent-substrate/substrate/internal/objectstoreplugin/objectstoreplugintest" + "github.com/agent-substrate/substrate/internal/pluginsocket" + "github.com/agent-substrate/substrate/internal/serverboot" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + testAssetBucket = "cluster-bucket" + testPluginSock = "/run/snapshot-plugin/node.sock" +) + +// bucketStorage is an in-memory object store keyed by "bucket/object" that +// counts reads. +type bucketStorage struct { + mu sync.Mutex + objects map[string][]byte + gets atomic.Int64 +} + +func (b *bucketStorage) GetObject(_ context.Context, bucket, object string) (io.ReadCloser, error) { + b.gets.Add(1) + b.mu.Lock() + defer b.mu.Unlock() + data, ok := b.objects[bucket+"/"+object] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", objectstorage.ErrObjectNotFound, bucket, object) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (b *bucketStorage) PutObject(_ context.Context, bucket, object string, r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + b.mu.Lock() + defer b.mu.Unlock() + b.objects[bucket+"/"+object] = data + return nil +} + +// countingAssetClient counts FetchAsset calls before passing them on. +type countingAssetClient struct { + objectstorev1.AssetProviderClient + calls atomic.Int64 +} + +func (c *countingAssetClient) FetchAsset(ctx context.Context, in *objectstorev1.FetchAssetRequest, opts ...grpc.CallOption) (*objectstorev1.FetchAssetResponse, error) { + c.calls.Add(1) + return c.AssetProviderClient.FetchAsset(ctx, in, opts...) +} + +// assetHerderFixture is an AteomHerder whose assets come from the in-tree +// plugin on an in-memory bucket, with an anonymous client that cannot read +// that bucket. +type assetHerderFixture struct { + herder *AteomHerder + bucket *bucketStorage + anon *bucketStorage + plugin *countingAssetClient + staging string +} + +// newAssetHerderFixture's plugin serves assets of scheme ("gs" or "s3"), as +// the in-tree plugin on that backend does. +func newAssetHerderFixture(t *testing.T, scheme string) *assetHerderFixture { + t.Helper() + origDir, origCap := nodepath.StaticFilesDir, maxAssetBytes + t.Cleanup(func() { nodepath.StaticFilesDir, maxAssetBytes = origDir, origCap }) + root := t.TempDir() + nodepath.StaticFilesDir = filepath.Join(root, "static-files") + if err := os.MkdirAll(nodepath.StaticFilesDir, 0o700); err != nil { + t.Fatal(err) + } + f := &assetHerderFixture{ + bucket: &bucketStorage{objects: map[string][]byte{}}, + anon: &bucketStorage{objects: map[string][]byte{}}, + staging: t.TempDir(), + } + plugin, err := objectstoreplugin.NewAssetPlugin(f.bucket, scheme, root, f.staging, maxAssetBytes) + if err != nil { + t.Fatal(err) + } + f.plugin = &countingAssetClient{AssetProviderClient: objectstoreplugintest.AssetClient(plugin)} + f.herder = &AteomHerder{anonGCSClient: f.anon, assetPlugin: f.plugin, pluginSocket: testPluginSock} + return f +} + +// put stores content in the cluster bucket and returns its sha256. +func (f *assetHerderFixture) put(t *testing.T, object string, content []byte) string { + t.Helper() + if err := f.bucket.PutObject(t.Context(), testAssetBucket, object, bytes.NewReader(content)); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%x", sha256.Sum256(content)) +} + +// assertNotCached fails if fetching left anything in the static-files cache: +// neither the cache file nor a temp file. +func assertNotCached(t *testing.T) { + t.Helper() + entries, err := os.ReadDir(nodepath.StaticFilesDir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + t.Errorf("failed fetch left %s in the static-files cache", e.Name()) + } +} + +// TestFetchAssetThroughPlugin covers assets atelet cannot read anonymously: +// they come from the object-store plugin, and atelet still verifies what it +// caches. +func TestFetchAssetThroughPlugin(t *testing.T) { + content := []byte("micro-vm kernel bytes") + + t.Run("non-gs scheme goes straight to the plugin", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + sum := f.put(t, "kata-assets/vmlinux", content) + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + if err != nil { + t.Fatalf("fetchAsset: %v", err) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, content) { + t.Errorf("cached bytes = %q, want %q", got, content) + } + if n := f.anon.gets.Load(); n != 0 { + t.Errorf("an s3:// asset was tried anonymously %d times", n) + } + }) + + t.Run("gs scheme served by the plugin is not read anonymously", func(t *testing.T) { + f := newAssetHerderFixture(t, "gs") + sum := f.put(t, "kata-assets/vmlinux", content) + f.anon.objects[testAssetBucket+"/kata-assets/vmlinux"] = content + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + if err != nil { + t.Fatalf("fetchAsset: %v", err) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, content) { + t.Errorf("cached bytes = %q, want %q", got, content) + } + if f.anon.gets.Load() != 0 || f.plugin.calls.Load() != 1 { + t.Errorf("anonymous reads = %d, plugin calls = %d; want 0 and 1", f.anon.gets.Load(), f.plugin.calls.Load()) + } + }) + + t.Run("gs asset on an S3 plugin is read anonymously without touching S3", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + // An S3 bucket of the same name, which on AWS could be anyone's. + if err := f.bucket.PutObject(t.Context(), "gvisor", "release.tar", strings.NewReader("someone else's bytes")); err != nil { + t.Fatal(err) + } + f.anon.objects["gvisor/release.tar"] = content + sum := fmt.Sprintf("%x", sha256.Sum256(content)) + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://gvisor/release.tar", SHA256: sum}) + if err != nil { + t.Fatalf("fetchAsset: %v", err) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, content) { + t.Errorf("cached bytes = %q, want %q", got, content) + } + if f.bucket.gets.Load() != 0 || f.anon.gets.Load() != 1 { + t.Errorf("S3 reads = %d, anonymous reads = %d; want 0 and 1", f.bucket.gets.Load(), f.anon.gets.Load()) + } + }) + + t.Run("public gs asset the plugin does not have is read anonymously", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.anon.objects["gvisor/release.tar"] = content + sum := fmt.Sprintf("%x", sha256.Sum256(content)) + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://gvisor/release.tar", SHA256: sum}) + if err != nil { + t.Fatalf("fetchAsset: %v", err) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, content) { + t.Errorf("cached bytes = %q, want %q", got, content) + } + if f.anon.gets.Load() != 1 || f.plugin.calls.Load() != 1 { + t.Errorf("anonymous reads = %d, plugin calls = %d; want 1 each", f.anon.gets.Load(), f.plugin.calls.Load()) + } + }) + + t.Run("cache hit", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + sum := f.put(t, "kata-assets/vmlinux", content) + cached := ateletpath.RunSCBinaryPath(sum) + if err := os.WriteFile(cached, content, 0o755); err != nil { + t.Fatal(err) + } + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + if err != nil || path != cached { + t.Fatalf("fetchAsset = %q, %v; want %q", path, err, cached) + } + if n := f.plugin.calls.Load(); n != 0 { + t.Errorf("plugin called %d times on a cache hit", n) + } + }) + + t.Run("sha256 mismatch", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.put(t, "kata-assets/vmlinux", content) + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) + if apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("fetchAsset with a wrong sha256 = %v (code %s), want %s", err, apierror.Code(err), codes.FailedPrecondition) + } + assertNotCached(t) + }) + + t.Run("size cap", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + sum := f.put(t, "kata-assets/vmlinux", content) + maxAssetBytes = int64(len(content)) - 1 + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}) + if apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("fetchAsset over the cap = %v (code %s), want %s", err, apierror.Code(err), codes.FailedPrecondition) + } + assertNotCached(t) + }) + + t.Run("not found", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/missing", SHA256: strings.Repeat("a", 64)}) + // Not NotFound: ate-api-server reads NotFound from Terminate as + // "already terminated". + if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "s3://"+testAssetBucket+"/kata-assets/missing") { + t.Fatalf("fetchAsset of a missing object = %v (code %s), want %s naming the URL", err, apierror.Code(err), codes.FailedPrecondition) + } + assertNotCached(t) + }) + + t.Run("snapshot in the same bucket is refused", func(t *testing.T) { + f := newAssetHerderFixture(t, "gs") + sum := f.put(t, "root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd", []byte("another actor's memory")) + f.anon.objects[testAssetBucket+"/root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd"] = []byte("another actor's memory") + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "gs://" + testAssetBucket + "/root/atespaces/team-a/actors/uid1/snapshots/snap1/checkpoint.img.zstd", SHA256: sum}) + if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "snapshot location") { + t.Fatalf("fetchAsset of a snapshot file = %v (code %s), want %s about the snapshot location", err, apierror.Code(err), codes.FailedPrecondition) + } + if n := f.bucket.gets.Load(); n != 0 { + t.Errorf("the plugin read %d objects for a refused request", n) + } + if n := f.anon.gets.Load(); n != 0 { + t.Errorf("a refused snapshot file was tried anonymously %d times", n) + } + assertNotCached(t) + }) + + t.Run("connection error is Unavailable", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + conn, err := pluginsocket.Dial(filepath.Join(t.TempDir(), "missing.sock"), 200*time.Millisecond) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + f.herder.assetPlugin = objectstorev1.NewAssetProviderClient(conn) + _, err = f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) + if apierror.Code(err) != codes.Unavailable { + t.Fatalf("fetchAsset with the plugin down = %v (code %s), want %s", err, apierror.Code(err), codes.Unavailable) + } + assertNotCached(t) + }) + + t.Run("plugin without AssetProvider", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + sockDir, err := os.MkdirTemp("", "atelet-assets") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + sock := filepath.Join(sockDir, "node.sock") + lis, err := pluginsocket.Listen(sock) + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer() + healthpb.RegisterHealthServer(srv, health.NewServer()) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + conn, err := pluginsocket.Dial(sock, pluginsocket.ReadyWait) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + f.herder.assetPlugin = objectstorev1.NewAssetProviderClient(conn) + f.herder.pluginSocket = sock + _, err = f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: strings.Repeat("a", 64)}) + if apierror.Code(err) != codes.FailedPrecondition { + t.Fatalf("fetchAsset with an old plugin = %v (code %s), want %s", err, apierror.Code(err), codes.FailedPrecondition) + } + for _, want := range []string{"s3://" + testAssetBucket + "/kata-assets/vmlinux", sock, "objectstore.v1.AssetProvider"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %q", err, want) + } + } + }) +} + +// fakeAssetClient writes data, if set, into write_path and returns err, +// standing in for a third-party plugin. +type fakeAssetClient struct { + data []byte + err error +} + +func (c fakeAssetClient) FetchAsset(_ context.Context, in *objectstorev1.FetchAssetRequest, _ ...grpc.CallOption) (*objectstorev1.FetchAssetResponse, error) { + if c.data != nil { + if err := os.WriteFile(in.GetWritePath(), c.data, 0o600); err != nil { + return nil, err + } + } + if c.err != nil { + return nil, c.err + } + return &objectstorev1.FetchAssetResponse{}, nil +} + +// TestFetchAssetAnonymousFallback covers when atelet reads an asset +// anonymously after the plugin fails: only for gs:// URLs, and only when the +// plugin cannot serve the asset rather than refuses it. +func TestFetchAssetAnonymousFallback(t *testing.T) { + content := []byte("gvisor release bytes") + sum := fmt.Sprintf("%x", sha256.Sum256(content)) + const ( + object = "gvisor/releases/runsc" + gsURL = "gs://" + object + pluginMsg = "plugin text SECRET-SIG" + ) + pluginErr := func(code codes.Code) error { return status.Error(code, pluginMsg) } + + // Pinned here rather than read from anonymousFallbackCodes, so dropping a + // code from production fails this test. + for _, code := range []codes.Code{codes.NotFound, codes.InvalidArgument, codes.Unimplemented, codes.Unavailable} { + t.Run(code.String()+" falls back", func(t *testing.T) { + logs := captureLogs(t) + f := newAssetHerderFixture(t, "s3") + f.anon.objects[object] = content + // A partial write the plugin leaves behind is replaced. + f.herder.assetPlugin = fakeAssetClient{data: []byte("partial plugin write, longer than the asset"), err: pluginErr(code)} + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + if err != nil { + t.Fatalf("fetchAsset: %v", err) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, content) { + t.Errorf("cached bytes = %q, want %q", got, content) + } + if n := f.anon.gets.Load(); n != 1 { + t.Errorf("anonymous reads = %d, want 1", n) + } + if !strings.Contains(logs.String(), `"code":"`+code.String()+`"`) || strings.Contains(logs.String(), "SECRET") { + t.Errorf("logs do not carry the plugin code %s alone:\n%s", code, logs) + } + }) + + t.Run(code.String()+" then anonymous failure reports both", func(t *testing.T) { + logs := captureLogs(t) + f := newAssetHerderFixture(t, "s3") + f.herder.assetPlugin = fakeAssetClient{err: pluginErr(code)} + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + if err == nil { + t.Fatal("fetchAsset succeeded") + } + if n := f.anon.gets.Load(); n != 1 { + t.Errorf("anonymous reads = %d, want 1", n) + } + // The status ate-api-server receives carries the apierror's own + // message only, so both descriptions must be in it. + st, _ := apierror.FromError(err) + wantCode := codes.FailedPrecondition + if code == codes.Unavailable { + wantCode = codes.Unavailable + } + if st.Code() != wantCode { + t.Errorf("code = %s, want %s (%v)", st.Code(), wantCode, err) + } + for _, want := range []string{gsURL, "object-store plugin", "anonymous download failed", "object not found"} { + if !strings.Contains(st.Message(), want) { + t.Errorf("status message %q does not carry %q", st.Message(), want) + } + } + for name, text := range map[string]string{"error": err.Error(), "logs": logs.String()} { + if strings.Contains(text, "SECRET") { + t.Errorf("%s carries the plugin's status text:\n%s", name, text) + } + } + assertNotCached(t) + }) + + for _, u := range []string{"s3://" + object, "https://storage.example/" + object} { + t.Run(code.String()+" "+u+" does not fall back", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.anon.objects[object] = content + f.herder.assetPlugin = fakeAssetClient{err: pluginErr(code)} + if _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: u, SHA256: sum}); err == nil { + t.Fatal("fetchAsset succeeded") + } + if n := f.anon.gets.Load(); n != 0 { + t.Errorf("anonymous reads = %d, want 0", n) + } + assertNotCached(t) + }) + } + } + + for _, code := range []codes.Code{codes.PermissionDenied, codes.FailedPrecondition, codes.Internal, codes.Unknown, codes.ResourceExhausted, codes.Canceled, codes.DeadlineExceeded} { + t.Run(code.String()+" does not fall back", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.anon.objects[object] = content + f.herder.assetPlugin = fakeAssetClient{err: pluginErr(code)} + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + if err == nil { + t.Fatal("fetchAsset succeeded") + } + if n := f.anon.gets.Load(); n != 0 { + t.Errorf("anonymous reads = %d, want 0", n) + } + if strings.Contains(err.Error(), "anonymous") || strings.Contains(err.Error(), "SECRET") { + t.Errorf("error %q mentions an anonymous read or the plugin's text", err) + } + assertNotCached(t) + }) + } + + t.Run("canceled request does not fall back", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.anon.objects[object] = content + ctx, cancel := context.WithCancel(t.Context()) + f.herder.assetPlugin = cancelingAssetClient{cancel: cancel, err: pluginErr(codes.Unavailable)} + _, err := f.herder.fetchAsset(ctx, assetEntry{URL: gsURL, SHA256: sum}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("fetchAsset = %v, want context.Canceled", err) + } + if n := f.anon.gets.Load(); n != 0 { + t.Errorf("anonymous reads = %d, want 0", n) + } + assertNotCached(t) + }) + + t.Run("no plugin reads gs anonymously", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.anon.objects[object] = content + f.herder.assetPlugin = nil + path, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + if err != nil { + t.Fatalf("fetchAsset: %v", err) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, content) { + t.Errorf("cached bytes = %q, want %q", got, content) + } + if n := f.anon.gets.Load(); n != 1 { + t.Errorf("anonymous reads = %d, want 1", n) + } + }) + + t.Run("no plugin, gs anonymous failure", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.herder.assetPlugin = nil + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: gsURL, SHA256: sum}) + if err == nil || !strings.Contains(err.Error(), "object not found") || strings.Contains(err.Error(), "object-store plugin") { + t.Fatalf("fetchAsset = %v, want the anonymous read's error alone", err) + } + assertNotCached(t) + }) + + t.Run("no plugin, non-gs is refused", func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + f.anon.objects[object] = content + f.herder.assetPlugin = nil + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + object, SHA256: sum}) + if apierror.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), "no object-store plugin") { + t.Fatalf("fetchAsset = %v (code %s), want %s naming the missing plugin", err, apierror.Code(err), codes.FailedPrecondition) + } + if n := f.anon.gets.Load(); n != 0 { + t.Errorf("anonymous reads = %d, want 0", n) + } + assertNotCached(t) + }) +} + +// cancelingAssetClient cancels the caller's request, then fails with err. +type cancelingAssetClient struct { + cancel context.CancelFunc + err error +} + +func (c cancelingAssetClient) FetchAsset(context.Context, *objectstorev1.FetchAssetRequest, ...grpc.CallOption) (*objectstorev1.FetchAssetResponse, error) { + c.cancel() + return nil, c.err +} + +// TestFetchAssetVerifiesPluginOutput checks that atelet does not trust a +// plugin's success: it re-checks the size cap and sha256 of what it caches. +func TestFetchAssetVerifiesPluginOutput(t *testing.T) { + content := []byte("micro-vm kernel bytes") + sum := fmt.Sprintf("%x", sha256.Sum256(content)) + for _, tc := range []struct { + name string + data []byte + cap int64 + }{ + {"wrong bytes", []byte("a snapshot's bytes"), 1 << 20}, + {"over the cap", content, int64(len(content)) - 1}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newAssetHerderFixture(t, "s3") + maxAssetBytes = tc.cap + f.herder.assetPlugin = fakeAssetClient{data: tc.data} + if _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "s3://" + testAssetBucket + "/kata-assets/vmlinux", SHA256: sum}); err == nil { + t.Fatal("fetchAsset accepted bytes the plugin wrote without checking them") + } + assertNotCached(t) + }) + } +} + +// captureLogs routes the default logger, with atelet's redaction handler, to +// the returned buffer until the test ends. +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + orig := slog.Default() + t.Cleanup(func() { slog.SetDefault(orig) }) + serverboot.InitLoggerWithWriter(&buf) + return &buf +} + +// TestFetchAssetKeepsCredentialsOutOfErrors checks that neither a signed +// asset URL nor a plugin's status text reaches atelet's logs or the error +// atelet returns to ate-api-server, which becomes the actor's crash message. +// The plugin texts include credentials that are not URLs, which no scrubbing +// of URLs would catch. +func TestFetchAssetKeepsCredentialsOutOfErrors(t *testing.T) { + const secret = "SECRET-SIG" + signed := "https://usr0:pw-" + secret + "@acct.blob.example/assets/vmlinux?sig=" + secret + sum := strings.Repeat("a", 64) + for _, tc := range []struct { + name string + plugin objectstorev1.AssetProviderClient + wantCode codes.Code + }{ + {"in-tree plugin rejects the URL", nil, codes.Internal}, + {"plugin quotes the URL", fakeAssetClient{err: status.Error(codes.PermissionDenied, "denied "+signed)}, codes.FailedPrecondition}, + {"plugin quotes the URL in the query only", fakeAssetClient{err: status.Error(codes.NotFound, "get https://acct.blob.example/assets/vmlinux?sig="+secret+": 404")}, codes.FailedPrecondition}, + {"plugin fails with credentials in its text", fakeAssetClient{err: status.Error(codes.Internal, "AccountKey="+secret)}, codes.Internal}, + {"plugin text is a bearer token", fakeAssetClient{err: status.Error(codes.FailedPrecondition, "Authorization: Bearer "+secret)}, codes.FailedPrecondition}, + {"plugin unavailable", fakeAssetClient{err: status.Error(codes.Unavailable, "dial "+signed)}, codes.Unavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + logs := captureLogs(t) + f := newAssetHerderFixture(t, "s3") + if tc.plugin != nil { + f.herder.assetPlugin = tc.plugin + } + // atelet's server interceptor logs the handler's error and decides + // what ate-api-server receives. + _, err := ateinterceptors.InternalServerUnaryInterceptor(t.Context(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { + return f.herder.fetchAsset(ctx, assetEntry{URL: signed, SHA256: sum}) + }) + if err == nil { + t.Fatal("fetchAsset succeeded") + } + if tc.plugin != nil { + if got := status.Code(err); got != tc.wantCode { + t.Errorf("code = %s (%v), want %s", got, err, tc.wantCode) + } + } + for name, text := range map[string]string{"error": err.Error(), "logs": logs.String()} { + for _, leak := range []string{secret, "usr0", "sig=", "Bearer", "AccountKey"} { + if strings.Contains(text, leak) { + t.Errorf("%s carries %q:\n%s", name, leak, text) + } + } + } + if !strings.Contains(err.Error(), "https://acct.blob.example/assets/vmlinux") { + t.Errorf("error %q does not name the asset as scheme://host/path", err) + } + }) + } +} + +// TestFetchAssetLogsPluginCodeAndRedactedURL checks what the log does carry +// when the plugin fails: its code and the asset as scheme://host/path. +func TestFetchAssetLogsPluginCodeAndRedactedURL(t *testing.T) { + logs := captureLogs(t) + f := newAssetHerderFixture(t, "s3") + f.herder.assetPlugin = fakeAssetClient{err: status.Error(codes.PermissionDenied, "denied SECRET")} + _, err := f.herder.fetchAsset(t.Context(), assetEntry{URL: "https://acct.blob.example/assets/vmlinux?sig=SECRET", SHA256: strings.Repeat("a", 64)}) + if err == nil { + t.Fatal("fetchAsset succeeded") + } + var line string + for l := range strings.Lines(logs.String()) { + if strings.Contains(l, "Object-store plugin failed to fetch sandbox asset") { + line = l + } + } + if line == "" { + t.Fatalf("no plugin failure log line:\n%s", logs) + } + for _, want := range []string{`"url":"https://acct.blob.example/assets/vmlinux"`, `"code":"PermissionDenied"`} { + if !strings.Contains(line, want) { + t.Errorf("log line missing %s:\n%s", want, line) + } + } + if strings.Contains(line, "SECRET") || strings.Contains(line, "denied") { + t.Errorf("log line carries the plugin's status text:\n%s", line) + } +} + +func TestRejectAteletStorageEnv(t *testing.T) { + t.Setenv("ATE_STORAGE_BACKEND", "") + os.Unsetenv("ATE_STORAGE_BACKEND") + if err := rejectStorageEnv(); err != nil { + t.Errorf("rejectStorageEnv() with the variable unset = %v", err) + } + for _, v := range []string{"s3", "gcs", ""} { + os.Setenv("ATE_STORAGE_BACKEND", v) + if err := rejectStorageEnv(); err == nil || !strings.Contains(err.Error(), "snapshot-plugin sidecar") { + t.Errorf("rejectStorageEnv() with %q = %v, want an error naming the sidecar", v, err) + } + } +} diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index 57a4924e3a..065b70a2ae 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -23,12 +23,31 @@ import ( "github.com/agent-substrate/substrate/internal/pluginsocket" objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // External snapshots move between the node and storage only through the // snapshot plugin. These helpers adapt the plugin's file-oriented API to the // manifest bytes atelet works with. +// pluginCallError replaces a snapshot plugin's error with one that keeps its +// gRPC code and drops its message. The message may quote a signed URL or other +// credential, and atelet's errors reach ate-api-server, its RPC log and the +// actor's crash message, so only the code and the redacted snapshot URI are +// reported. Callers read the code: NotFound means no such snapshot, and +// Unavailable is retried. +func pluginCallError(op, snapshotURI string, err error) error { + if err == nil { + return nil + } + code := status.Code(err) + if code == codes.Unknown { + code = status.FromContextError(err).Code() // a context error that is not a status + } + return status.Errorf(code, "snapshot plugin %s of %s failed (%s)", op, redactURLString(snapshotURI), code) +} + // fetchSnapshotFiles downloads the named snapshot files into dstDir. func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDir string, files []string) error { if len(files) == 0 { @@ -39,13 +58,14 @@ func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDi WritePath: dstDir, Files: files, }) - return pluginsocket.CallError(err) + return pluginsocket.CallError(pluginCallError("fetch", snapshotURI, err)) } // uploadSnapshotFiles uploads the named files in srcDir to the snapshot, -// passing opts to the plugin call. It returns the plugin's error as is: an -// upload that can be retried reports it through pluginsocket.CallError, -// one that cannot does not. +// passing opts to the plugin call. It returns the plugin's code without its +// message, and leaves the Unavailable mapping to the caller: an upload that can +// be retried reports it through pluginsocket.CallError, one that cannot +// does not. func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcDir string, files []string, opts ...grpc.CallOption) error { if len(files) == 0 { return nil @@ -55,7 +75,7 @@ func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcD LocalPath: srcDir, Files: files, }, opts...) - return err + return pluginCallError("upload", snapshotURI, err) } // fetchManifest returns a snapshot's manifest. The returned error is the diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 4dd11c5a17..7d5ab06dde 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -223,6 +223,87 @@ func TestSnapshotPluginErrorCodeThroughAtelet(t *testing.T) { } } +// A snapshot plugin's status text can quote a signed URL or other credential. +// Neither it nor the snapshot URI's query reaches atelet's RPC log or the +// error ate-api-server receives: only the plugin's code and the snapshot URI +// as scheme://host/path do, and callers still read the code. +func TestSnapshotPluginErrorKeepsCredentialsOut(t *testing.T) { + const ( + secret = "SECRET-SIG" + signedURI = "s3://bucket/snapshots/a?sig=" + secret + shownURI = "s3://bucket/snapshots/a" + ) + uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) + if err != nil { + t.Fatal(err) + } + rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + transfers := []struct { + name string + uri string // the URI the transfer names in its error, when it takes one + run func(ctx context.Context, s *AteomHerder) error + }{ + {"download", signedURI, func(ctx context.Context, s *AteomHerder) error { + return s.downloadExternalCheckpoint(ctx, signedURI, t.TempDir(), []string{"a"}) + }}, + {"upload", signedURI, func(ctx context.Context, s *AteomHerder) error { + return s.uploadSnapshotFiles(ctx, signedURI, t.TempDir(), []string{"a"}) + }}, + {"manifest fetch", signedURI, func(ctx context.Context, s *AteomHerder) error { + _, err := s.fetchManifest(ctx, signedURI) + return err + }}, + {"paused upload", "", func(ctx context.Context, s *AteomHerder) error { + dir := t.TempDir() + writeLocalSnapshot(t, dir, rec, map[string]string{"a": "a"}) + _, err := s.uploadLocalCheckpointDir(ctx, validUploadPausedCheckpointRequest(), dir, uri) + return err + }}, + } + for _, pluginCode := range []codes.Code{codes.Unavailable, codes.Internal, codes.NotFound} { + for _, tr := range transfers { + t.Run(pluginCode.String()+"/"+tr.name, func(t *testing.T) { + logs := captureLogs(t) + s := &AteomHerder{ + snapshotPlugin: failingNodePlugin{err: status.Error(pluginCode, "get "+signedURI+": AccountKey="+secret)}, + snapshotScratchDir: t.TempDir(), + } + _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, + func(ctx context.Context, _ any) (any, error) { return nil, tr.run(ctx, s) }) + if err == nil { + t.Fatal("transfer succeeded") + } + for name, text := range map[string]string{"error": err.Error(), "logs": logs.String()} { + for _, leak := range []string{secret, "sig=", "AccountKey"} { + if strings.Contains(text, leak) { + t.Errorf("%s carries %q:\n%s", name, leak, text) + } + } + } + if tr.uri != "" && !strings.Contains(err.Error(), shownURI) { + t.Errorf("error %q does not name the snapshot as %s", err, shownURI) + } + }) + } + } +} + +// The plugin's code survives the message being dropped. +func TestPluginCallErrorKeepsCode(t *testing.T) { + for _, code := range []codes.Code{codes.Unavailable, codes.NotFound, codes.Internal, codes.Canceled} { + err := pluginCallError("fetch", "s3://b/o?sig=x", status.Error(code, "secret")) + if got := status.Code(err); got != code { + t.Errorf("code = %s, want %s", got, code) + } + } + if err := pluginCallError("fetch", "s3://b/o", nil); err != nil { + t.Errorf("pluginCallError(nil) = %v", err) + } + if got := status.Code(pluginCallError("fetch", "s3://b/o", context.DeadlineExceeded)); got != codes.DeadlineExceeded { + t.Errorf("code for a context error = %s, want %s", got, codes.DeadlineExceeded) + } +} + // pluginSidecar serves a NodePlugin on a Unix socket and can be stopped and // started again on the same path, as kubelet restarts a sidecar container. type pluginSidecar struct { diff --git a/cmd/snapshot-plugin/main.go b/cmd/snapshot-plugin/main.go index f2a2e7d1c1..ff2a67ea06 100644 --- a/cmd/snapshot-plugin/main.go +++ b/cmd/snapshot-plugin/main.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Command snapshot-plugin serves the snapshot plugin API on GCS or S3 over a -// Unix socket. It runs as a sidecar: "node" next to atelet, "control" next to -// ate-api-server. The backend is chosen by ATE_STORAGE_BACKEND ("s3", or GCS -// by default) with the ambient credentials, as atelet and ate-api-server do. +// Command snapshot-plugin serves the object-store plugin API on GCS or S3 over +// a Unix socket. It runs as a sidecar: "node" next to atelet, serving snapshot +// transfers and sandbox assets, and "control" next to ate-api-server. The +// backend is chosen by ATE_STORAGE_BACKEND ("s3", or GCS by default) with the +// ambient credentials. // "healthcheck" reports whether a running plugin is serving, for the sidecar's // startup probe. package main @@ -42,6 +43,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "k8s.io/apimachinery/pkg/api/resource" ) const usage = `usage: snapshot-plugin [flags]` @@ -54,7 +56,9 @@ func main() { mode := os.Args[1] flags := pflag.NewFlagSet(mode, pflag.ExitOnError) socket := flags.String("socket", "", "Unix socket to serve on, or to check in healthcheck mode") - root := flags.String("root", nodepath.BasePath, "node mode: the only directory tree local snapshot files may be read from or written to") + root := flags.String("root", nodepath.BasePath, "node mode: the only directory tree local snapshot and asset files may be read from or written to") + assetStagingDir := flags.String("asset-staging-dir", "/var/lib/snapshot-plugin/asset-staging", "node mode: directory sandbox assets are downloaded into and verified before they are copied below --root; must not be below --root or readable by the caller") + assetStagingCapacity := flags.String("asset-staging-capacity", "8Gi", "node mode: the most sandbox asset downloads in --asset-staging-dir may hold at once, as a Kubernetes quantity; downloads wait for space, and an asset larger than this is refused") timeout := flags.Duration("timeout", 5*time.Second, "healthcheck mode: how long to wait for the plugin to report that it is serving") _ = flags.Parse(os.Args[2:]) @@ -94,6 +98,18 @@ func main() { serverboot.Fatal(ctx, "Invalid --root", err) } objectstorev1.RegisterNodeProviderServer(srv, plugin) + capacity, err := parseCapacity(*assetStagingCapacity) + if err != nil { + serverboot.Fatal(ctx, "Invalid --asset-staging-capacity", err) + } + if err := os.MkdirAll(*assetStagingDir, 0o700); err != nil { + serverboot.Fatal(ctx, "Failed to create --asset-staging-dir", err) + } + assets, err := objectstoreplugin.NewAssetPlugin(objects, assetScheme(), *root, *assetStagingDir, capacity) + if err != nil { + serverboot.Fatal(ctx, "Invalid --asset-staging-dir", err) + } + objectstorev1.RegisterAssetProviderServer(srv, assets) case "control": store, err := newObjectStore(ctx) if err != nil { @@ -147,6 +163,15 @@ func newObjectStore(ctx context.Context) (objectstore.Store, error) { return objectstore.NewGCS(client), nil } +// assetScheme is the asset URI scheme the node plugin serves: its backend's, +// as objectstorage.NewFromEnv selects it. +func assetScheme() string { + if objectstorage.UsesS3() { + return "s3" + } + return "gs" +} + // healthcheck dials the plugin at socket and waits up to timeout for it to // report that it is serving. func healthcheck(ctx context.Context, socket string, timeout time.Duration) error { @@ -159,3 +184,17 @@ func healthcheck(ctx context.Context, socket string, timeout time.Duration) erro defer cancel() return pluginsocket.WaitReady(ctx, conn) } + +// parseCapacity parses a positive byte count written as a Kubernetes quantity, +// such as 8Gi. +func parseCapacity(s string) (int64, error) { + q, err := resource.ParseQuantity(s) + if err != nil { + return 0, err + } + n, ok := q.AsInt64() + if !ok || n <= 0 { + return 0, fmt.Errorf("%q is not a positive whole number of bytes", s) + } + return n, nil +} diff --git a/cmd/snapshot-plugin/main_test.go b/cmd/snapshot-plugin/main_test.go new file mode 100644 index 0000000000..2f35272cfc --- /dev/null +++ b/cmd/snapshot-plugin/main_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/yaml" +) + +func TestParseCapacity(t *testing.T) { + for _, tc := range []struct { + in string + want int64 + }{ + {"8Gi", 8 << 30}, + {"512Mi", 512 << 20}, + {"1000", 1000}, + } { + got, err := parseCapacity(tc.in) + if err != nil || got != tc.want { + t.Errorf("parseCapacity(%q) = %d, %v; want %d", tc.in, got, err, tc.want) + } + } + for _, in := range []string{"", "0", "-1Gi", "0.5", "1m", "lots"} { + if got, err := parseCapacity(in); err == nil { + t.Errorf("parseCapacity(%q) = %d, want an error", in, got) + } + } +} + +// TestInstallStagingCapacityFitsVolume checks that the install's node plugin +// stages no more than its asset-staging volume holds, so concurrent asset +// downloads cannot get atelet's pod evicted. +func TestInstallStagingCapacityFitsVolume(t *testing.T) { + raw, err := os.ReadFile("../../manifests/ate-install/atelet.yaml") + if err != nil { + t.Fatal(err) + } + var ds *appsv1.DaemonSet + for _, doc := range strings.Split(string(raw), "\n---") { + var obj appsv1.DaemonSet + if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { + t.Fatal(err) + } + if obj.Kind == "DaemonSet" && strings.HasPrefix(obj.Name, "atelet-") { + ds = &obj + } + } + if ds == nil { + t.Fatal("atelet.yaml has no atelet DaemonSet") + } + var capacityArg string + for _, c := range ds.Spec.Template.Spec.InitContainers { + if c.Name != "snapshot-plugin" { + continue + } + for _, a := range c.Args { + if v, ok := strings.CutPrefix(a, "--asset-staging-capacity="); ok { + capacityArg = v + } + } + } + if capacityArg == "" { + t.Fatal("the snapshot-plugin sidecar does not set --asset-staging-capacity") + } + capacity, err := parseCapacity(capacityArg) + if err != nil { + t.Fatal(err) + } + for _, v := range ds.Spec.Template.Spec.Volumes { + if v.Name != "asset-staging" { + continue + } + if v.EmptyDir == nil || v.EmptyDir.SizeLimit == nil { + t.Fatal("asset-staging is not an emptyDir with a sizeLimit") + } + if limit := v.EmptyDir.SizeLimit.Value(); capacity >= limit { + t.Errorf("--asset-staging-capacity=%s (%d bytes) leaves no room below the asset-staging sizeLimit of %d bytes", capacityArg, capacity, limit) + } + return + } + t.Fatal("atelet DaemonSet has no asset-staging volume") +} diff --git a/docs/api-guide.md b/docs/api-guide.md index 754ace24da..d87e300aa6 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -453,7 +453,7 @@ Each entry of `versions` (`SandboxVersionConfig`): | `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | | `pauseImage` | `string` | **Required for `gvisor`; not allowed for `microvm`**, which runs no pause container. The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must include a digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | -| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. | +| `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, or is unreachable) is then read anonymously, which serves public buckets such as `gs://gvisor`; a refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (an `ActorTemplate`'s `snapshotConfig.location` followed by `/atespaces/`), so assets may share the snapshot bucket but must not be stored there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | A cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor templates can name it via `sandboxConfig.configName` without any extra setup. diff --git a/docs/architecture.md b/docs/architecture.md index 49886b63ec..3bc6e90c8a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -327,7 +327,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov * **Lifecycle Management**: The `ateom` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for micro-VMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.) - * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. `atelet` keeps a storage client of its own only for sandbox asset downloads. + * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own. ### Sandbox Classes diff --git a/docs/threat-model.md b/docs/threat-model.md index a674f2c8fd..0ccda36af9 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -74,7 +74,7 @@ While it's probably unlikely that Substrate, which is expected to be an internal | T-05 | High | Privilege escalation via access to sensitive labels. | If Substrate offers its own resource labeling mechanism, it must also offer a way to authorize label updates on a per-label basis. | Substrate authorization system requires explicit authorization to update metadata, separate from updating resource body. Substrate authorization system supports per-label authorization rules. | Plenty of attacks in K8s were possible because labels had semantic meaning, but the permission model could implicitly granted access to modify labels, even if it was inappropriate. For example, /status subresource allows label updates. Substrate shouldn't repeat this mistake. | | T-06 | High | Attacker gains control of the Substrate API server or an ingress/egress gateway. | Isolate the control plane from the data plane, and isolate data plane ingress from sandboxes. | Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. This also supports control-plane / data-plane isolation from a reliability and performance perspective. Consider running any gateway that enables direct interaction with sandboxes on a separate VM from the sandboxes. Consider using a zero-trust architecture where traffic is encrypted and authenticated end-to-end and routed via a mesh. | While the sandbox is trusted as an isolation layer, the residual risk is typically not worth the marginal cost savings of colocating, and isolation is recommend to avoid noisy neighbor problems or other reliability issues that stem from mixing the control-plane and data-plane, especially for large scale deployments of Substrate. | | T-07 | High | Attacker who can create ActorTemplates specifies malicious runtime. | Ensure available runtime can only be configured by administrators. | Consider a mechanism like RuntimeClass to decouple configuration of available runtimes from consumption of available runtimes. | | -| T-08 | High | Attacker who can create ActorTemplates can read or write any storage buckets atelet has access to. | Ensure that bucket access is least-privilege. | Use credentials derived from actor identity to read snapshots. Configure permissions to prevent atelet or nodes from having access to sensitive buckets. Don't support arbitrary URLs in APIs, instead design a standard approach for accessing the correct resource given the ActorTemplate name and compute the mapping internally, subject to scheduling-aware authorization checks. | For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket. | +| T-08 | High | Attacker who can create ActorTemplates can read or write any storage buckets atelet has access to. | Ensure that bucket access is least-privilege. | Use credentials derived from actor identity to read snapshots. Configure permissions to prevent atelet or nodes from having access to sensitive buckets. Don't support arbitrary URLs in APIs, instead design a standard approach for accessing the correct resource given the ActorTemplate name and compute the mapping internally, subject to scheduling-aware authorization checks. | For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket. Current state for sandbox asset URLs (SandboxConfig): atelet holds no storage credentials. It fetches every asset through the node's object-store plugin, which refuses any URL under a snapshot location before reading it, and returns an object only if its content matches the sha256 the SandboxConfig pins, staging the download where atelet cannot read it until it is verified. Only when the plugin cannot serve a `gs://` URL (no such object, a URL it rejects, no `AssetProvider`, or unreachable) does atelet read it anonymously, which returns only publicly readable objects; a refusal by the plugin is final. A SandboxConfig author therefore cannot use an asset URL to pull another actor's snapshot onto a node, even when assets share a bucket with snapshots, and a modified asset fails the digest check in the plugin and again in atelet. These checks do not protect against a compromised node: root on the node can use the plugin's identity directly, which already reads and writes snapshots. Closing that needs storage-level scoping such as per-actor snapshot credentials: [https://github.com/agent-substrate/substrate/issues/777](https://github.com/agent-substrate/substrate/issues/777) | | T-09 | Medium | DoS attack against API, gateway, or available cluster resources. | Minimize exposure and ensure APIs and proxies implement appropriately scoped quotas and rate limiting. | Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. Don't expose ate-apiserver directly to the internet. Consider an identity-aware WAF (likely up to providers) if external access is required. Use network policy to block direct interaction between untrusted sandboxes and the ate-apiserver. Implement quotas and rate limiting in the API and in proxies. This includes quotas for the number of actors a user can allocate. Use a zero-trust architecture that prevents identity spoofing or intentional misrouting to get around limits. | The gateway calls ate-apiserver to resolve Actor assignments and uses a short-lived cache. A flood of Actor traffic could still create high API load; cache invalidation during Actor rescheduling must avoid stale routing. | | T-10 | Medium | Internal network traffic is intercepted or spoofed | Encrypt all traffic by default | Use mTLS between all system components, including the gateway-to-worker `atunnel` connection. | It may be desirable to rely on cloud providers to transparently encrypt traffic between VMs on their internal network. Worth discussing what makes the most sense. | diff --git a/docs/threats.json b/docs/threats.json index 13a7d3060d..cd1820dbda 100644 --- a/docs/threats.json +++ b/docs/threats.json @@ -61,7 +61,7 @@ "threat": "Attacker who can create ActorTemplates can read or write any storage buckets atelet has access to.", "mitigating_invariants": "Ensure that bucket access is least-privilege.", "suggested_concrete_mitigations": "Use credentials derived from actor identity to read snapshots. Configure permissions to prevent atelet or nodes from having access to sensitive buckets. Don't support arbitrary URLs in APIs, instead design a standard approach for accessing the correct resource given the ActorTemplate name and compute the mapping internally, subject to scheduling-aware authorization checks.", - "notes": "For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket." + "notes": "For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket. Current state for sandbox asset URLs (SandboxConfig): atelet holds no storage credentials. It fetches every asset through the node's object-store plugin, which refuses any URL under a snapshot location before reading it, and returns an object only if its content matches the sha256 the SandboxConfig pins, staging the download where atelet cannot read it until it is verified. Only when the plugin cannot serve a `gs://` URL (no such object, a URL it rejects, no `AssetProvider`, or unreachable) does atelet read it anonymously, which returns only publicly readable objects; a refusal by the plugin is final. A SandboxConfig author therefore cannot use an asset URL to pull another actor's snapshot onto a node, even when assets share a bucket with snapshots, and a modified asset fails the digest check in the plugin and again in atelet. These checks do not protect against a compromised node: root on the node can use the plugin's identity directly, which already reads and writes snapshots. Closing that needs storage-level scoping such as per-actor snapshot credentials: [https://github.com/agent-substrate/substrate/issues/777](https://github.com/agent-substrate/substrate/issues/777)" }, { "threat_id": "T-09", diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index c9786a6d11..cc4516952e 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -120,6 +120,14 @@ run_kubectl() { MANIFEST_TEMPLATE="manifests/microvm/sandboxconfig-microvm.yaml.tmpl" +# The asset URLs' scheme: the node's object-store plugin serves only its own +# backend's, which is S3 (rustfs) on kind and GCS on GKE. +if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + BUCKET_SCHEME="s3" +else + BUCKET_SCHEME="gs" +fi + if [[ "${action}" == "delete" ]]; then # A WorkerPool that uses the config holds it. The delete would only mark # the config Terminating, and `kubectl delete` would wait for it without @@ -149,7 +157,7 @@ if [[ "${action}" == "delete" ]]; then echo " again." >&2 exit 1 fi - log "Done. (Bucket assets at gs://${BUCKET_NAME}/kata-assets/ left in place.)" + log "Done. (Bucket assets at ${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/ left in place.)" exit 0 fi @@ -208,9 +216,11 @@ fi # --- 3. apply the cluster-wide microvm SandboxConfig ----------------------- # Every asset is downloaded rather than built, so all four carry committed, -# reproducible per-arch shas and the bucket name is the only substitution left. +# reproducible per-arch shas and the bucket's scheme and name are the only +# substitutions left. log "Applying microvm SandboxConfig from ${MANIFEST_TEMPLATE}..." -sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ +sed -e "s|\${BUCKET_SCHEME}|${BUCKET_SCHEME}|g" \ + -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ "${MANIFEST_TEMPLATE}" \ | run_kubectl apply -f - diff --git a/hack/microvm-assets/README.md b/hack/microvm-assets/README.md index 6382505969..d4b9f26af8 100644 --- a/hack/microvm-assets/README.md +++ b/hack/microvm-assets/README.md @@ -49,4 +49,8 @@ it as a device, which is what places micro-VM workers there. guest-memory snapshot round-tripped across pods. ## Notes +- Assets may share the snapshot bucket, but must not be stored under a snapshot location + (an ActorTemplate's `snapshotConfig.location` followed by `/atespaces/`): the node's + object-store plugin refuses to serve anything there as an asset. The staging scripts use + `kata-assets/`, which is outside it. - `assets` is single-arch (unlike runsc's amd64/arm64): stage assets matching the node arch. diff --git a/hack/microvm-assets/stage-to-gcs.sh b/hack/microvm-assets/stage-to-gcs.sh index 9595022470..ba1afad6bd 100755 --- a/hack/microvm-assets/stage-to-gcs.sh +++ b/hack/microvm-assets/stage-to-gcs.sh @@ -15,8 +15,11 @@ # limitations under the License. # Stage the assembled micro-VM asset set into the GCS snapshot bucket under -# kata-assets/, where atelet fetches it (per demos/counter/counter-microvm.yaml.tmpl). -# The GKE counterpart of stage-to-rustfs.sh. Run after assemble.sh has produced $OUT. +# kata-assets/, where atelet fetches it through the node's object-store plugin +# (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin refuses +# assets stored under a snapshot location (/atespaces/), +# so keep them outside it. The GKE counterpart of stage-to-rustfs.sh. Run after +# assemble.sh has produced $OUT. # # Requires the `gcloud` CLI authenticated for the bucket's project. Env: OUT (asset # dir, default ./bin/microvm-assets/amd64), BUCKET (default ate-snapshots), diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh index 19e3d4348b..2f8c66f7b2 100755 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ b/hack/microvm-assets/stage-to-rustfs.sh @@ -15,7 +15,10 @@ # limitations under the License. # Stage the assembled micro-VM asset set into the kind cluster's rustfs S3 bucket -# under kata-assets/, where atelet fetches it (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). +# under kata-assets/, where atelet fetches it through the node's object-store +# plugin (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). The plugin +# refuses assets stored under a snapshot location +# (/atespaces/), so keep them outside it. # Run after the cluster is up (hack/install-ate-kind.sh) and assemble.sh has produced $OUT. # # The S3 client runs in a throwaway container (the same pinned amazon/aws-cli diff --git a/internal/objectstoreplugin/asset.go b/internal/objectstoreplugin/asset.go new file mode 100644 index 0000000000..9be7eff19e --- /dev/null +++ b/internal/objectstoreplugin/asset.go @@ -0,0 +1,107 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/agent-substrate/substrate/pkg/assetfetch" + "github.com/agent-substrate/substrate/pkg/objectstorage" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// AssetPlugin serves AssetProvider on an object storage client. It reads +// URIs of its backend's scheme (gs:// on GCS, s3:// on S3) as +// /, and returns InvalidArgument for any other scheme without +// reading anything: a gs:// bucket name read on S3 names someone else's +// bucket, or none. atelet then reads a gs:// asset anonymously. +// +// It downloads each asset into stagingDir, which must be private to the +// plugin, and copies it to the caller's file only once its size and sha256 +// check out (see assetfetch.Stager). The caller can therefore never read bytes +// it did not pin, such as a snapshot's. +type AssetPlugin struct { + objectstorev1.UnimplementedAssetProviderServer + + client objectstorage.ObjectStorage + scheme string + stager *assetfetch.Stager +} + +// NewAssetPlugin returns an AssetPlugin that serves URIs of scheme ("gs" or +// "s3", client's backend) from client, writes caller files only below root and +// stages downloads in stagingDir, which must not be below root, holding at +// most stagingCapacity bytes there at once. +func NewAssetPlugin(client objectstorage.ObjectStorage, scheme, root, stagingDir string, stagingCapacity int64) (*AssetPlugin, error) { + if scheme != "gs" && scheme != "s3" { + return nil, fmt.Errorf("asset URI scheme %q is not gs or s3", scheme) + } + stager, err := assetfetch.NewStager(root, stagingDir, stagingCapacity) + if err != nil { + return nil, err + } + return &AssetPlugin{client: client, scheme: scheme, stager: stager}, nil +} + +// FetchAsset downloads the object at asset_uri into the staging directory and, +// if it is at most max_bytes long and hashes to sha256, copies it into the +// existing file at write_path (see assetfetch.Stager.Fetch). +func (p *AssetPlugin) FetchAsset(ctx context.Context, req *objectstorev1.FetchAssetRequest) (*objectstorev1.FetchAssetResponse, error) { + bucket, object, err := parseAssetURI(req.GetAssetUri(), p.scheme) + if err != nil { + return nil, err + } + open := func(ctx context.Context) (io.ReadCloser, int64, error) { + rc, err := p.client.GetObject(ctx, bucket, object) + if err != nil { + return nil, 0, err + } + size, ok := objectstorage.ObjectSize(rc) + if !ok { + size = -1 + } + return rc, size, nil + } + if err := p.stager.Fetch(ctx, req, req.GetAssetUri(), open, toStatus); err != nil { + return nil, err + } + return &objectstorev1.FetchAssetResponse{}, nil +} + +// parseAssetURI accepts :/// whose object name is in +// canonical form and is not stored under a snapshot location, and returns the +// bucket and object name. +func parseAssetURI(uri, scheme string) (bucket, object string, err error) { + u, err := assetfetch.ParseURI(uri) + if err != nil { + return "", "", err + } + if u.Scheme != scheme { + return "", "", status.Errorf(codes.InvalidArgument, "asset URI %s: scheme %q is not served by this plugin, whose storage backend serves %s://", uri, u.Scheme, scheme) + } + object = strings.TrimPrefix(u.Path, "/") + if u.Host == "" || object == "" { + return "", "", status.Errorf(codes.InvalidArgument, "asset URI %s must name a bucket and an object", uri) + } + if err := assetfetch.CheckObjectName(uri, u.Scheme+"://"+u.Host, object); err != nil { + return "", "", err + } + return u.Host, object, nil +} diff --git a/internal/objectstoreplugin/asset_test.go b/internal/objectstoreplugin/asset_test.go new file mode 100644 index 0000000000..92778f8840 --- /dev/null +++ b/internal/objectstoreplugin/asset_test.go @@ -0,0 +1,495 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstoreplugin + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/agent-substrate/substrate/internal/pluginsocket" + "github.com/agent-substrate/substrate/pkg/assetfetch" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// countingObjects counts reads, so a test can tell that a request was refused +// before anything was read. With reportSize set, its readers report the +// object's size, as the GCS and S3 clients' do. +type countingObjects struct { + *memObjects + gets atomic.Int64 + reportSize bool +} + +func (c *countingObjects) GetObject(ctx context.Context, bucket, object string) (io.ReadCloser, error) { + c.gets.Add(1) + rc, err := c.memObjects.GetObject(ctx, bucket, object) + if err != nil || !c.reportSize { + return rc, err + } + data, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + return &sizedReader{ReadCloser: io.NopCloser(bytes.NewReader(data)), size: int64(len(data))}, nil +} + +type sizedReader struct { + io.ReadCloser + size int64 +} + +func (r *sizedReader) ObjectSize() int64 { return r.size } + +// assetFixture is an AssetProvider served on a real Unix socket, as the +// sidecar serves it, with a write root and a separate staging directory. +type assetFixture struct { + client objectstorev1.AssetProviderClient + backend *countingObjects + root string + staging string +} + +// newAssetFixture serves assets of scheme ("gs" or "s3"). +func newAssetFixture(t *testing.T, scheme string) *assetFixture { + t.Helper() + f := &assetFixture{ + backend: &countingObjects{memObjects: newMemObjects()}, + root: t.TempDir(), + staging: t.TempDir(), + } + plugin, err := NewAssetPlugin(f.backend, scheme, f.root, f.staging, 1<<30) + if err != nil { + t.Fatal(err) + } + sockDir, err := os.MkdirTemp("", "assetplug") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + sock := filepath.Join(sockDir, "plugin.sock") + lis, err := pluginsocket.Listen(sock) + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer() + objectstorev1.RegisterAssetProviderServer(srv, plugin) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + conn, err := pluginsocket.Dial(sock, pluginsocket.ReadyWait) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + f.client = objectstorev1.NewAssetProviderClient(conn) + return f +} + +// put stores content at bucket/object and returns its sha256. +func (f *assetFixture) put(t *testing.T, object string, content []byte) string { + t.Helper() + if err := f.backend.PutObject(t.Context(), testBucket, object, bytes.NewReader(content)); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%x", sha256.Sum256(content)) +} + +// writeFile creates the caller's file below root with content and returns +// its path. +func (f *assetFixture) writeFile(t *testing.T, content string) string { + t.Helper() + p := filepath.Join(f.root, "static-files", "asset-download-1") + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// assertContent fails unless path holds want. +func assertContent(t *testing.T, path, want string) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != want { + t.Errorf("%s = %q, want %q", path, got, want) + } +} + +// assertStagingEmpty fails if a download was left in the staging directory. +func (f *assetFixture) assertStagingEmpty(t *testing.T) { + t.Helper() + entries, err := os.ReadDir(f.staging) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("staging directory holds %d entries after the call, want none", len(entries)) + } +} + +const untouched = "the caller's original bytes" + +func TestFetchAsset(t *testing.T) { + content := []byte("micro-vm kernel bytes") + + t.Run("happy path", func(t *testing.T) { + for _, scheme := range []string{"gs", "s3"} { + f := newAssetFixture(t, scheme) + sum := f.put(t, "kata-assets/vmlinux", content) + dst := f.writeFile(t, untouched) + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: scheme + "://" + testBucket + "/kata-assets/vmlinux", + Sha256: sum, + WritePath: dst, + MaxBytes: int64(len(content)), + }) + if err != nil { + t.Fatalf("%s: FetchAsset: %v", scheme, err) + } + assertContent(t, dst, string(content)) + f.assertStagingEmpty(t) + } + }) + + t.Run("digest mismatch leaves write_path untouched", func(t *testing.T) { + f := newAssetFixture(t, "gs") + f.put(t, "kata-assets/vmlinux", content) + dst := f.writeFile(t, untouched) + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: "gs://" + testBucket + "/kata-assets/vmlinux", + Sha256: strings.Repeat("a", 64), + WritePath: dst, + MaxBytes: 1 << 20, + }) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("FetchAsset with a wrong sha256 = %v, want %s", err, codes.FailedPrecondition) + } + assertContent(t, dst, untouched) + f.assertStagingEmpty(t) + }) + + t.Run("max_bytes is enforced", func(t *testing.T) { + f := newAssetFixture(t, "gs") + sum := f.put(t, "kata-assets/vmlinux", content) + dst := f.writeFile(t, untouched) + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: "gs://" + testBucket + "/kata-assets/vmlinux", + Sha256: sum, + WritePath: dst, + MaxBytes: int64(len(content)) - 1, + }) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("FetchAsset of an object over max_bytes = %v, want %s", err, codes.FailedPrecondition) + } + assertContent(t, dst, untouched) + f.assertStagingEmpty(t) + }) + + t.Run("other scheme is refused before reading", func(t *testing.T) { + for _, tc := range []struct{ plugin, asset string }{{"s3", "gs"}, {"gs", "s3"}} { + f := newAssetFixture(t, tc.plugin) + sum := f.put(t, "kata-assets/vmlinux", content) + dst := f.writeFile(t, untouched) + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: tc.asset + "://" + testBucket + "/kata-assets/vmlinux", + Sha256: sum, + WritePath: dst, + MaxBytes: 1 << 20, + }) + if status.Code(err) != codes.InvalidArgument { + t.Errorf("%s plugin: FetchAsset of a %s:// asset = %v, want %s", tc.plugin, tc.asset, err, codes.InvalidArgument) + } + assertContent(t, dst, untouched) + if n := f.backend.gets.Load(); n != 0 { + t.Errorf("%s plugin read %d objects for a %s:// asset, want 0", tc.plugin, n, tc.asset) + } + } + }) + + t.Run("reported size over max_bytes fails before staging", func(t *testing.T) { + f := newAssetFixture(t, "gs") + f.backend.reportSize = true + sum := f.put(t, "kata-assets/vmlinux", content) + dst := f.writeFile(t, untouched) + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: "gs://" + testBucket + "/kata-assets/vmlinux", + Sha256: sum, + WritePath: dst, + MaxBytes: int64(len(content)) - 1, + }) + want := fmt.Sprintf("is %d bytes", len(content)) + if status.Code(err) != codes.FailedPrecondition || !strings.Contains(err.Error(), want) { + t.Fatalf("FetchAsset = %v, want %s naming the reported size (%q)", err, codes.FailedPrecondition, want) + } + assertContent(t, dst, untouched) + f.assertStagingEmpty(t) + }) + + t.Run("not found", func(t *testing.T) { + f := newAssetFixture(t, "gs") + dst := f.writeFile(t, untouched) + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: "gs://" + testBucket + "/kata-assets/missing", + Sha256: strings.Repeat("a", 64), + WritePath: dst, + MaxBytes: 1 << 20, + }) + if status.Code(err) != codes.NotFound { + t.Fatalf("FetchAsset of a missing object = %v, want %s", err, codes.NotFound) + } + assertContent(t, dst, untouched) + f.assertStagingEmpty(t) + }) + + t.Run("write_path must be an existing regular file below root", func(t *testing.T) { + f := newAssetFixture(t, "gs") + sum := f.put(t, "kata-assets/vmlinux", content) + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte(untouched), 0o600); err != nil { + t.Fatal(err) + } + dir := filepath.Join(f.root, "dir") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(f.root, "link") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, path string + want codes.Code + }{ + {"outside root", outside, codes.InvalidArgument}, + {"escapes root", filepath.Join(f.root, "..", filepath.Base(outside)), codes.InvalidArgument}, + {"relative", "static-files/x", codes.InvalidArgument}, + {"missing", filepath.Join(f.root, "missing"), codes.FailedPrecondition}, + {"directory", dir, codes.FailedPrecondition}, + {"symlink to a file outside root", link, codes.FailedPrecondition}, + } { + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: "gs://" + testBucket + "/kata-assets/vmlinux", + Sha256: sum, + WritePath: tc.path, + MaxBytes: 1 << 20, + }) + if status.Code(err) != tc.want { + t.Errorf("%s: FetchAsset = %v, want %s", tc.name, err, tc.want) + } + } + assertContent(t, outside, untouched) + if _, err := os.Lstat(filepath.Join(f.root, "missing")); !os.IsNotExist(err) { + t.Errorf("FetchAsset created the missing write path (lstat err = %v)", err) + } + if n := f.backend.gets.Load(); n != 0 { + t.Errorf("rejected requests read %d objects, want 0", n) + } + }) + + t.Run("invalid requests", func(t *testing.T) { + f := newAssetFixture(t, "gs") + sum := f.put(t, "kata-assets/vmlinux", content) + dst := f.writeFile(t, untouched) + for _, tc := range []struct { + name, uri, sum string + max int64 + }{ + {"https scheme", "https://storage.example.com/" + testBucket + "/kata-assets/vmlinux", sum, 1 << 20}, + {"no object", "gs://" + testBucket, sum, 1 << 20}, + {"no bucket", "gs:///kata-assets/vmlinux", sum, 1 << 20}, + {"query", "gs://" + testBucket + "/kata-assets/vmlinux?sig=secret", sum, 1 << 20}, + {"user info", "gs://usr0:secret@" + testBucket + "/kata-assets/vmlinux", sum, 1 << 20}, + {"fragment", "gs://" + testBucket + "/kata-assets/vmlinux#x", sum, 1 << 20}, + {"dot segment", "gs://" + testBucket + "/kata-assets/../kata-assets/vmlinux", sum, 1 << 20}, + {"empty segment", "gs://" + testBucket + "/kata-assets//vmlinux", sum, 1 << 20}, + {"trailing slash", "gs://" + testBucket + "/kata-assets/", sum, 1 << 20}, + {"padded segment", "gs://" + testBucket + "/kata-assets /vmlinux", sum, 1 << 20}, + {"backslash", "gs://" + testBucket + `/kata-assets\vmlinux`, sum, 1 << 20}, + {"upper-case sha256", "gs://" + testBucket + "/kata-assets/vmlinux", strings.ToUpper(sum), 1 << 20}, + {"short sha256", "gs://" + testBucket + "/kata-assets/vmlinux", sum[:63], 1 << 20}, + {"zero max_bytes", "gs://" + testBucket + "/kata-assets/vmlinux", sum, 0}, + {"over-long URI", "gs://" + testBucket + "/" + strings.Repeat("x/", assetfetch.MaxURIBytes/2) + "vmlinux", sum, 1 << 20}, + } { + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: tc.uri, Sha256: tc.sum, WritePath: dst, MaxBytes: tc.max, + }) + if status.Code(err) != codes.InvalidArgument { + t.Errorf("%s: FetchAsset = %v, want %s", tc.name, err, codes.InvalidArgument) + } + for _, secret := range []string{"secret", "usr0"} { + if strings.Contains(status.Convert(err).Message(), secret) { + t.Errorf("%s: error %q repeats the URI's credentials", tc.name, err) + } + } + } + assertContent(t, dst, untouched) + if n := f.backend.gets.Load(); n != 0 { + t.Errorf("invalid requests read %d objects, want 0", n) + } + }) +} + +// TestFetchAssetRefusesSnapshots seeds snapshot and tag objects in the same +// bucket as the assets, and asks for each with its correct sha256: the plugin +// must refuse every one before reading it. +func TestFetchAssetRefusesSnapshots(t *testing.T) { + for _, scheme := range []string{"gs", "s3"} { + t.Run(scheme, func(t *testing.T) { testFetchAssetRefusesSnapshots(t, scheme) }) + } +} + +func testFetchAssetRefusesSnapshots(t *testing.T, scheme string) { + f := newAssetFixture(t, scheme) + snapshotObjects := []string{ + testPrefix + "/manifest.json", + testPrefix + "/checkpoint.img.zstd", + "root/atespaces/team-a/tags/tag1/manifest.json", + "root/atespaces/team-a/tags/tag1/checkpoint.img.zstd", + // A snapshot at the bucket root, and the bare snapshot and tag + // prefixes, in case a backend stores an object under that name. + "atespaces/team-a/actors/uid1/snapshots/snap1/manifest.json", + testPrefix, + "root/atespaces/team-a/tags/tag1", + } + sums := map[string]string{} + for _, object := range snapshotObjects { + sums[object] = f.put(t, object, []byte("snapshot bytes of "+object)) + } + assetSum := f.put(t, "kata-assets/vmlinux", []byte("kernel")) + + dst := f.writeFile(t, untouched) + for _, object := range snapshotObjects { + uri := scheme + "://" + testBucket + "/" + object + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: uri, + Sha256: sums[object], + WritePath: dst, + MaxBytes: 1 << 20, + }) + if status.Code(err) != codes.PermissionDenied { + t.Errorf("FetchAsset(%s) = %v, want %s", uri, err, codes.PermissionDenied) + } + } + assertContent(t, dst, untouched) + if n := f.backend.gets.Load(); n != 0 { + t.Errorf("refused requests read %d objects, want 0", n) + } + + // Assets in the same bucket are still served, including names that + // share a segment with the snapshot layout. + for _, object := range []string{"kata-assets/vmlinux", "atespaces/readme", "root/atespaces/team-a/actors/uid1"} { + sum := assetSum + if object != "kata-assets/vmlinux" { + sum = f.put(t, object, []byte("asset "+object)) + } + if _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: scheme + "://" + testBucket + "/" + object, + Sha256: sum, + WritePath: dst, + MaxBytes: 1 << 20, + }); err != nil { + t.Errorf("FetchAsset(%s) = %v, want success", object, err) + } + } +} + +// TestFetchAssetManySegmentURIs sends URIs of up to assetfetch.MaxURIBytes made of +// as many path segments as fit, which the snapshot check reads in one pass. +func TestFetchAssetManySegmentURIs(t *testing.T) { + f := newAssetFixture(t, "s3") + dst := f.writeFile(t, untouched) + // pad returns "s3:///" + repeated unit + a non-empty filler + // segment + suffix, exactly assetfetch.MaxURIBytes long. + pad := func(unit, suffix string) string { + head := "s3://" + testBucket + "/" + avail := assetfetch.MaxURIBytes - len(head) - len(suffix) + n := (avail - 1) / len(unit) + uri := head + strings.Repeat(unit, n) + strings.Repeat("y", avail-n*len(unit)) + suffix + if len(uri) != assetfetch.MaxURIBytes { + t.Fatalf("pad built a %d byte URI, want %d", len(uri), assetfetch.MaxURIBytes) + } + return uri + } + + served := pad("x/", "/vmlinux") + object := strings.TrimPrefix(served, "s3://"+testBucket+"/") + sum := f.put(t, object, []byte("kernel")) + if _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: served, Sha256: sum, WritePath: dst, MaxBytes: 1 << 20, + }); err != nil { + t.Fatalf("FetchAsset(%d segments) = %v, want success", strings.Count(served, "/")-2, err) + } + assertContent(t, dst, "kernel") + gets := f.backend.gets.Load() + + for _, tc := range []struct { + name, uri string + want codes.Code + }{ + {"snapshot after many owner-like segments", pad("atespaces/team-a/", "/atespaces/team-a/actors/uid1/snapshots/snap1/manifest.json"), codes.PermissionDenied}, + {"tag after many owner-like segments", pad("atespaces/team-a/actors/", "/atespaces/team-a/tags/tag1/manifest.json"), codes.PermissionDenied}, + {"one byte over the limit", served + "x", codes.InvalidArgument}, + } { + _, err := f.client.FetchAsset(t.Context(), &objectstorev1.FetchAssetRequest{ + AssetUri: tc.uri, Sha256: sum, WritePath: dst, MaxBytes: 1 << 20, + }) + if status.Code(err) != tc.want { + t.Errorf("%s: FetchAsset = %v, want %s", tc.name, err, tc.want) + } + } + if n := f.backend.gets.Load() - gets; n != 0 { + t.Errorf("refused requests read %d objects, want 0", n) + } +} + +func TestNewAssetPluginRejectsStagingBelowRoot(t *testing.T) { + root := t.TempDir() + for _, staging := range []string{root, filepath.Join(root, "staging")} { + if _, err := NewAssetPlugin(newMemObjects(), "gs", root, staging, 1<<30); err == nil { + t.Errorf("NewAssetPlugin(root=%s, staging=%s) succeeded, want an error", root, staging) + } + } + for _, tc := range []struct{ root, staging string }{{"relative", t.TempDir()}, {root, "relative"}} { + if _, err := NewAssetPlugin(newMemObjects(), "gs", tc.root, tc.staging, 1<<30); err == nil { + t.Errorf("NewAssetPlugin(root=%s, staging=%s) succeeded, want an error", tc.root, tc.staging) + } + } + if _, err := NewAssetPlugin(newMemObjects(), "gs", root, t.TempDir(), 1<<30); err != nil { + t.Errorf("NewAssetPlugin with a separate staging directory: %v", err) + } +} + +func TestNewAssetPluginRejectsUnknownScheme(t *testing.T) { + for _, scheme := range []string{"", "https", "GS"} { + if _, err := NewAssetPlugin(newMemObjects(), scheme, t.TempDir(), t.TempDir(), 1<<30); err == nil { + t.Errorf("NewAssetPlugin(scheme=%q) succeeded, want an error", scheme) + } + } +} diff --git a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go index 1e4b592d2e..bc3441ac6a 100644 --- a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go +++ b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go @@ -43,6 +43,19 @@ func (c nodeClient) UploadSnapshot(ctx context.Context, in *objectstorev1.Upload return c.p.UploadSnapshot(ctx, in) } +// AssetClient returns a client that calls p directly. +func AssetClient(p *objectstoreplugin.AssetPlugin) objectstorev1.AssetProviderClient { + return assetClient{p} +} + +type assetClient struct { + p *objectstoreplugin.AssetPlugin +} + +func (c assetClient) FetchAsset(ctx context.Context, in *objectstorev1.FetchAssetRequest, _ ...grpc.CallOption) (*objectstorev1.FetchAssetResponse, error) { + return c.p.FetchAsset(ctx, in) +} + // ControlClient returns a client that calls a ControlPlugin on store directly. func ControlClient(store objectstore.Store) objectstorev1.ControlProviderClient { return controlClient{objectstoreplugin.NewControlPlugin(store)} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index fbf2536a06..fea6efe1c3 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -1236,7 +1236,11 @@ func (x *EgressGateway) GetAddress() string { // (e.g. the gVisor release tarball). type AssetFile struct { state protoimpl.MessageState `protogen:"open.v1"` - // gs:// URL to download the asset from. + // URL to download the asset from. atelet fetches every URL through the + // node's object-store plugin, and reads a gs:// URL anonymously only when + // the plugin cannot serve it. + // Masked in logged requests, since a signed URL carries its credential in + // the query. Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // Lower-case hex SHA256; names the cached file and verifies its integrity. Sha256 string `protobuf:"bytes,2,opt,name=sha256,proto3" json:"sha256,omitempty"` @@ -3312,9 +3316,9 @@ const file_atelet_proto_rawDesc = "" + "\fmemory_bytes\x18\v \x01(\x03R\vmemoryBytesB\x11\n" + "\x0f_egress_gateway\")\n" + "\rEgressGateway\x12\x18\n" + - "\aaddress\x18\x01 \x01(\tR\aaddress\"5\n" + - "\tAssetFile\x12\x10\n" + - "\x03url\x18\x01 \x01(\tR\x03url\x12\x16\n" + + "\aaddress\x18\x01 \x01(\tR\aaddress\":\n" + + "\tAssetFile\x12\x15\n" + + "\x03url\x18\x01 \x01(\tB\x03\x80\x01\x01R\x03url\x12\x16\n" + "\x06sha256\x18\x02 \x01(\tR\x06sha256\"\x8e\x01\n" + "\n" + "ArchAssets\x123\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 32446bced8..48c5a0a4a1 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -315,8 +315,12 @@ message EgressGateway { // AssetFile is one content-addressed file atelet fetches for a sandbox runtime // (e.g. the gVisor release tarball). message AssetFile { - // gs:// URL to download the asset from. - string url = 1; + // URL to download the asset from. atelet fetches every URL through the + // node's object-store plugin, and reads a gs:// URL anonymously only when + // the plugin cannot serve it. + // Masked in logged requests, since a signed URL carries its credential in + // the query. + string url = 1 [ debug_redact = true ]; // Lower-case hex SHA256; names the cached file and verifies its integrity. string sha256 = 2; } diff --git a/internal/protoredact/protoredact_test.go b/internal/protoredact/protoredact_test.go index cc1b946d4a..bd6bc5126b 100644 --- a/internal/protoredact/protoredact_test.go +++ b/internal/protoredact/protoredact_test.go @@ -40,7 +40,8 @@ import ( // TestRedactedRecursesIntoRealMapFields walks real messages whose maps hold // messages (atelet sandbox assets) and strings (ateapi selectors): map values -// are visited without panicking and non-sensitive content is left intact. +// are visited without panicking, labeled fields in them are masked, and +// non-sensitive content is left intact. func TestRedactedRecursesIntoRealMapFields(t *testing.T) { run := &ateletpb.RunRequest{ SandboxAssets: &ateletpb.SandboxAssets{ @@ -61,8 +62,13 @@ func TestRedactedRecursesIntoRealMapFields(t *testing.T) { if v := got.GetSpec().GetContainers()[0].GetEnv()[0].GetValue(); v != protoredact.Placeholder { t.Fatalf("env value = %q, want placeholder", v) } - if u := got.GetSandboxAssets().GetAssets()["amd64"].GetFiles()["runsc"].GetUrl(); u != "https://assets.example/runsc" { - t.Fatalf("map-of-message content was altered: %q", u) + // Inside the map value, the labeled url is masked and the sha256 is kept. + file := got.GetSandboxAssets().GetAssets()["amd64"].GetFiles()["runsc"] + if u := file.GetUrl(); u != protoredact.Placeholder { + t.Fatalf("asset url in a map value = %q, want placeholder", u) + } + if sum := file.GetSha256(); sum != "abc123" { + t.Fatalf("map-of-message content was altered: sha256 = %q", sum) } if run.GetSpec().GetContainers()[0].GetEnv()[0].GetValue() != "sk-secret" { t.Fatal("original mutated") @@ -411,9 +417,11 @@ func TestDebugRedactFieldsArePinned(t *testing.T) { want := map[string]bool{ "ateapi.EnvVar.value": true, "ateapi.MintActorJWTResponse.actor_jwt": true, + "atelet.AssetFile.url": true, "atelet.EnvEntry.value": true, "ateom.ContainerSpec.env": true, "credprovider.FetchSecretResponse.opaque_bytes": true, + "objectstore.v1.FetchAssetRequest.asset_uri": true, "objectstore.v1.FetchSnapshotRequest.actor_jwt": true, "objectstore.v1.UploadSnapshotRequest.actor_jwt": true, } diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 7abddfc91f..725d9696ac 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -160,6 +160,11 @@ spec: - node - --socket=/run/snapshot-plugin/node.sock - --root=/var/lib/ate + - --asset-staging-dir=/var/lib/snapshot-plugin/asset-staging + # atelet's 8 GiB per-asset cap, so any asset fits. Each download + # reserves its object's size; ones past the capacity wait their turn, + # so the asset-staging volume below never fills up. + - --asset-staging-capacity=8Gi # The plugin listens on a Unix socket, which a gRPC probe cannot # reach, and the image has no shell, so the binary checks itself. startupProbe: @@ -170,8 +175,8 @@ spec: - --socket=/run/snapshot-plugin/node.sock periodSeconds: 2 failureThreshold: 30 - # Reads and writes snapshot files under the /var/lib/ate hostPath as - # root, like atelet; needs no capabilities. + # Reads and writes snapshot and asset files under the /var/lib/ate + # hostPath as root, like atelet; needs no capabilities. securityContext: runAsUser: 0 runAsGroup: 0 @@ -185,10 +190,11 @@ spec: cpu: 50m memory: 128Mi env: - # Object storage settings for snapshot transfers. Unset or "gcs" - # selects GCS with ambient credentials; "s3" with the AWS_* variables - # selects S3 (see the kind overlay). atelet keeps its own copy below - # for sandbox asset downloads, which do not go through the plugin. + # Object storage settings for snapshot transfers and sandbox assets. + # Unset or "gcs" selects GCS with ambient credentials; "s3" with the + # AWS_* variables selects S3 (see the kind overlay). atelet has no + # storage settings of its own and exits if ATE_STORAGE_BACKEND is set + # on it. - name: ATE_STORAGE_BACKEND value: "gcs" envFrom: @@ -199,6 +205,11 @@ spec: mountPath: /var/lib/ate - name: snapshot-plugin mountPath: /run/snapshot-plugin + # Sandbox assets are downloaded here and verified against their sha256 + # before the plugin copies them below /var/lib/ate. Only this container + # mounts it, so atelet never sees bytes it did not pin. + - name: asset-staging + mountPath: /var/lib/snapshot-plugin/asset-staging containers: - name: atelet image: ko://github.com/agent-substrate/substrate/cmd/atelet @@ -258,8 +269,6 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) - - name: ATE_STORAGE_BACKEND - value: "gcs" # OTLP exporter settings are shared by every control plane component; # see manifests/ate-install/ate-otel-config.yaml. envFrom: @@ -314,6 +323,12 @@ spec: volumes: - name: snapshot-plugin emptyDir: {} + # Holds asset downloads until they are verified. The plugin keeps them + # within --asset-staging-capacity; the limit adds 1Gi for filesystem + # overhead, since kubelet evicts the pod if the volume exceeds it. + - name: asset-staging + emptyDir: + sizeLimit: 9Gi - name: run-ateom hostPath: path: /var/lib/ate diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index 7d3e919371..5260bac99b 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -109,9 +109,14 @@ spec: type: string url: description: |- - URL is where to download the asset from (e.g. a gs:// URL). It may be - fetched anonymously or with credentials depending on atelet's - configuration. + URL is where to download the asset from (e.g. a gs:// URL). atelet + fetches every URL through the node's object-store plugin, with the + plugin's credentials. The plugin refuses URLs under a snapshot location. + The in-tree plugin serves only its storage backend's scheme: gs:// on + GCS, s3:// on S3. A gs:// URL the plugin has no object for or cannot + serve is then read anonymously, which serves public buckets such as + gs://gvisor. + maxLength: 2048 minLength: 1 type: string required: diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 0963cc3ebc..089cd1454e 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -41,20 +41,6 @@ patches: # atelet DaemonSet consumes via envFrom; the kind copy lives at # ../ate-otel-config.yaml and is applied by the parent overlay or # by hack/install-ate.sh for a targeted redeploy. - env: - - name: ATE_STORAGE_BACKEND - value: s3 - - name: AWS_REGION - value: us-east-1 - - name: AWS_ENDPOINT_URL - value: http://rustfs.ate-system.svc:9000 - - name: AWS_S3_USE_PATH_STYLE - value: "true" - # TODO: use a secret / identity management - - name: AWS_ACCESS_KEY_ID - value: rustfsadmin - - name: AWS_SECRET_ACCESS_KEY - value: rustfsadmin # kind nodes ship no credential provider plugin (the args above # leave the --image-credential-provider-* flags unset), so drop # the base manifest's host mounts instead of fabricating empty @@ -70,9 +56,9 @@ patches: $patch: delete - name: image-credential-provider-config $patch: delete - # The snapshot plugin transfers atelet's external snapshots, so it - # needs the same rustfs settings. atelet keeps its own copy for - # sandbox assets. + # The snapshot plugin transfers atelet's external snapshots and + # fetches its private sandbox assets (the micro-VM assets + # stage-to-rustfs.sh uploads), so it points at rustfs. initContainers: - name: snapshot-plugin env: diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 9747a7c9c0..81702215cb 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -22,9 +22,13 @@ # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs) # are FETCHED at runtime from the cluster object store bucket ${BUCKET_NAME} -# under kata-assets/ (rustfs on kind, GCS on GKE) - NOT baked into the worker -# image. ateom boots cloud-hypervisor itself and gives the actor an overlay -# rootfs (virtio-fs RO lower + guest-tmpfs upper), so it fetches virtiofsd too. +# under kata-assets/ (rustfs on kind, GCS on GKE), through the node's +# object-store plugin - NOT baked into the worker image. ${BUCKET_SCHEME} is +# that store's URL scheme (s3 on kind, gs on GKE): the plugin serves only its +# own backend's scheme. The assets must stay outside any snapshot location, +# which the plugin refuses to serve as assets. ateom boots cloud-hypervisor +# itself and gives the actor an overlay rootfs (virtio-fs RO lower + +# guest-tmpfs upper), so it fetches virtiofsd too. # # The per-arch sha256 values below are the asset sets produced by # hack/microvm-assets/assemble.sh; atelet selects the block matching the node's @@ -46,29 +50,29 @@ spec: assets: arm64: cloud-hypervisor: - url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/cloud-hypervisor" sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" # virtiofsd serves the overlay RO lower (virtio-fs). virtiofsd: - url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/virtiofsd" sha256: "3aa9240045d2261930ef04138fe05cf288f94c867e63e5d0845758bac5e3d458" kata-kernel: - url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/vmlinux" sha256: "4960f6cf781274eaa445fc9f88bb8c5ce951649278bc9caa13afb1e99cfe80e6" kata-image: - url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/rootfs.img" sha256: "7e3bdd500eb92de17c377bd84f8ed57e50d6bb68c7ec08c88f5e1916cd49f30d" # amd64 assets are kata 4.1.0 (assemble.sh ARCH=amd64). amd64: cloud-hypervisor: - url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/cloud-hypervisor" sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" virtiofsd: - url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/virtiofsd" sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4" kata-kernel: - url: "gs://${BUCKET_NAME}/kata-assets/vmlinux" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/vmlinux" sha256: "8e9dbc3a6e4c26adb089d23d31201edebce905c7f3e31aa67b32bdd41df1b86f" kata-image: - url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img" + url: "${BUCKET_SCHEME}://${BUCKET_NAME}/kata-assets/rootfs.img" sha256: "96497f64da1de9c7473fef46c3d29ddd0805d334731cf9d903a21a5b2c33cefb" diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 6234d3c271..131f9cb50c 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -33,12 +33,17 @@ const ( // AssetFile is one content-addressed file that atelet fetches for a sandbox // runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). type AssetFile struct { - // URL is where to download the asset from (e.g. a gs:// URL). It may be - // fetched anonymously or with credentials depending on atelet's - // configuration. + // URL is where to download the asset from (e.g. a gs:// URL). atelet + // fetches every URL through the node's object-store plugin, with the + // plugin's credentials. The plugin refuses URLs under a snapshot location. + // The in-tree plugin serves only its storage backend's scheme: gs:// on + // GCS, s3:// on S3. A gs:// URL the plugin has no object for or cannot + // serve is then read anonymously, which serves public buckets such as + // gs://gvisor. // // +required // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=2048 URL string `json:"url"` // SHA256 is the lower-case hex SHA256 of the asset. It both names the cached diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index d7f98b0c08..fd6e4536b9 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -388,7 +388,7 @@ func TestShippedSandboxConfigManifests(t *testing.T) { if err != nil { t.Fatalf("read manifest: %v", err) } - raw = []byte(strings.ReplaceAll(string(raw), "${BUCKET_NAME}", "test-bucket")) + raw = []byte(strings.NewReplacer("${BUCKET_SCHEME}", "s3", "${BUCKET_NAME}", "test-bucket").Replace(string(raw))) sc := &SandboxConfig{} if err := yaml.UnmarshalStrict(raw, sc); err != nil { t.Fatalf("decode manifest: %v", err) diff --git a/pkg/assetfetch/assetfetch.go b/pkg/assetfetch/assetfetch.go new file mode 100644 index 0000000000..450a26917e --- /dev/null +++ b/pkg/assetfetch/assetfetch.go @@ -0,0 +1,446 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package assetfetch holds the checks every object-store plugin applies when +// it serves AssetProvider.FetchAsset, whatever its storage backend: +// +// - CheckObjectName refuses an object stored under a snapshot or tag +// location, before anything is read. +// - Stager.Fetch downloads into a directory private to the plugin, within +// its capacity, enforces max_bytes while copying, verifies the sha256, and +// only then replaces the caller's file. +// +// A plugin parses its own URI scheme into an object name and an opener, and +// leaves the rest to this package. Errors are gRPC statuses with the codes +// objectstore.v1 documents for FetchAsset. +package assetfetch + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "golang.org/x/sync/semaphore" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/api/validate/content" +) + +// MaxURIBytes bounds asset_uri, as the SandboxConfig CRD bounds +// AssetFile.url, so the checks below stay cheap whatever a SandboxConfig names. +const MaxURIBytes = 2048 + +// sha256Pattern matches the SandboxConfig's sha256 format. +var sha256Pattern = regexp.MustCompile(`^[a-f0-9]{64}$`) + +// ParseURI parses an asset URI of at most MaxURIBytes that carries no user +// info, query or fragment and is not opaque. Its errors do not quote the URI, +// since user info or a query may carry credentials; once it succeeds, the URI +// may be named in messages. +func ParseURI(uri string) (*url.URL, error) { + if len(uri) > MaxURIBytes { + return nil, status.Errorf(codes.InvalidArgument, "asset URI is %d bytes long, more than the limit of %d", len(uri), MaxURIBytes) + } + u, err := url.Parse(uri) + if err != nil { + return nil, status.Error(codes.InvalidArgument, "asset URI is not a valid URI") + } + if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.Opaque != "" { + return nil, status.Error(codes.InvalidArgument, "asset URI must not carry user info, a query or a fragment") + } + return u, nil +} + +// CheckObjectName requires object, the name of the object uri points at +// within its bucket or container, to be in canonical form and not stored +// under a snapshot or tag location. base is uri up to the object name (e.g. +// gs://); the error names the snapshot location as base/. +// +// Snapshot files, manifests and tags are never assets, however their sha256 +// came to be known, so this returns PermissionDenied for them. Every +// a snapshot location may have is matched, so the check holds whatever key +// prefix the plugin stores snapshots under. +func CheckObjectName(uri, base, object string) error { + if object == "" { + return status.Errorf(codes.InvalidArgument, "asset URI %s must name an object", uri) + } + // Snapshot URIs are built with url.JoinPath, so a snapshot object's name + // is always canonical. A non-canonical name could still resolve to one on + // a backend that cleans or trims names, past the check below. + if strings.Contains(object, `\`) { + return status.Errorf(codes.InvalidArgument, `asset URI %s must not contain '\'`, uri) + } + segments := strings.Split(object, "/") + for _, seg := range segments { + if seg == "" || strings.TrimSpace(seg) != seg || strings.HasSuffix(seg, ".") { + return status.Errorf(codes.InvalidArgument, "asset URI %s has an empty, '.'-terminated or space-padded path segment", uri) + } + } + if n := snapshotPrefixLen(segments); n > 0 { + return status.Errorf(codes.PermissionDenied, "asset URI %s is stored under snapshot %s/%s; assets must not live under a snapshot location", + uri, strings.TrimSuffix(base, "/"), strings.Join(segments[:n], "/")) + } + return nil +} + +// snapshotPrefixLen returns the length of a prefix of segments that names a +// snapshot or a tag, the way resources.ParseSnapshotURI reads one, or 0 if no +// prefix does: +// +// /atespaces//actors//snapshots/ +// /atespaces//tags/ +// +// segments must be canonical (see CheckObjectName), which makes every +// a valid snapshot location. Each segment is read at most a fixed number of +// times, so the cost is linear in the URI's length. +func snapshotPrefixLen(segments []string) int { + for i, seg := range segments { + if seg != "atespaces" { + continue + } + owner := segments[i+1:] + switch { + case len(owner) >= 3 && owner[1] == "tags" && + isResourceName(owner[0]) && isResourceName(owner[2]): + return i + 4 + case len(owner) >= 5 && owner[1] == "actors" && owner[3] == "snapshots" && + isResourceName(owner[0]) && isResourceName(owner[2]) && isResourceName(owner[4]): + return i + 6 + } + } + return 0 +} + +// isResourceName matches resources.IsValidResourceName, which names every +// atespace, actor UID, snapshot and tag in a snapshot location. +func isResourceName(name string) bool { + return len(content.IsDNS1123Label(name)) == 0 +} + +// Stager downloads assets into a staging directory private to the plugin and +// copies each to the caller's file only once its size and sha256 check out. +// The caller can therefore never read bytes it did not pin, such as a +// snapshot's. +// +// Downloads in flight never hold more than the staging capacity. Each opens +// its object first, so a missing object or one larger than max_bytes or the +// capacity fails without waiting for space. It then reserves the object's +// size, or as much as max_bytes allows if the storage did not report one, and +// waits until that much is free, with the object closed meanwhile. +type Stager struct { + // root confines every local path a caller names. + root string + // stagingDir holds downloads until they are verified. + stagingDir string + // capacity is the most the downloads in stagingDir may hold together. + capacity int64 + // reserved counts the capacity downloads in flight have reserved. + reserved *semaphore.Weighted +} + +// stagingPrefix starts the name of every download in the staging directory. +const stagingPrefix = "asset-" + +// NewStager returns a Stager that writes caller files only below root and +// stages downloads in stagingDir, which must not be below root, holding at +// most capacity bytes there at once. +// +// stagingDir is private to the plugin, so NewStager removes the downloads a +// previous run of the plugin left there; they would otherwise use up its +// capacity. +func NewStager(root, stagingDir string, capacity int64) (*Stager, error) { + if !filepath.IsAbs(root) { + return nil, fmt.Errorf("root %q is not an absolute path", root) + } + if !filepath.IsAbs(stagingDir) { + return nil, fmt.Errorf("asset staging directory %q is not an absolute path", stagingDir) + } + if capacity <= 0 { + return nil, fmt.Errorf("asset staging capacity %d is not positive", capacity) + } + root, stagingDir = filepath.Clean(root), filepath.Clean(stagingDir) + if rel, err := filepath.Rel(root, stagingDir); err == nil && (rel == "." || filepath.IsLocal(rel)) { + return nil, fmt.Errorf("asset staging directory %q is inside root %q, where the caller could read unverified downloads", stagingDir, root) + } + entries, err := os.ReadDir(stagingDir) + if err != nil { + return nil, fmt.Errorf("while reading asset staging directory: %w", err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), stagingPrefix) { + if err := os.Remove(filepath.Join(stagingDir, e.Name())); err != nil { + return nil, fmt.Errorf("while removing a previous download from the asset staging directory: %w", err) + } + } + } + return &Stager{root: root, stagingDir: stagingDir, capacity: capacity, reserved: semaphore.NewWeighted(capacity)}, nil +} + +// OpenFunc opens the asset for reading and returns its size in bytes as the +// storage reported it, or a negative size if it reported none. Fetch may call +// it twice, if it has to wait for staging space. +type OpenFunc func(ctx context.Context) (rc io.ReadCloser, size int64, err error) + +// Fetch checks req's sha256, max_bytes and write_path, downloads the asset +// open returns into the staging directory and, if it is at most max_bytes +// long and hashes to sha256, replaces the existing file at write_path with it. +// uri is req's asset URI once it is safe to quote (see ParseURI). toStatus +// maps every other error, from open, the download or a local file, to a gRPC +// status; it must keep a context error's code. +// +// The file at write_path keeps its old content on any error: the asset is +// written to a new file in the same directory, which is then swapped into +// place. Callers must therefore open write_path after Fetch returns; a +// descriptor opened before reads the old file. +// +// The caller checks the URI (ParseURI, CheckObjectName) first, so a refused +// request reads nothing. +func (s *Stager) Fetch(ctx context.Context, req *objectstorev1.FetchAssetRequest, uri string, open OpenFunc, toStatus func(error) error) error { + if !sha256Pattern.MatchString(req.GetSha256()) { + return status.Errorf(codes.InvalidArgument, "sha256 %q is not 64 lower-case hex characters", req.GetSha256()) + } + wantSum, _ := hex.DecodeString(req.GetSha256()) + if req.GetMaxBytes() <= 0 { + return status.Errorf(codes.InvalidArgument, "max_bytes %d is not positive", req.GetMaxBytes()) + } + rel, err := s.writePath(req.GetWritePath()) + if err != nil { + return err + } + root, err := os.OpenRoot(s.root) + if err != nil { + return toStatus(fmt.Errorf("while opening %s: %w", s.root, err)) + } + defer root.Close() + // Checked before downloading so a bad request costs no transfer. Lstat: + // a symlink is never the file the caller created, a directory cannot be + // replaced by a file, and a FIFO is not where an asset belongs. + info, err := checkWriteFile(root, rel, req.GetWritePath(), toStatus) + if err != nil { + return err + } + + maxBytes := req.GetMaxBytes() + rc, size, err := s.open(ctx, open, uri, maxBytes, toStatus) + if err != nil { + return err + } + reserve := s.reservation(size, maxBytes) + if !s.reserved.TryAcquire(reserve) { + // Waiting can take as long as other downloads do. Closing the object + // meanwhile frees its connection and any read-ahead the storage + // client started. + rc.Close() + if err := s.reserved.Acquire(ctx, reserve); err != nil { + return toStatus(fmt.Errorf("while waiting for asset staging space: %w", err)) + } + var reopened int64 + rc, reopened, err = s.open(ctx, open, uri, maxBytes, toStatus) + if err == nil && reopened != size { + rc.Close() + err = status.Errorf(codes.Unavailable, "asset %s changed while waiting for staging space", uri) + } + if err != nil { + s.reserved.Release(reserve) + return err + } + } + defer s.reserved.Release(reserve) + + staged, err := s.stage(rc, uri, maxBytes, size, reserve, wantSum, toStatus) + if err != nil { + return err + } + defer func() { + staged.Close() + os.Remove(staged.Name()) + }() + return replace(root, rel, info.Mode().Perm(), staged, uri, req.GetWritePath(), toStatus) +} + +// open opens the asset and checks the size the storage reported for it, if +// any, against maxBytes and the staging capacity. +func (s *Stager) open(ctx context.Context, open OpenFunc, uri string, maxBytes int64, toStatus func(error) error) (io.ReadCloser, int64, error) { + rc, size, err := open(ctx) + if err != nil { + return nil, 0, toStatus(fmt.Errorf("while opening asset %s: %w", uri, err)) + } + switch { + case size > maxBytes: + rc.Close() + return nil, 0, status.Errorf(codes.FailedPrecondition, "asset %s is %d bytes, larger than max_bytes %d", uri, size, maxBytes) + case size > s.capacity: + rc.Close() + return nil, 0, status.Errorf(codes.FailedPrecondition, "asset %s is %d bytes, larger than the plugin's asset staging capacity of %d bytes", uri, size, s.capacity) + case size < 0: + size = -1 + } + return rc, size, nil +} + +// reservation is how much staging space a download of an object of the given +// size, negative if unknown, reserves: its size, or else as much as maxBytes +// allows. An object larger than the capacity cannot be staged whatever +// max_bytes allows, so a download never reserves more than all of it. +func (s *Stager) reservation(size, maxBytes int64) int64 { + if size >= 0 { + return size + } + return min(maxBytes, s.capacity) +} + +// stage downloads the asset from rc, which it closes, into a new file in the +// staging directory and returns it, positioned at its start, once its size +// and sha256 check out. size is the object's size as the storage reported it, +// or negative if unknown; reserve is the staging space reserved for it, which +// bounds what is written there. The file is removed on error; on success the +// caller removes it. +func (s *Stager) stage(rc io.ReadCloser, uri string, maxBytes, size, reserve int64, wantSum []byte, toStatus func(error) error) (_ *os.File, retErr error) { + defer rc.Close() + f, err := os.CreateTemp(s.stagingDir, stagingPrefix) + if err != nil { + return nil, toStatus(fmt.Errorf("while creating staging file: %w", err)) + } + defer func() { + if retErr != nil { + f.Close() + os.Remove(f.Name()) + } + }() + + hasher := sha256.New() + n, err := io.Copy(io.MultiWriter(f, hasher), io.LimitReader(rc, reserve)) + if err != nil { + return nil, toStatus(fmt.Errorf("while downloading asset %s: %w", uri, err)) + } + if n == reserve { + // One more byte, read but not staged, tells a full-length object + // from an over-long one. + var probe [1]byte + switch _, err := io.ReadFull(rc, probe[:]); { + case err == nil && size >= 0: + return nil, status.Errorf(codes.Unavailable, "asset %s is longer than the %d bytes its storage reported", uri, size) + case err == nil && reserve < maxBytes: + return nil, status.Errorf(codes.FailedPrecondition, "asset %s is larger than the plugin's asset staging capacity of %d bytes", uri, reserve) + case err == nil: + return nil, status.Errorf(codes.FailedPrecondition, "asset %s is larger than max_bytes %d", uri, maxBytes) + case err != io.EOF: + return nil, toStatus(fmt.Errorf("while downloading asset %s: %w", uri, err)) + } + } + if got := hasher.Sum(nil); !bytes.Equal(got, wantSum) { + return nil, status.Errorf(codes.FailedPrecondition, "asset %s has sha256 %x, want %x", uri, got, wantSum) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, toStatus(fmt.Errorf("while rewinding staging file: %w", err)) + } + return f, nil +} + +// copyToTemp copies a verified asset into the file that replaces the caller's. +// Tests replace it to fail the copy. +var copyToTemp = io.Copy + +// replace writes staged to a new file with mode perm next to rel, below root, +// and swaps it into rel's place, so the file at rel holds either its old +// content or all of staged. It never creates rel: if the caller removed it +// meanwhile, replace fails and leaves nothing behind. +func replace(root *os.Root, rel string, perm fs.FileMode, staged io.Reader, uri, shown string, toStatus func(error) error) error { + dir, base := filepath.Split(rel) + if dir == "" { + dir = "." + } + name, err := tempName() + if err != nil { + return toStatus(err) + } + tmpRel := filepath.Join(dir, name) + tmp, err := root.OpenFile(tmpRel, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return toStatus(fmt.Errorf("while creating a file next to write path %q: %w", shown, err)) + } + // After a swap, tmpRel names the caller's old file; either way it goes. + defer func() { + tmp.Close() + _ = root.Remove(tmpRel) + }() + if err := tmp.Chmod(perm); err != nil { + return toStatus(fmt.Errorf("while setting the mode of a file next to write path %q: %w", shown, err)) + } + if _, err := copyToTemp(tmp, staged); err != nil { + return toStatus(fmt.Errorf("while writing asset %s next to write path %q: %w", uri, shown, err)) + } + if err := tmp.Close(); err != nil { + return toStatus(fmt.Errorf("while closing a file next to write path %q: %w", shown, err)) + } + err = exchange(root, dir, name, base) + if errors.Is(err, os.ErrNotExist) { + return status.Errorf(codes.FailedPrecondition, "write path %q does not exist", shown) + } + if err != nil { + return toStatus(fmt.Errorf("while replacing write path %q: %w", shown, err)) + } + return nil +} + +// tempName returns a fixed-length random file name, so the caller's file name +// may be as long as the filesystem allows. +func tempName() (string, error) { + var b [12]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("while naming a temporary file: %w", err) + } + return ".assetfetch-" + hex.EncodeToString(b[:]), nil +} + +// writePath checks that path is absolute and below s.root, and returns it +// relative to s.root. +func (s *Stager) writePath(path string) (string, error) { + if !filepath.IsAbs(path) { + return "", status.Errorf(codes.InvalidArgument, "write path %q is not absolute", path) + } + rel, err := filepath.Rel(s.root, path) + if err != nil || !filepath.IsLocal(rel) { + return "", status.Errorf(codes.InvalidArgument, "write path %q is not below %q", path, s.root) + } + return rel, nil +} + +// checkWriteFile requires rel, below root, to be an existing regular file and +// returns its FileInfo. +func checkWriteFile(root *os.Root, rel, shown string, toStatus func(error) error) (fs.FileInfo, error) { + info, err := root.Lstat(rel) + if errors.Is(err, os.ErrNotExist) { + return nil, status.Errorf(codes.FailedPrecondition, "write path %q does not exist", shown) + } + if err != nil { + return nil, toStatus(fmt.Errorf("while inspecting write path %q: %w", shown, err)) + } + if !info.Mode().IsRegular() { + return nil, status.Errorf(codes.FailedPrecondition, "write path %q is not a regular file (mode %v)", shown, info.Mode().Type()) + } + return info, nil +} diff --git a/pkg/assetfetch/assetfetch_test.go b/pkg/assetfetch/assetfetch_test.go new file mode 100644 index 0000000000..9b8ee6006b --- /dev/null +++ b/pkg/assetfetch/assetfetch_test.go @@ -0,0 +1,916 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assetfetch + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/resources" + objectstorev1 "github.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const testBucket = "bucket" + +func TestParseURI(t *testing.T) { + for _, tc := range []struct { + uri string + want codes.Code + }{ + {"gs://bucket/kata-assets/vmlinux", codes.OK}, + {"https://acct.blob.core.windows.net/assets/vmlinux", codes.OK}, + {"https://acct.blob.core.windows.net/assets/vmlinux?sig=secret", codes.InvalidArgument}, + {"https://acct.blob.core.windows.net/assets/vmlinux?", codes.InvalidArgument}, + {"https://user:secret@acct.blob.core.windows.net/assets/vmlinux", codes.InvalidArgument}, + {"https://acct.blob.core.windows.net/assets/vmlinux#frag", codes.InvalidArgument}, + {"mailto:secret@example.com", codes.InvalidArgument}, + {"https://acct\x7f/x", codes.InvalidArgument}, + {"gs://bucket/" + strings.Repeat("x", MaxURIBytes), codes.InvalidArgument}, + } { + _, err := ParseURI(tc.uri) + if got := status.Code(err); got != tc.want { + t.Errorf("ParseURI(%.60q) = %v, want %s", tc.uri, err, tc.want) + } + if err != nil && strings.Contains(err.Error(), "secret") { + t.Errorf("ParseURI(%q) error %q quotes the URI's credentials", tc.uri, err) + } + } +} + +func TestCheckObjectName(t *testing.T) { + const base = "https://acct.blob.core.windows.net/assets" + for _, tc := range []struct { + object string + want codes.Code + }{ + {"kata-assets/vmlinux", codes.OK}, + {"vmlinux", codes.OK}, + {"atespaces/readme", codes.OK}, + {"root/atespaces/team-a/actors/uid1", codes.OK}, + {"root/atespaces/Not_A_Name/tags/tag1/manifest.json", codes.OK}, + {"", codes.InvalidArgument}, + {"a//b", codes.InvalidArgument}, + {"a/b/", codes.InvalidArgument}, + {"/a", codes.InvalidArgument}, + {"a/../b", codes.InvalidArgument}, + {"a/./b", codes.InvalidArgument}, + {"a/b.", codes.InvalidArgument}, + {"a/ b", codes.InvalidArgument}, + {`a\b`, codes.InvalidArgument}, + {"substrate/atespaces/team-a/actors/uid1/snapshots/snap1/pages.img", codes.PermissionDenied}, + {"substrate/atespaces/team-a/actors/uid1/snapshots/snap1", codes.PermissionDenied}, + {"atespaces/team-a/actors/uid1/snapshots/snap1/manifest.json", codes.PermissionDenied}, + {"x/y/atespaces/ate-golden/tags/d33a8f2f-c6ce-45b7-9659-69d1fee12fa5/checkpoint.img", codes.PermissionDenied}, + {"x/atespaces/team-a/tags/tag1", codes.PermissionDenied}, + } { + uri := base + "/" + tc.object + err := CheckObjectName(uri, base, tc.object) + if got := status.Code(err); got != tc.want { + t.Errorf("CheckObjectName(%q) = %v, want %s", tc.object, err, tc.want) + } + } + err := CheckObjectName(base+"/s/atespaces/a/tags/t/f", base, "s/atespaces/a/tags/t/f") + if want := base + "/s/atespaces/a/tags/t;"; !strings.Contains(err.Error(), want) { + t.Errorf("CheckObjectName error %q does not name the tag location %q", err, want) + } +} + +// TestSnapshotPrefixLenMatchesParseSnapshotURI checks snapshotPrefixLen +// against resources.ParseSnapshotURI run on every prefix, for every path of +// up to 7 segments drawn from the snapshot layout's keywords and an invalid +// resource name. +func TestSnapshotPrefixLenMatchesParseSnapshotURI(t *testing.T) { + alphabet := []string{"atespaces", "actors", "tags", "snapshots", "Not_A_Name"} + parsesAt := func(segments []string) int { + for i := 1; i <= len(segments); i++ { + if _, err := resources.ParseSnapshotURI("gs://" + testBucket + "/" + strings.Join(segments[:i], "/")); err == nil { + return i + } + } + return 0 + } + var walk func(segments []string) + walk = func(segments []string) { + if len(segments) > 0 { + got, want := snapshotPrefixLen(segments), parsesAt(segments) + if (got > 0) != (want > 0) { + t.Errorf("snapshotPrefixLen(%q) = %d, but ParseSnapshotURI accepts a prefix of length %d", segments, got, want) + } else if got > 0 { + if _, err := resources.ParseSnapshotURI("gs://" + testBucket + "/" + strings.Join(segments[:got], "/")); err != nil { + t.Errorf("snapshotPrefixLen(%q) = %d, but ParseSnapshotURI rejects that prefix: %v", segments, got, err) + } + } + } + if len(segments) == 7 { + return + } + for _, seg := range alphabet { + walk(append(segments[:len(segments):len(segments)], seg)) + } + } + walk(nil) +} + +// TestSnapshotPrefixLenIsLinear runs the check on far more segments than a +// URI can hold. A check that re-reads every prefix would not finish. +func TestSnapshotPrefixLenIsLinear(t *testing.T) { + unit := []string{"atespaces", "team-a", "actors", "uid1", "snapshots", "Not_A_Name", "atespaces", "team-a", "atespaces"} + segments := make([]string, 0, 1<<20+6) + for len(segments) < 1<<20 { + segments = append(segments, unit...) + } + if n := snapshotPrefixLen(segments); n != 0 { + t.Fatalf("snapshotPrefixLen = %d, want 0", n) + } + segments = append(segments, "atespaces", "team-a", "tags", "tag1") + if n, want := snapshotPrefixLen(segments), len(segments); n != want { + t.Fatalf("snapshotPrefixLen = %d, want %d", n, want) + } +} + +// stagerFixture is a Stager with a write root, a separate staging directory +// and an existing caller file below the root. +type stagerFixture struct { + stager *Stager + staging string + dst string +} + +const untouched = "the caller's original bytes" + +func newStagerFixture(t *testing.T) *stagerFixture { + t.Helper() + root, staging := t.TempDir(), t.TempDir() + stager, err := NewStager(root, staging, 1<<30) + if err != nil { + t.Fatal(err) + } + dst := filepath.Join(root, "static-files", "asset-download-1") + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dst, []byte(untouched), 0o600); err != nil { + t.Fatal(err) + } + return &stagerFixture{stager: stager, staging: staging, dst: dst} +} + +func (f *stagerFixture) assertDst(t *testing.T, want string) { + t.Helper() + got, err := os.ReadFile(f.dst) + if err != nil { + t.Fatal(err) + } + if string(got) != want { + t.Errorf("write path holds %q, want %q", got, want) + } + f.assertNoTempFiles(t) +} + +// assertNoTempFiles checks that no file written next to the write path is +// left behind. +func (f *stagerFixture) assertNoTempFiles(t *testing.T) { + t.Helper() + entries, err := os.ReadDir(filepath.Dir(f.dst)) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".assetfetch-") { + t.Errorf("write path's directory holds %s after the call", e.Name()) + } + } +} + +func (f *stagerFixture) assertStagingEmpty(t *testing.T) { + t.Helper() + entries, err := os.ReadDir(f.staging) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("staging directory holds %d entries after the call, want none", len(entries)) + } +} + +// opener serves content and counts how often it is opened and closed. It +// reports size as the object's size if sized is set, and no size otherwise. +type opener struct { + content string + sized bool + size int64 + err error + opens int + closes int +} + +func (o *opener) open(context.Context) (io.ReadCloser, int64, error) { + o.opens++ + if o.err != nil { + return nil, 0, o.err + } + size := int64(-1) + if o.sized { + size = o.size + } + return &closeCounter{Reader: strings.NewReader(o.content), closes: &o.closes}, size, nil +} + +// closeCounter counts Close calls. +type closeCounter struct { + io.Reader + closes *int +} + +func (c *closeCounter) Close() error { + *c.closes++ + return nil +} + +var ( + errBackend = errors.New("backend error") + errNotFound = errors.New("no such object") +) + +// testStatus maps errBackend to Unavailable, errNotFound to NotFound, a +// context error to its code, and everything else to Internal, like a +// plugin's own mapping. +func testStatus(err error) error { + switch { + case errors.Is(err, errBackend): + return status.Error(codes.Unavailable, err.Error()) + case errors.Is(err, errNotFound): + return status.Error(codes.NotFound, err.Error()) + case errors.Is(err, context.Canceled): + return status.Error(codes.Canceled, err.Error()) + } + return status.Error(codes.Internal, err.Error()) +} + +func sum(s string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) } + +func TestStagerFetch(t *testing.T) { + const kernel = "kernel bytes" + t.Run("verified content replaces the caller's file", func(t *testing.T) { + f := newStagerFixture(t) + o := &opener{content: kernel} + err := f.stager.Fetch(t.Context(), &objectstorev1.FetchAssetRequest{ + Sha256: sum(kernel), WritePath: f.dst, MaxBytes: int64(len(kernel)), + }, "gs://bucket/vmlinux", o.open, testStatus) + if err != nil { + t.Fatalf("Fetch = %v, want success", err) + } + f.assertDst(t, kernel) + f.assertStagingEmpty(t) + }) + + for _, tc := range []struct { + name string + req func(f *stagerFixture) *objectstorev1.FetchAssetRequest + opener *opener + want codes.Code + wantOpens int + }{ + { + name: "sha256 mismatch", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { return req(sum("other"), f.dst, 1<<20) }, + opener: &opener{content: kernel}, + want: codes.FailedPrecondition, + wantOpens: 1, + }, + { + name: "one byte over max_bytes", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(sum(kernel), f.dst, int64(len(kernel)-1)) + }, + opener: &opener{content: kernel}, + want: codes.FailedPrecondition, + wantOpens: 1, + }, + { + name: "reported size over max_bytes fails before reading", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(sum(kernel), f.dst, int64(len(kernel)-1)) + }, + opener: &opener{content: kernel, sized: true, size: int64(len(kernel))}, + want: codes.FailedPrecondition, + wantOpens: 1, + }, + { + name: "reported size over the staging capacity fails before reading", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { return req(sum(kernel), f.dst, 1<<40) }, + opener: &opener{content: kernel, sized: true, size: 1 << 31}, + want: codes.FailedPrecondition, + wantOpens: 1, + }, + { + name: "object longer than its reported size", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { return req(sum(kernel), f.dst, 1<<20) }, + opener: &opener{content: kernel, sized: true, size: int64(len(kernel) - 1)}, + want: codes.Unavailable, + wantOpens: 1, + }, + { + name: "open error goes through toStatus", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { return req(sum(kernel), f.dst, 1<<20) }, + opener: &opener{err: errBackend}, + want: codes.Unavailable, + wantOpens: 1, + }, + { + name: "upper-case sha256", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(strings.ToUpper(sum(kernel)), f.dst, 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.InvalidArgument, + }, + { + name: "zero max_bytes", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { return req(sum(kernel), f.dst, 0) }, + opener: &opener{content: kernel}, + want: codes.InvalidArgument, + }, + { + name: "relative write path", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(sum(kernel), "static-files/x", 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.InvalidArgument, + }, + { + name: "write path outside root", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(sum(kernel), filepath.Join(f.staging, "x"), 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.InvalidArgument, + }, + { + name: "missing write path", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(sum(kernel), f.dst+"-missing", 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.FailedPrecondition, + }, + { + name: "write path is a directory", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + return req(sum(kernel), filepath.Dir(f.dst), 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.FailedPrecondition, + }, + { + name: "write path is a symlink", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + link := f.dst + "-link" + if err := os.Symlink(f.dst, link); err != nil { + panic(err) + } + return req(sum(kernel), link, 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.FailedPrecondition, + }, + { + name: "write path is a FIFO", + req: func(f *stagerFixture) *objectstorev1.FetchAssetRequest { + fifo := f.dst + "-fifo" + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + panic(err) + } + return req(sum(kernel), fifo, 1<<20) + }, + opener: &opener{content: kernel}, + want: codes.FailedPrecondition, + }, + } { + t.Run(tc.name, func(t *testing.T) { + f := newStagerFixture(t) + err := f.stager.Fetch(t.Context(), tc.req(f), "gs://bucket/vmlinux", tc.opener.open, testStatus) + if got := status.Code(err); got != tc.want { + t.Fatalf("Fetch = %v, want %s", err, tc.want) + } + if tc.opener.opens != tc.wantOpens { + t.Errorf("Fetch opened the asset %d times, want %d", tc.opener.opens, tc.wantOpens) + } + if tc.opener.closes != tc.opener.opens && tc.opener.err == nil { + t.Errorf("Fetch opened the asset %d times and closed it %d times", tc.opener.opens, tc.opener.closes) + } + f.assertDst(t, untouched) + f.assertStagingEmpty(t) + }) + } + + t.Run("reported size", func(t *testing.T) { + f := newStagerFixture(t) + o := &opener{content: kernel, sized: true, size: int64(len(kernel))} + if err := f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus); err != nil { + t.Fatalf("Fetch = %v, want success", err) + } + f.assertDst(t, kernel) + f.assertStagingEmpty(t) + }) + + t.Run("canceled context", func(t *testing.T) { + f := newStagerFixture(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + o := &opener{err: context.Canceled} + err := f.stager.Fetch(ctx, req(sum(kernel), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus) + if got := status.Code(err); got != codes.Canceled { + t.Fatalf("Fetch = %v, want %s", err, codes.Canceled) + } + f.assertDst(t, untouched) + f.assertStagingEmpty(t) + }) +} + +func req(sha string, writePath string, maxBytes int64) *objectstorev1.FetchAssetRequest { + return &objectstorev1.FetchAssetRequest{Sha256: sha, WritePath: writePath, MaxBytes: maxBytes} +} + +func TestNewStagerRejectsStagingBelowRoot(t *testing.T) { + root := t.TempDir() + for _, staging := range []string{root, filepath.Join(root, "staging"), root + "/./staging/.."} { + if _, err := NewStager(root, staging, 1<<30); err == nil { + t.Errorf("NewStager(root=%s, staging=%s) succeeded, want an error", root, staging) + } + } + for _, tc := range []struct{ root, staging string }{{"relative", t.TempDir()}, {root, "relative"}} { + if _, err := NewStager(tc.root, tc.staging, 1<<30); err == nil { + t.Errorf("NewStager(root=%s, staging=%s) succeeded, want an error", tc.root, tc.staging) + } + } + if _, err := NewStager(root, t.TempDir(), 1<<30); err != nil { + t.Errorf("NewStager with a separate staging directory: %v", err) + } +} + +func TestNewStagerRejectsNonPositiveCapacity(t *testing.T) { + for _, capacity := range []int64{0, -1} { + if _, err := NewStager(t.TempDir(), t.TempDir(), capacity); err == nil { + t.Errorf("NewStager(capacity=%d) succeeded, want an error", capacity) + } + } +} + +func TestNewStagerRemovesPreviousDownloads(t *testing.T) { + staging := t.TempDir() + for _, name := range []string{"asset-123", "asset-456", "unrelated"} { + if err := os.WriteFile(filepath.Join(staging, name), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + if _, err := NewStager(t.TempDir(), staging, 1<<30); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(staging) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != "unrelated" { + t.Errorf("staging directory holds %v after NewStager, want only unrelated", entries) + } +} + +// failingCopy writes half of src and then fails, as a full disk would. +func failingCopy(dst io.Writer, src io.Reader) (int64, error) { + b, err := io.ReadAll(src) + if err != nil { + return 0, err + } + n, _ := dst.Write(b[:len(b)/2]) + return int64(n), errors.New("no space left on device") +} + +func setCopyToTemp(t *testing.T, fn func(io.Writer, io.Reader) (int64, error)) { + t.Helper() + orig := copyToTemp + copyToTemp = fn + t.Cleanup(func() { copyToTemp = orig }) +} + +func TestStagerFetchReplacesWritePathAtomically(t *testing.T) { + const kernel = "kernel bytes" + + t.Run("failed copy leaves the write path untouched", func(t *testing.T) { + setCopyToTemp(t, failingCopy) + f := newStagerFixture(t) + o := &opener{content: kernel} + err := f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus) + if got := status.Code(err); got != codes.Internal { + t.Fatalf("Fetch = %v, want %s", err, codes.Internal) + } + f.assertDst(t, untouched) + f.assertStagingEmpty(t) + }) + + t.Run("write path removed during the copy is not recreated", func(t *testing.T) { + f := newStagerFixture(t) + setCopyToTemp(t, func(dst io.Writer, src io.Reader) (int64, error) { + if err := os.Remove(f.dst); err != nil { + return 0, err + } + return io.Copy(dst, src) + }) + o := &opener{content: kernel} + err := f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus) + if got := status.Code(err); got != codes.FailedPrecondition { + t.Fatalf("Fetch = %v, want %s", err, codes.FailedPrecondition) + } + if _, err := os.Lstat(f.dst); !errors.Is(err, os.ErrNotExist) { + t.Errorf("write path exists after the call (%v), want it left removed", err) + } + f.assertNoTempFiles(t) + f.assertStagingEmpty(t) + }) + + t.Run("keeps the write path's mode", func(t *testing.T) { + f := newStagerFixture(t) + if err := os.Chmod(f.dst, 0o640); err != nil { + t.Fatal(err) + } + o := &opener{content: kernel} + if err := f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus); err != nil { + t.Fatalf("Fetch = %v, want success", err) + } + f.assertDst(t, kernel) + info, err := os.Stat(f.dst) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o640 { + t.Errorf("write path mode = %v, want %v", got, fs.FileMode(0o640)) + } + }) + + t.Run("longest file name", func(t *testing.T) { + f := newStagerFixture(t) + f.dst = filepath.Join(filepath.Dir(f.dst), strings.Repeat("a", 255)) + if err := os.WriteFile(f.dst, []byte(untouched), 0o600); err != nil { + t.Fatal(err) + } + o := &opener{content: kernel} + if err := f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus); err != nil { + t.Fatalf("Fetch = %v, want success", err) + } + f.assertDst(t, kernel) + }) +} + +// gatedOpener serves content, reporting its size if sized is set. Each +// reader it returns reports its first Read on reading, and then blocks until +// release is closed, and reports Close on closed. Opens and closes are +// counted. +type gatedOpener struct { + content string + sized bool + reading chan struct{} + closed chan struct{} + release chan struct{} + opens atomic.Int64 + closes atomic.Int64 +} + +func newGatedOpener(content string) *gatedOpener { + return &gatedOpener{content: content, reading: make(chan struct{}, 4), closed: make(chan struct{}, 4), release: make(chan struct{})} +} + +func (o *gatedOpener) open(ctx context.Context) (io.ReadCloser, int64, error) { + o.opens.Add(1) + size := int64(-1) + if o.sized { + size = int64(len(o.content)) + } + return &gatedReader{o: o, ctx: ctx, r: strings.NewReader(o.content)}, size, nil +} + +type gatedReader struct { + o *gatedOpener + ctx context.Context + r io.Reader + started bool +} + +func (g *gatedReader) Read(p []byte) (int, error) { + if !g.started { + g.started = true + g.o.reading <- struct{}{} + select { + case <-g.o.release: + case <-g.ctx.Done(): + return 0, g.ctx.Err() + } + } + return g.r.Read(p) +} + +func (g *gatedReader) Close() error { + g.o.closes.Add(1) + g.o.closed <- struct{}{} + return nil +} + +func newStagerFixtureWithCapacity(t *testing.T, capacity int64) *stagerFixture { + t.Helper() + f := newStagerFixture(t) + stager, err := NewStager(f.stager.root, f.staging, capacity) + if err != nil { + t.Fatal(err) + } + f.stager = stager + return f +} + +// newCallerFile creates another existing caller file next to f.dst. +func (f *stagerFixture) newCallerFile(t *testing.T, name string) string { + t.Helper() + p := filepath.Join(filepath.Dir(f.dst), name) + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestStagerFetchStagingCapacity(t *testing.T) { + const kernel = "kernel bytes" + const maxBytes = 60 + + // holdCapacity starts a download that reserves maxBytes and blocks in + // its first read until the returned opener's release is closed. + holdCapacity := func(t *testing.T, f *stagerFixture) (*gatedOpener, <-chan error) { + t.Helper() + first := newGatedOpener(kernel) + firstDone := make(chan error, 1) + dst := f.newCallerFile(t, "held") + go func() { + firstDone <- f.stager.Fetch(t.Context(), req(sum(kernel), dst, maxBytes), "gs://bucket/held", first.open, testStatus) + }() + <-first.reading + return first, firstDone + } + + t.Run("a download waits while the capacity is reserved", func(t *testing.T) { + f := newStagerFixtureWithCapacity(t, 100) + first, firstDone := holdCapacity(t, f) + second := newGatedOpener(kernel) + close(second.release) + secondDone := make(chan error, 1) + go func() { + secondDone <- f.stager.Fetch(t.Context(), req(sum(kernel), f.newCallerFile(t, "b"), maxBytes), "gs://bucket/b", second.open, testStatus) + }() + // The second download opens its object, finds no space, and closes + // it to wait. + select { + case <-second.closed: + case <-time.After(10 * time.Second): + t.Fatal("second download did not close its object to wait for space") + } + select { + case <-second.reading: + t.Fatal("second download started while the first held more than the remaining capacity") + default: + } + if opens, closes := second.opens.Load(), second.closes.Load(); opens != 1 || closes != 1 { + t.Errorf("waiting download opened its object %d times and closed it %d times, want 1 and 1", opens, closes) + } + close(first.release) + if err := <-firstDone; err != nil { + t.Fatalf("first Fetch = %v", err) + } + if err := <-secondDone; err != nil { + t.Fatalf("second Fetch = %v", err) + } + if opens, closes := second.opens.Load(), second.closes.Load(); opens != 2 || closes != 2 { + t.Errorf("second download opened its object %d times and closed it %d times, want 2 and 2", opens, closes) + } + f.assertStagingEmpty(t) + }) + + t.Run("downloads that fit run together", func(t *testing.T) { + for _, tc := range []struct { + name string + capacity int64 + sized bool + }{ + {"max_bytes each", 2 * maxBytes, false}, + // Each reserves its reported size, so both fit although their + // max_bytes together do not. + {"reported sizes", maxBytes + int64(len(kernel)), true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newStagerFixtureWithCapacity(t, tc.capacity) + o := newGatedOpener(kernel) + o.sized = tc.sized + done := make(chan error, 2) + for _, name := range []string{"a", "b"} { + p := f.newCallerFile(t, name) + go func() { + done <- f.stager.Fetch(t.Context(), req(sum(kernel), p, maxBytes), "gs://bucket/"+name, o.open, testStatus) + }() + } + for range 2 { + select { + case <-o.reading: + case <-time.After(10 * time.Second): + t.Fatal("two downloads within the capacity did not run together") + } + } + close(o.release) + for range 2 { + if err := <-done; err != nil { + t.Fatalf("Fetch = %v", err) + } + } + }) + } + }) + + t.Run("a missing object does not wait for space", func(t *testing.T) { + f := newStagerFixtureWithCapacity(t, maxBytes) + first, firstDone := holdCapacity(t, f) + o := &opener{err: errNotFound} + done := make(chan error, 1) + go func() { + done <- f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, maxBytes), "gs://bucket/missing", o.open, testStatus) + }() + select { + case err := <-done: + if got := status.Code(err); got != codes.NotFound { + t.Errorf("Fetch = %v, want %s", err, codes.NotFound) + } + case <-time.After(10 * time.Second): + t.Fatal("Fetch of a missing object waited for staging space") + } + close(first.release) + if err := <-firstDone; err != nil { + t.Fatalf("first Fetch = %v", err) + } + }) + + t.Run("object changed while waiting", func(t *testing.T) { + f := newStagerFixtureWithCapacity(t, maxBytes) + first, firstDone := holdCapacity(t, f) + o := &changingOpener{contents: []string{kernel, kernel + " v2"}, closed: make(chan struct{}, 2)} + done := make(chan error, 1) + go func() { + done <- f.stager.Fetch(t.Context(), req(sum(kernel), f.dst, maxBytes), "gs://bucket/vmlinux", o.open, testStatus) + }() + <-o.closed // found no space; waiting + close(first.release) + if err := <-firstDone; err != nil { + t.Fatalf("first Fetch = %v", err) + } + if err := <-done; status.Code(err) != codes.Unavailable { + t.Fatalf("Fetch = %v, want %s", err, codes.Unavailable) + } + f.assertDst(t, untouched) + f.assertStagingEmpty(t) + // Space reserved for the changed object was released. + if !f.stager.reserved.TryAcquire(maxBytes) { + t.Error("staging space is still reserved after the failed Fetch") + } + }) + + t.Run("canceled while waiting", func(t *testing.T) { + f := newStagerFixtureWithCapacity(t, maxBytes) + first, firstDone := holdCapacity(t, f) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + o := &opener{content: kernel} + err := f.stager.Fetch(ctx, req(sum(kernel), f.dst, maxBytes), "gs://bucket/vmlinux", o.open, testStatus) + if got := status.Code(err); got != codes.Canceled { + t.Fatalf("Fetch = %v, want %s", err, codes.Canceled) + } + if o.opens != 1 || o.closes != 1 { + t.Errorf("Fetch opened the asset %d times and closed it %d times, want 1 and 1", o.opens, o.closes) + } + f.assertDst(t, untouched) + close(first.release) + if err := <-firstDone; err != nil { + t.Fatalf("first Fetch = %v", err) + } + }) + + t.Run("max_bytes above the capacity", func(t *testing.T) { + const capacity = 16 + for _, tc := range []struct { + name string + content string + want codes.Code + }{ + {"asset fits", strings.Repeat("k", capacity), codes.OK}, + {"asset larger than the capacity", strings.Repeat("k", capacity+10), codes.FailedPrecondition}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newStagerFixtureWithCapacity(t, capacity) + o := &stagingWatcher{content: tc.content, staging: f.staging} + err := f.stager.Fetch(t.Context(), req(sum(tc.content), f.dst, 1<<20), "gs://bucket/vmlinux", o.open, testStatus) + if got := status.Code(err); got != tc.want { + t.Fatalf("Fetch = %v, want %s", err, tc.want) + } + if o.maxStaged > capacity { + t.Errorf("staging directory held %d bytes, more than the capacity of %d", o.maxStaged, capacity) + } + if tc.want == codes.OK { + f.assertDst(t, tc.content) + } else { + f.assertDst(t, untouched) + if !strings.Contains(err.Error(), "staging capacity") { + t.Errorf("Fetch error %q does not name the staging capacity", err) + } + } + f.assertStagingEmpty(t) + }) + } + }) +} + +// stagingWatcher serves content a byte at a time and records the most bytes +// the staging directory held before each read. +type stagingWatcher struct { + content string + staging string + read int + maxStaged int64 +} + +func (w *stagingWatcher) open(context.Context) (io.ReadCloser, int64, error) { + return io.NopCloser(w), -1, nil +} + +// changingOpener serves contents[i] with its size on the i-th open, and the +// last one after that, as if the object were replaced between opens. It +// reports each Close on closed. +type changingOpener struct { + contents []string + opens atomic.Int64 + closed chan struct{} +} + +func (o *changingOpener) open(context.Context) (io.ReadCloser, int64, error) { + i := min(int(o.opens.Add(1))-1, len(o.contents)-1) + return ¬ifyCloser{Reader: strings.NewReader(o.contents[i]), closed: o.closed}, int64(len(o.contents[i])), nil +} + +// notifyCloser reports each Close on closed. +type notifyCloser struct { + io.Reader + closed chan<- struct{} +} + +func (n *notifyCloser) Close() error { + n.closed <- struct{}{} + return nil +} + +func (w *stagingWatcher) Read(p []byte) (int, error) { + entries, err := os.ReadDir(w.staging) + if err != nil { + return 0, err + } + var staged int64 + for _, e := range entries { + info, err := e.Info() + if err != nil { + return 0, err + } + staged += info.Size() + } + w.maxStaged = max(w.maxStaged, staged) + if w.read == len(w.content) || len(p) == 0 { + return 0, io.EOF + } + p[0] = w.content[w.read] + w.read++ + return 1, nil +} diff --git a/pkg/assetfetch/exchange_linux.go b/pkg/assetfetch/exchange_linux.go new file mode 100644 index 0000000000..7bf6404310 --- /dev/null +++ b/pkg/assetfetch/exchange_linux.go @@ -0,0 +1,37 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assetfetch + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// exchange atomically swaps the entries a and b of dir, below root. It fails +// with an error matching os.ErrNotExist if either is missing, so it never +// creates b. +func exchange(root *os.Root, dir, a, b string) error { + d, err := root.Open(dir) + if err != nil { + return err + } + defer d.Close() + fd := int(d.Fd()) + if err := unix.Renameat2(fd, a, fd, b, unix.RENAME_EXCHANGE); err != nil { + return &os.LinkError{Op: "renameat2", Old: a, New: b, Err: err} + } + return nil +} diff --git a/pkg/assetfetch/exchange_other.go b/pkg/assetfetch/exchange_other.go new file mode 100644 index 0000000000..61453b858f --- /dev/null +++ b/pkg/assetfetch/exchange_other.go @@ -0,0 +1,34 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package assetfetch + +import ( + "os" + "path/filepath" +) + +// exchange moves a over b, both entries of dir below root, if b exists. Unlike +// the Linux version it checks for b and then renames, so a b removed between +// the two is recreated. Plugins run on Linux; this keeps the package building +// and testable elsewhere. +func exchange(root *os.Root, dir, a, b string) error { + target := filepath.Join(dir, b) + if _, err := root.Lstat(target); err != nil { + return err + } + return root.Rename(filepath.Join(dir, a), target) +} diff --git a/pkg/objectstorage/gcsranged.go b/pkg/objectstorage/gcsranged.go index f23d215741..63cf352301 100644 --- a/pkg/objectstorage/gcsranged.go +++ b/pkg/objectstorage/gcsranged.go @@ -50,6 +50,9 @@ type gcsReader struct { cancel context.CancelFunc } +// ObjectSize implements SizedReader. +func (r *gcsReader) ObjectSize() int64 { return r.Attrs.Size } + func (r *gcsReader) Close() error { // Cancel before closing so a blocked read cannot reopen the GCS request. r.cancel() diff --git a/pkg/objectstorage/objects.go b/pkg/objectstorage/objects.go index 8dd6a6b7f4..5f0cef3fe7 100644 --- a/pkg/objectstorage/objects.go +++ b/pkg/objectstorage/objects.go @@ -43,6 +43,27 @@ type ObjectStorage interface { PutObject(ctx context.Context, bucket, object string, reader io.Reader) error } +// SizedReader is implemented by the readers GetObject returns when the +// backend reported the object's size as it opened it. The GCS and S3 clients +// in this package return one whenever the response carries the size. +type SizedReader interface { + io.ReadCloser + // ObjectSize is the whole object's size in bytes, as the backend reported + // it, or a negative number if it is unknown. + ObjectSize() int64 +} + +// ObjectSize returns the size of the object r, a reader GetObject returned, +// reads, and false if its backend did not report one. +func ObjectSize(r io.Reader) (int64, bool) { + s, ok := r.(SizedReader) + if !ok { + return 0, false + } + n := s.ObjectSize() + return n, n >= 0 +} + func FetchFromGCS(ctx context.Context, client ObjectStorage, gsURL string) ([]byte, error) { ctx, span := tracer.Start(ctx, "fetchFromGCS") defer span.End() diff --git a/pkg/objectstorage/objectsize_test.go b/pkg/objectstorage/objectsize_test.go new file mode 100644 index 0000000000..219aeff50d --- /dev/null +++ b/pkg/objectstorage/objectsize_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package objectstorage + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "cloud.google.com/go/storage" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "google.golang.org/api/option" +) + +// s3ObjectClient serves content as every object, honoring Range headers +// unless ignoreRange is set. +func s3ObjectClient(t *testing.T, content []byte, ignoreRange bool) ObjectStorage { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rng := strings.TrimPrefix(r.Header.Get("Range"), "bytes=") + if ignoreRange || rng == "" { + w.Header().Set("Content-Length", strconv.Itoa(len(content))) + _, _ = w.Write(content) + return + } + from, to, _ := strings.Cut(rng, "-") + off, _ := strconv.Atoi(from) + end, _ := strconv.Atoi(to) + end = min(end, len(content)-1) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", off, end, len(content))) + w.Header().Set("Content-Length", strconv.Itoa(end-off+1)) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(content[off : end+1]) + })) + t.Cleanup(srv.Close) + return NewS3Client(s3.New(s3.Options{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider("ak", "sk", ""), + BaseEndpoint: aws.String(srv.URL), + UsePathStyle: true, + RetryMaxAttempts: 1, + })) +} + +func TestS3GetObjectReportsSize(t *testing.T) { + for _, tc := range []struct { + name string + size int + ignoreRange bool + }{ + {"one range", 10, false}, + {"empty object", 0, true}, + {"range ignored", 10, true}, + {"several ranges", downloadChunkSize + 1, false}, + } { + t.Run(tc.name, func(t *testing.T) { + content := bytes.Repeat([]byte("a"), tc.size) + rc, err := s3ObjectClient(t, content, tc.ignoreRange).GetObject(t.Context(), "bucket", "object") + if err != nil { + t.Fatal(err) + } + defer rc.Close() + if n, ok := ObjectSize(rc); !ok || n != int64(tc.size) { + t.Errorf("ObjectSize() = %d, %t; want %d, true", n, ok, tc.size) + } + got, err := io.ReadAll(rc) + if err != nil || !bytes.Equal(got, content) { + t.Errorf("read %d bytes, %v; want %d bytes", len(got), err, len(content)) + } + }) + } +} + +func TestGCSGetObjectReportsSize(t *testing.T) { + content := []byte("micro-vm kernel bytes") + client, err := storage.NewClient(t.Context(), option.WithHTTPClient(&http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusPartialContent, + Header: http.Header{ + "Content-Range": {fmt.Sprintf("bytes 0-%d/%d", len(content)-1, len(content))}, + }, + ContentLength: int64(len(content)), + Body: io.NopCloser(bytes.NewReader(content)), + }, nil + })})) + if err != nil { + t.Fatal(err) + } + defer client.Close() + rc, err := (&gcsClient{client: client}).GetObject(t.Context(), "bucket", "object") + if err != nil { + t.Fatal(err) + } + defer rc.Close() + if n, ok := ObjectSize(rc); !ok || n != int64(len(content)) { + t.Errorf("ObjectSize() = %d, %t; want %d, true", n, ok, len(content)) + } +} + +func TestObjectSizeUnknown(t *testing.T) { + if n, ok := ObjectSize(io.NopCloser(strings.NewReader("x"))); ok { + t.Errorf("ObjectSize of a plain reader = %d, true; want false", n) + } + if n, ok := ObjectSize(&s3Body{ReadCloser: io.NopCloser(strings.NewReader("x")), size: -1}); ok { + t.Errorf("ObjectSize of a reader reporting -1 = %d, true; want false", n) + } +} diff --git a/pkg/objectstorage/rangedget.go b/pkg/objectstorage/rangedget.go index a7837f2abc..42bde25446 100644 --- a/pkg/objectstorage/rangedget.go +++ b/pkg/objectstorage/rangedget.go @@ -45,6 +45,7 @@ type fetchRangeFunc func(ctx context.Context, i int, off, n int64, buf []byte) e // rangedReader reassembles parallel ranged reads into one ordered stream. head, if // set, is the already-open body of the first range, so the size probe is not wasted. type rangedReader struct { + size int64 cancel context.CancelFunc head io.ReadCloser ordered chan chan rangeResult @@ -65,6 +66,7 @@ type rangeResult struct { func newRangedReader(ctx context.Context, size int64, head io.ReadCloser, fetch fetchRangeFunc) *rangedReader { ctx, cancel := context.WithCancel(ctx) r := &rangedReader{ + size: size, cancel: cancel, head: head, ordered: make(chan chan rangeResult, downloadConcurrency), @@ -144,6 +146,9 @@ func (r *rangedReader) Read(p []byte) (int, error) { return n, nil } +// ObjectSize implements SizedReader. +func (r *rangedReader) ObjectSize() int64 { return r.size } + func (r *rangedReader) Close() error { r.cancel() if r.head != nil { diff --git a/pkg/objectstorage/s3ranged.go b/pkg/objectstorage/s3ranged.go index 3b346fad70..ae1b99fc20 100644 --- a/pkg/objectstorage/s3ranged.go +++ b/pkg/objectstorage/s3ranged.go @@ -50,14 +50,29 @@ func (s *s3Client) GetObject(ctx context.Context, bucket, object string) (io.Rea return nil, err } size, ok := totalFromContentRange(head.ContentRange) - if !ok || size <= downloadChunkSize { - // No Content-Range (a server that ignored the range, so this is the whole - // object) or small enough to be one request either way. - return head.Body, nil + if !ok { + // No Content-Range: a server that ignored the range, so this is the + // whole object, and Content-Length, if set, is its size. + if head.ContentLength == nil { + return head.Body, nil + } + return &s3Body{ReadCloser: head.Body, size: *head.ContentLength}, nil + } + if size <= downloadChunkSize { + return &s3Body{ReadCloser: head.Body, size: size}, nil } return newRangedReader(ctx, size, head.Body, s.fetchRange(bucket, object)), nil } +// s3Body is an object read in one request, whose size the response reported. +type s3Body struct { + io.ReadCloser + size int64 +} + +// ObjectSize implements SizedReader. +func (b *s3Body) ObjectSize() int64 { return b.size } + // fetchRange reads one range into buf. func (s *s3Client) fetchRange(bucket, object string) fetchRangeFunc { return func(ctx context.Context, _ int, off, n int64, buf []byte) error { diff --git a/pkg/proto/objectstorepb/v1/objectstore.pb.go b/pkg/proto/objectstorepb/v1/objectstore.pb.go index bc1a9d680a..83f9bb8c1a 100644 --- a/pkg/proto/objectstorepb/v1/objectstore.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore.pb.go @@ -433,6 +433,123 @@ func (*CopySnapshotResponse) Descriptor() ([]byte, []int) { return file_objectstore_proto_rawDescGZIP(), []int{7} } +// FetchAssetRequest asks the plugin to fetch one content-addressed asset. +type FetchAssetRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The object to read, e.g. gs:///kata-assets/vmlinux. Which schemes + // a plugin serves is up to the plugin. At most 2048 bytes, the SandboxConfig + // URL's limit. Masked in logged requests, like the SandboxConfig URL it + // comes from. + AssetUri string `protobuf:"bytes,1,opt,name=asset_uri,json=assetUri,proto3" json:"asset_uri,omitempty"` + // The expected SHA-256 of the object's content, as 64 lower-case hex + // characters. + Sha256 string `protobuf:"bytes,2,opt,name=sha256,proto3" json:"sha256,omitempty"` + // Full local path of an existing regular file the asset is written to. The + // plugin replaces the file, keeping its mode, only once the content has been + // verified, never creates it, and leaves it unchanged on any error. The + // caller opens it after the call: a descriptor opened before still reads + // the old content. + WritePath string `protobuf:"bytes,3,opt,name=write_path,json=writePath,proto3" json:"write_path,omitempty"` + // The largest object, in bytes, the caller accepts. Must be positive. + MaxBytes int64 `protobuf:"varint,4,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchAssetRequest) Reset() { + *x = FetchAssetRequest{} + mi := &file_objectstore_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchAssetRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchAssetRequest) ProtoMessage() {} + +func (x *FetchAssetRequest) ProtoReflect() protoreflect.Message { + mi := &file_objectstore_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchAssetRequest.ProtoReflect.Descriptor instead. +func (*FetchAssetRequest) Descriptor() ([]byte, []int) { + return file_objectstore_proto_rawDescGZIP(), []int{8} +} + +func (x *FetchAssetRequest) GetAssetUri() string { + if x != nil { + return x.AssetUri + } + return "" +} + +func (x *FetchAssetRequest) GetSha256() string { + if x != nil { + return x.Sha256 + } + return "" +} + +func (x *FetchAssetRequest) GetWritePath() string { + if x != nil { + return x.WritePath + } + return "" +} + +func (x *FetchAssetRequest) GetMaxBytes() int64 { + if x != nil { + return x.MaxBytes + } + return 0 +} + +type FetchAssetResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchAssetResponse) Reset() { + *x = FetchAssetResponse{} + mi := &file_objectstore_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchAssetResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchAssetResponse) ProtoMessage() {} + +func (x *FetchAssetResponse) ProtoReflect() protoreflect.Message { + mi := &file_objectstore_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchAssetResponse.ProtoReflect.Descriptor instead. +func (*FetchAssetResponse) Descriptor() ([]byte, []int) { + return file_objectstore_proto_rawDescGZIP(), []int{9} +} + var File_objectstore_proto protoreflect.FileDescriptor const file_objectstore_proto_rawDesc = "" + @@ -458,13 +575,23 @@ const file_objectstore_proto_rawDesc = "" + "\x13CopySnapshotRequest\x12\x17\n" + "\asrc_uri\x18\x01 \x01(\tR\x06srcUri\x12\x17\n" + "\adst_uri\x18\x02 \x01(\tR\x06dstUri\"\x16\n" + - "\x14CopySnapshotResponse2\xd1\x01\n" + + "\x14CopySnapshotResponse\"\x89\x01\n" + + "\x11FetchAssetRequest\x12 \n" + + "\tasset_uri\x18\x01 \x01(\tB\x03\x80\x01\x01R\bassetUri\x12\x16\n" + + "\x06sha256\x18\x02 \x01(\tR\x06sha256\x12\x1d\n" + + "\n" + + "write_path\x18\x03 \x01(\tR\twritePath\x12\x1b\n" + + "\tmax_bytes\x18\x04 \x01(\x03R\bmaxBytes\"\x14\n" + + "\x12FetchAssetResponse2\xd1\x01\n" + "\fNodeProvider\x12^\n" + "\rFetchSnapshot\x12$.objectstore.v1.FetchSnapshotRequest\x1a%.objectstore.v1.FetchSnapshotResponse\"\x00\x12a\n" + "\x0eUploadSnapshot\x12%.objectstore.v1.UploadSnapshotRequest\x1a&.objectstore.v1.UploadSnapshotResponse\"\x002\xd4\x01\n" + "\x0fControlProvider\x12d\n" + "\x0fCleanupSnapshot\x12&.objectstore.v1.CleanupSnapshotRequest\x1a'.objectstore.v1.CleanupSnapshotResponse\"\x00\x12[\n" + - "\fCopySnapshot\x12#.objectstore.v1.CopySnapshotRequest\x1a$.objectstore.v1.CopySnapshotResponse\"\x00BOZMgithub.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1b\x06proto3" + "\fCopySnapshot\x12#.objectstore.v1.CopySnapshotRequest\x1a$.objectstore.v1.CopySnapshotResponse\"\x002f\n" + + "\rAssetProvider\x12U\n" + + "\n" + + "FetchAsset\x12!.objectstore.v1.FetchAssetRequest\x1a\".objectstore.v1.FetchAssetResponse\"\x00BOZMgithub.com/agent-substrate/substrate/pkg/proto/objectstorepb/v1;objectstorev1b\x06proto3" var ( file_objectstore_proto_rawDescOnce sync.Once @@ -478,7 +605,7 @@ func file_objectstore_proto_rawDescGZIP() []byte { return file_objectstore_proto_rawDescData } -var file_objectstore_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_objectstore_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_objectstore_proto_goTypes = []any{ (*FetchSnapshotRequest)(nil), // 0: objectstore.v1.FetchSnapshotRequest (*FetchSnapshotResponse)(nil), // 1: objectstore.v1.FetchSnapshotResponse @@ -488,18 +615,22 @@ var file_objectstore_proto_goTypes = []any{ (*CleanupSnapshotResponse)(nil), // 5: objectstore.v1.CleanupSnapshotResponse (*CopySnapshotRequest)(nil), // 6: objectstore.v1.CopySnapshotRequest (*CopySnapshotResponse)(nil), // 7: objectstore.v1.CopySnapshotResponse + (*FetchAssetRequest)(nil), // 8: objectstore.v1.FetchAssetRequest + (*FetchAssetResponse)(nil), // 9: objectstore.v1.FetchAssetResponse } var file_objectstore_proto_depIdxs = []int32{ 0, // 0: objectstore.v1.NodeProvider.FetchSnapshot:input_type -> objectstore.v1.FetchSnapshotRequest 2, // 1: objectstore.v1.NodeProvider.UploadSnapshot:input_type -> objectstore.v1.UploadSnapshotRequest 4, // 2: objectstore.v1.ControlProvider.CleanupSnapshot:input_type -> objectstore.v1.CleanupSnapshotRequest 6, // 3: objectstore.v1.ControlProvider.CopySnapshot:input_type -> objectstore.v1.CopySnapshotRequest - 1, // 4: objectstore.v1.NodeProvider.FetchSnapshot:output_type -> objectstore.v1.FetchSnapshotResponse - 3, // 5: objectstore.v1.NodeProvider.UploadSnapshot:output_type -> objectstore.v1.UploadSnapshotResponse - 5, // 6: objectstore.v1.ControlProvider.CleanupSnapshot:output_type -> objectstore.v1.CleanupSnapshotResponse - 7, // 7: objectstore.v1.ControlProvider.CopySnapshot:output_type -> objectstore.v1.CopySnapshotResponse - 4, // [4:8] is the sub-list for method output_type - 0, // [0:4] is the sub-list for method input_type + 8, // 4: objectstore.v1.AssetProvider.FetchAsset:input_type -> objectstore.v1.FetchAssetRequest + 1, // 5: objectstore.v1.NodeProvider.FetchSnapshot:output_type -> objectstore.v1.FetchSnapshotResponse + 3, // 6: objectstore.v1.NodeProvider.UploadSnapshot:output_type -> objectstore.v1.UploadSnapshotResponse + 5, // 7: objectstore.v1.ControlProvider.CleanupSnapshot:output_type -> objectstore.v1.CleanupSnapshotResponse + 7, // 8: objectstore.v1.ControlProvider.CopySnapshot:output_type -> objectstore.v1.CopySnapshotResponse + 9, // 9: objectstore.v1.AssetProvider.FetchAsset:output_type -> objectstore.v1.FetchAssetResponse + 5, // [5:10] is the sub-list for method output_type + 0, // [0:5] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -516,9 +647,9 @@ func file_objectstore_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_objectstore_proto_rawDesc), len(file_objectstore_proto_rawDesc)), NumEnums: 0, - NumMessages: 8, + NumMessages: 10, NumExtensions: 0, - NumServices: 2, + NumServices: 3, }, GoTypes: file_objectstore_proto_goTypes, DependencyIndexes: file_objectstore_proto_depIdxs, diff --git a/pkg/proto/objectstorepb/v1/objectstore.proto b/pkg/proto/objectstorepb/v1/objectstore.proto index 6e20d27d6c..34ecf429cb 100644 --- a/pkg/proto/objectstorepb/v1/objectstore.proto +++ b/pkg/proto/objectstorepb/v1/objectstore.proto @@ -121,3 +121,50 @@ message CopySnapshotRequest { } message CopySnapshotResponse {} + +// AssetProvider is the plugin API an object storage platform implements on +// every node, next to NodeProvider, to serve sandbox runtime assets (the files +// a SandboxConfig names, such as a micro-VM kernel and guest image). atelet +// calls it for every asset. It reads a gs:// asset anonymously only if the +// plugin returns NOT_FOUND, INVALID_ARGUMENT, UNIMPLEMENTED or UNAVAILABLE for +// it; any other failure is final. +// +// An asset is pinned by its sha256, so the plugin hands back an object only if +// the caller already knows its content. The plugin must also refuse to read +// anything stored under a snapshot location: an asset URI never names a +// snapshot, a snapshot's file, or a tag. +service AssetProvider { + // FetchAsset writes the object at asset_uri to write_path if, and only if, + // its content hashes to sha256. Nothing is written to write_path otherwise. + // It returns PERMISSION_DENIED if asset_uri, or any prefix of it, is a + // snapshot or tag URI, before reading anything; NOT_FOUND if the object does + // not exist; and FAILED_PRECONDITION if the object is larger than max_bytes + // or than the plugin can hold while verifying it, or its sha256 does not + // match. + rpc FetchAsset(FetchAssetRequest) returns (FetchAssetResponse) {} +} + +// FetchAssetRequest asks the plugin to fetch one content-addressed asset. +message FetchAssetRequest { + // The object to read, e.g. gs:///kata-assets/vmlinux. Which schemes + // a plugin serves is up to the plugin. At most 2048 bytes, the SandboxConfig + // URL's limit. Masked in logged requests, like the SandboxConfig URL it + // comes from. + string asset_uri = 1 [ debug_redact = true ]; + + // The expected SHA-256 of the object's content, as 64 lower-case hex + // characters. + string sha256 = 2; + + // Full local path of an existing regular file the asset is written to. The + // plugin replaces the file, keeping its mode, only once the content has been + // verified, never creates it, and leaves it unchanged on any error. The + // caller opens it after the call: a descriptor opened before still reads + // the old content. + string write_path = 3; + + // The largest object, in bytes, the caller accepts. Must be positive. + int64 max_bytes = 4; +} + +message FetchAssetResponse {} diff --git a/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go b/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go index acda323a34..989147ceec 100644 --- a/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go +++ b/pkg/proto/objectstorepb/v1/objectstore_grpc.pb.go @@ -379,3 +379,143 @@ var ControlProvider_ServiceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "objectstore.proto", } + +const ( + AssetProvider_FetchAsset_FullMethodName = "/objectstore.v1.AssetProvider/FetchAsset" +) + +// AssetProviderClient is the client API for AssetProvider service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// AssetProvider is the plugin API an object storage platform implements on +// every node, next to NodeProvider, to serve sandbox runtime assets (the files +// a SandboxConfig names, such as a micro-VM kernel and guest image). atelet +// calls it for every asset. It reads a gs:// asset anonymously only if the +// plugin returns NOT_FOUND, INVALID_ARGUMENT, UNIMPLEMENTED or UNAVAILABLE for +// it; any other failure is final. +// +// An asset is pinned by its sha256, so the plugin hands back an object only if +// the caller already knows its content. The plugin must also refuse to read +// anything stored under a snapshot location: an asset URI never names a +// snapshot, a snapshot's file, or a tag. +type AssetProviderClient interface { + // FetchAsset writes the object at asset_uri to write_path if, and only if, + // its content hashes to sha256. Nothing is written to write_path otherwise. + // It returns PERMISSION_DENIED if asset_uri, or any prefix of it, is a + // snapshot or tag URI, before reading anything; NOT_FOUND if the object does + // not exist; and FAILED_PRECONDITION if the object is larger than max_bytes + // or than the plugin can hold while verifying it, or its sha256 does not + // match. + FetchAsset(ctx context.Context, in *FetchAssetRequest, opts ...grpc.CallOption) (*FetchAssetResponse, error) +} + +type assetProviderClient struct { + cc grpc.ClientConnInterface +} + +func NewAssetProviderClient(cc grpc.ClientConnInterface) AssetProviderClient { + return &assetProviderClient{cc} +} + +func (c *assetProviderClient) FetchAsset(ctx context.Context, in *FetchAssetRequest, opts ...grpc.CallOption) (*FetchAssetResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FetchAssetResponse) + err := c.cc.Invoke(ctx, AssetProvider_FetchAsset_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// AssetProviderServer is the server API for AssetProvider service. +// All implementations must embed UnimplementedAssetProviderServer +// for forward compatibility. +// +// AssetProvider is the plugin API an object storage platform implements on +// every node, next to NodeProvider, to serve sandbox runtime assets (the files +// a SandboxConfig names, such as a micro-VM kernel and guest image). atelet +// calls it for every asset. It reads a gs:// asset anonymously only if the +// plugin returns NOT_FOUND, INVALID_ARGUMENT, UNIMPLEMENTED or UNAVAILABLE for +// it; any other failure is final. +// +// An asset is pinned by its sha256, so the plugin hands back an object only if +// the caller already knows its content. The plugin must also refuse to read +// anything stored under a snapshot location: an asset URI never names a +// snapshot, a snapshot's file, or a tag. +type AssetProviderServer interface { + // FetchAsset writes the object at asset_uri to write_path if, and only if, + // its content hashes to sha256. Nothing is written to write_path otherwise. + // It returns PERMISSION_DENIED if asset_uri, or any prefix of it, is a + // snapshot or tag URI, before reading anything; NOT_FOUND if the object does + // not exist; and FAILED_PRECONDITION if the object is larger than max_bytes + // or than the plugin can hold while verifying it, or its sha256 does not + // match. + FetchAsset(context.Context, *FetchAssetRequest) (*FetchAssetResponse, error) + mustEmbedUnimplementedAssetProviderServer() +} + +// UnimplementedAssetProviderServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedAssetProviderServer struct{} + +func (UnimplementedAssetProviderServer) FetchAsset(context.Context, *FetchAssetRequest) (*FetchAssetResponse, error) { + return nil, status.Error(codes.Unimplemented, "method FetchAsset not implemented") +} +func (UnimplementedAssetProviderServer) mustEmbedUnimplementedAssetProviderServer() {} +func (UnimplementedAssetProviderServer) testEmbeddedByValue() {} + +// UnsafeAssetProviderServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to AssetProviderServer will +// result in compilation errors. +type UnsafeAssetProviderServer interface { + mustEmbedUnimplementedAssetProviderServer() +} + +func RegisterAssetProviderServer(s grpc.ServiceRegistrar, srv AssetProviderServer) { + // If the following call panics, it indicates UnimplementedAssetProviderServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&AssetProvider_ServiceDesc, srv) +} + +func _AssetProvider_FetchAsset_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FetchAssetRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AssetProviderServer).FetchAsset(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AssetProvider_FetchAsset_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AssetProviderServer).FetchAsset(ctx, req.(*FetchAssetRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// AssetProvider_ServiceDesc is the grpc.ServiceDesc for AssetProvider service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var AssetProvider_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "objectstore.v1.AssetProvider", + HandlerType: (*AssetProviderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "FetchAsset", + Handler: _AssetProvider_FetchAsset_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "objectstore.proto", +} From ab6dd77570867b8ee963f9d38269f4157d22b9f5 Mon Sep 17 00:00:00 2001 From: Taahir Ahmed Date: Mon, 12 Oct 2026 23:51:59 +0000 Subject: [PATCH 14/20] router authn/z part 1: Add simple mTLS client authentication to the ingress router (#2387) The ingress router can now be started up in two modes, controlled by the `--ingress-auth-mode` flag: `deprecated-insecure` (the default), and `static-mtls`. Both modes require you to specify the new `--ingress-client-ca-file` and `--ingress-allowed-spiffe-ids` flags. By default, the validation CA is set to the pod identity CA. In deprecated-insecure mode, the router listens on both plaintext and HTTPS ports. * If a client connects on the plaintext port, they are allowed to connect to any actor without authentication and authorization. * If a client connects on the HTTPS port, but does not send a client certificate in the TLS handshake, then they are allowed to connect to any actor without authentication and authorization. * If a client connects on the HTTPS port, and provides a client certificate, then the certificate will be checked against `--ingress-client-ca-file` and `--ingress-allowed-spiffe-ids`. In static-mtls mode, the router only listens on the HTTPS port: * Clients are required to send a client certificate, and it is always checked against `--ingress-client-ca-file` and `--ingress-allowed-spiffe-ids`. These two modes allow a smooth migration: * Land this PR, which keeps the default at deprecated-insecure. * The next PR will allow ate-setup to configure the cluster into static-mtls mode. * The next PR(s) will cut over end-to-end tests and demos so that they send a client certificate. * The final PR will remove the deprecated-insecure mode. --- AGENTS.md | 1 + cmd/atenet/internal/router/README.md | 19 +++ cmd/atenet/internal/router/cmd.go | 10 ++ cmd/atenet/internal/router/config.go | 116 +++++++++++++ cmd/atenet/internal/router/config_test.go | 191 ++++++++++++++++++++++ cmd/atenet/internal/router/dataplane.go | 22 +++ cmd/atenet/internal/router/xds.go | 91 ++++++++++- cmd/atenet/internal/router/xds_test.go | 146 +++++++++++++++++ docs/threat-model.md | 2 +- manifests/ate-install/atenet-router.yaml | 18 ++ 10 files changed, 607 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 010fb62b70..6a79db448c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,5 +103,6 @@ Keep this up to date when updating AGENTS.md. - **Workload Isolation**: The project uses `gVisor` (`runsc`) for sandboxing and security isolation of workloads on pods. - **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`cmd/ateapi/internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. +- **Ingress Authentication**: `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a SPIFFE certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids`; that list is the whole authorization policy, so any listed client may reach any Actor. The default, `deprecated-insecure`, checks a certificate only when a client presents one, and also serves plaintext listeners that accept any client. Install with it via `ate-setup --ingress-auth-mode=static-mtls` (`manifests/ate-install/components/router-static-mtls`); the allowlist lives in `manifests/ate-install/atenet-router.yaml`. For future plans for security, reference `docs/roadmap.md`. diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index fa5cda0200..9714653ea9 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -193,6 +193,25 @@ ingress dataplane, `atenet-egress` the egress gateway. `--atenet-dataplane` selects the dataplane for both Deployments. Each gateway has its own static configuration because ingress and egress scale independently. +## Ingress client authentication and authorization + +The ingress router is migrating to require clients to present a SPIFFE certificate. +* With `--ingress-auth-mode=static-mtls`, the ingress router no longer offers any + plaintext listeners, and all clients *must* present a SPIFFE certificate. +* With `--ingress-auth-mode=deprecated-insecure`, the ingress router offers both + plaintext and secure listeners. If a client presents a SPIFFE certificate, it + must be valid. + +Configure the CA that is used to validate client certificates with the +`--ingress-client-ca-file` flag, which points at a PEM file containing root +certificates. + +Right now the only "authorization" policy is to check the client's SPIFFE ID +against a static allowlist. Any ID on the allowlist may use the ingress router +to contact any actor. Richer authorization policies may be added post-GA. Use +the `--ingress-allowed-spiffe-ids` flag to set the allowlist (comma-separated +list). + ## status page Serve a `/statusz` page on port 8080. diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index fa1ba49691..b10ec9ad2d 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -70,6 +70,16 @@ func bindRouterFlags(fs *pflag.FlagSet, cfg *routerConfig) { fs.IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the router dataplane") fs.StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") + // Flags for client authentication and authorization in ingress mode. + fs.StringVar(&cfg.IngressAuth.ClientCAFile, "ingress-client-ca-file", "", "PEM bundle of the CAs allowed to sign ingress client certificates. Envoy reads the file, so it must be mounted at this path in the Envoy container as well. Required whenever a TLS ingress listener is enabled") + fs.StringSliceVar(&cfg.IngressAuth.AllowedSPIFFEIDs, "ingress-allowed-spiffe-ids", nil, "SPIFFE IDs (exact match) allowed to connect to actors through the ingress dataplane. Comma-separated or repeated. Required whenever a TLS ingress listener is enabled") + fs.StringVar( + (*string)(&cfg.IngressAuth.Mode), + "ingress-auth-mode", + string(IngressAuthDeprecatedInsecure), + "Ingress authentication and authorization mode. `deprecated-insecure` (both http and https served, client certificates optional on https) or `static-mtls` (only https served, client certificates required).", + ) + fs.StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80).") fs.StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") fs.StringVar(&cfg.UpstreamSpiffePrefix, "upstream-spiffe-prefix", "spiffe://cluster.local/", "SPIFFE URI SAN prefix (trust domain) the actor's atunnel server cert must match. Empty falls back to default SAN check against the dialed pod IP (which SPIFFE-only certs never match).") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 5ed0d901df..dc3aa927d0 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -18,6 +18,8 @@ import ( "fmt" "time" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) @@ -61,6 +63,51 @@ func (m Mode) ServesIngress() bool { return m == ModeIngress } // ServesEgress reports whether this mode answers egress CONNECTs. func (m Mode) ServesEgress() bool { return m == ModeEgress } +// IngressAuthMode selects how the ingress dataplane authenticates and +// authorizes the clients that connect to it. +type IngressAuthMode string + +const ( + // IngressAuthDeprecatedInsecure serves the plaintext ingress listeners, + // which accept every client and let any of them reach any actor, + // alongside the TLS listeners, which authenticate clients exactly as in + // IngressAuthStaticMTLS. It exists so clients can move to mTLS one at a + // time, and will be removed once they have. + IngressAuthDeprecatedInsecure IngressAuthMode = "deprecated-insecure" + // IngressAuthStaticMTLS requires every client to present a certificate + // that chains to the configured client CA bundle and carries one of the + // allowed SPIFFE IDs as a URI SAN. Envoy enforces both during the TLS + // handshake, so only the TLS listeners may be enabled. + IngressAuthStaticMTLS IngressAuthMode = "static-mtls" +) + +// ingressAuthConfig holds the client authentication settings for the ingress +// dataplane's listeners. +type ingressAuthConfig struct { + // Mode is the authentication mode. Empty means + // IngressAuthDeprecatedInsecure. + Mode IngressAuthMode + // ClientCAFile is the PEM bundle of CAs allowed to sign client + // certificates. Envoy reads it, so it must be mounted in the Envoy + // container at the same path. Required whenever a TLS listener is + // enabled. + ClientCAFile string + // AllowedSPIFFEIDs is the exact set of SPIFFE IDs allowed to connect to + // actors through the TLS listeners. Required whenever a TLS listener is + // enabled. + // + // Entries are complete SPIFFE IDs, like + // `spiffe://mytrustdomain.com/foo/bar/baz`. + AllowedSPIFFEIDs []string +} + +func (c ingressAuthConfig) mode() IngressAuthMode { + if c.Mode == "" { + return IngressAuthDeprecatedInsecure + } + return c.Mode +} + // authConfig holds the router's mTLS settings for dialing ateapi. type authConfig struct { AteapiCAFile string @@ -160,6 +207,9 @@ type routerConfig struct { Auth authConfig + // IngressAuth configures client authentication on the ingress listeners. + IngressAuth ingressAuthConfig + // RouteTimeout is Envoy's end-to-end timeout on the workload route: the // ceiling on one request from the ingress listener to the actor's response. // It bounds the actor's own handling time, not the resume that precedes it @@ -258,6 +308,9 @@ func (c routerConfig) validate() error { if err := c.ParkedRequest.Validate(); err != nil { return err } + if err := c.validateIngressAuth(); err != nil { + return err + } if c.ExtProcMaxRequests < 0 { return fmt.Errorf("--extproc-max-requests must not be negative, got %d (0 derives it from --parked-request-max)", c.ExtProcMaxRequests) @@ -281,3 +334,66 @@ func (c routerConfig) validate() error { } return nil } + +// tlsIngressEnabled reports whether the Envoy dataplane builds at least one +// TLS ingress listener. +func (c routerConfig) tlsIngressEnabled() bool { + return c.EnvoyCertPath != "" && (c.HttpsPort > 0 || c.ConnectTLSPort > 0) +} + +// validateIngressAuth rejects a configuration that would leave a TLS listener +// without client authentication, a static-mtls configuration that would leave +// any path to an actor unauthenticated, and one the dataplane cannot enforce. +func (c routerConfig) validateIngressAuth() error { + mode := c.IngressAuth.mode() + switch mode { + case IngressAuthDeprecatedInsecure, IngressAuthStaticMTLS: + default: + return fmt.Errorf("--ingress-auth-mode must be %q or %q, got %q", IngressAuthDeprecatedInsecure, IngressAuthStaticMTLS, c.IngressAuth.Mode) + } + // The egress gateway never builds the ingress listeners. + if !c.Mode.ServesIngress() { + return nil + } + if mode == IngressAuthDeprecatedInsecure { + // Agentgateway's listeners come from its static configuration, so + // the router has none to authenticate. + if c.atenetRouter() != atenetRouterEnvoy || !c.tlsIngressEnabled() { + return nil + } + return c.validateIngressClientAuth() + } + if c.atenetRouter() != atenetRouterEnvoy { + return fmt.Errorf("--ingress-auth-mode=%s requires --atenet-dataplane=%s: the %s dataplane is configured statically, not by the router", IngressAuthStaticMTLS, atenetRouterEnvoy, c.atenetRouter()) + } + if c.HttpPort > 0 { + return fmt.Errorf("--ingress-auth-mode=%s requires --port-http=0: the plaintext listener cannot authenticate clients", IngressAuthStaticMTLS) + } + if c.ConnectPlainTextPort > 0 { + return fmt.Errorf("--ingress-auth-mode=%s requires --port-connect=0: the plaintext listener cannot authenticate clients", IngressAuthStaticMTLS) + } + if c.EnvoyCertPath == "" { + return fmt.Errorf("--ingress-auth-mode=%s requires --envoy-cert-path: without a serving certificate there is no TLS listener", IngressAuthStaticMTLS) + } + if c.HttpsPort <= 0 && c.ConnectTLSPort <= 0 { + return fmt.Errorf("--ingress-auth-mode=%s requires --port-https or --port-connect-tls: there is no TLS listener", IngressAuthStaticMTLS) + } + return c.validateIngressClientAuth() +} + +// validateIngressClientAuth checks the settings the TLS listeners authenticate +// clients with. +func (c routerConfig) validateIngressClientAuth() error { + if c.IngressAuth.ClientCAFile == "" { + return fmt.Errorf("TLS ingress listeners (--port-https, --port-connect-tls) require --ingress-client-ca-file") + } + if len(c.IngressAuth.AllowedSPIFFEIDs) == 0 { + return fmt.Errorf("TLS ingress listeners (--port-https, --port-connect-tls) require at least one --ingress-allowed-spiffe-ids entry") + } + for _, id := range c.IngressAuth.AllowedSPIFFEIDs { + if _, err := spiffeid.FromString(id); err != nil { + return fmt.Errorf("--ingress-allowed-spiffe-ids: %q is not a valid SPIFFE ID: %w", id, err) + } + } + return nil +} diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index 4754cae1fd..679e9be8f3 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -129,6 +129,197 @@ func TestRouterConfigValidate(t *testing.T) { } } +// staticMTLSConfig returns a valid static-mtls ingress configuration for +// TestRouterConfigValidateIngressAuth to break one field at a time. +func staticMTLSConfig() routerConfig { + return routerConfig{ + Mode: ModeIngress, + ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, + HttpsPort: 8443, + ConnectTLSPort: 8444, + EnvoyCertPath: "/run/servicedns.podcert.ate.dev/credential-bundle.pem", + IngressAuth: ingressAuthConfig{ + Mode: IngressAuthStaticMTLS, + ClientCAFile: "/run/podidentity.podcert.ate.dev/trust-bundle.pem", + AllowedSPIFFEIDs: []string{"spiffe://cluster.local/ns/demo/sa/client"}, + }, + } +} + +func TestRouterConfigValidateIngressAuth(t *testing.T) { + tests := []struct { + name string + mutate func(*routerConfig) + wantErr string // substring; empty means valid + }{ + { + name: "static-mtls with only TLS listeners accepted", + mutate: func(*routerConfig) {}, + }, + { + name: "static-mtls with only the HTTPS listener accepted", + mutate: func(c *routerConfig) { c.ConnectTLSPort = 0 }, + }, + { + name: "static-mtls with only the CONNECT-TLS listener accepted", + mutate: func(c *routerConfig) { c.HttpsPort = 0 }, + }, + { + name: "unknown mode rejected", + mutate: func(c *routerConfig) { c.IngressAuth.Mode = "mtls" }, + wantErr: "--ingress-auth-mode must be", + }, + { + name: "old insecure mode name rejected", + mutate: func(c *routerConfig) { c.IngressAuth.Mode = "insecure" }, + wantErr: "--ingress-auth-mode must be", + }, + { + name: "egress-only instance ignores the ingress settings", + mutate: func(c *routerConfig) { + c.Mode = ModeEgress + c.HttpPort = 8080 + c.IngressAuth.ClientCAFile = "" + c.IngressAuth.AllowedSPIFFEIDs = nil + }, + }, + { + name: "deprecated-insecure accepts plaintext listeners alongside mTLS ones", + mutate: func(c *routerConfig) { + c.IngressAuth.Mode = IngressAuthDeprecatedInsecure + c.HttpPort = 8080 + c.ConnectPlainTextPort = 8081 + }, + }, + { + name: "empty mode means deprecated-insecure", + mutate: func(c *routerConfig) { + c.IngressAuth.Mode = "" + c.HttpPort = 8080 + }, + }, + { + name: "deprecated-insecure with only plaintext listeners needs no client auth", + mutate: func(c *routerConfig) { + c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure, AllowedSPIFFEIDs: []string{"not a spiffe id"}} + c.HttpPort = 8080 + c.HttpsPort = 0 + c.ConnectTLSPort = 0 + }, + }, + { + name: "deprecated-insecure without a serving cert needs no client auth", + mutate: func(c *routerConfig) { + c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure} + c.HttpPort = 8080 + c.EnvoyCertPath = "" + }, + }, + { + name: "deprecated-insecure on agentgateway needs no client auth", + mutate: func(c *routerConfig) { + c.IngressAuth = ingressAuthConfig{Mode: IngressAuthDeprecatedInsecure} + c.AtenetRouter = string(atenetRouterAgentgateway) + c.HttpPort = 8080 + }, + }, + { + name: "deprecated-insecure TLS listener without client CA rejected", + mutate: func(c *routerConfig) { + c.IngressAuth.Mode = IngressAuthDeprecatedInsecure + c.IngressAuth.ClientCAFile = "" + c.HttpPort = 8080 + }, + wantErr: "require --ingress-client-ca-file", + }, + { + name: "deprecated-insecure TLS listener without allowlist rejected", + mutate: func(c *routerConfig) { + c.IngressAuth.Mode = IngressAuthDeprecatedInsecure + c.IngressAuth.AllowedSPIFFEIDs = nil + c.HttpPort = 8080 + }, + wantErr: "require at least one --ingress-allowed-spiffe-ids", + }, + { + name: "deprecated-insecure non-SPIFFE ID rejected", + mutate: func(c *routerConfig) { + c.IngressAuth.Mode = IngressAuthDeprecatedInsecure + c.IngressAuth.AllowedSPIFFEIDs = []string{"https://cluster.local/ns/demo/sa/client"} + }, + wantErr: "is not a valid SPIFFE ID", + }, + { + name: "agentgateway rejected", + mutate: func(c *routerConfig) { c.AtenetRouter = string(atenetRouterAgentgateway) }, + wantErr: "requires --atenet-dataplane=envoy", + }, + { + name: "plaintext HTTP listener rejected", + mutate: func(c *routerConfig) { c.HttpPort = 8080 }, + wantErr: "requires --port-http=0", + }, + { + name: "plaintext CONNECT listener rejected", + mutate: func(c *routerConfig) { c.ConnectPlainTextPort = 8081 }, + wantErr: "requires --port-connect=0", + }, + { + name: "missing serving cert rejected", + mutate: func(c *routerConfig) { c.EnvoyCertPath = "" }, + wantErr: "requires --envoy-cert-path", + }, + { + name: "no TLS listener rejected", + mutate: func(c *routerConfig) { + c.HttpsPort = 0 + c.ConnectTLSPort = 0 + }, + wantErr: "requires --port-https or --port-connect-tls", + }, + { + name: "missing client CA rejected", + mutate: func(c *routerConfig) { c.IngressAuth.ClientCAFile = "" }, + wantErr: "require --ingress-client-ca-file", + }, + { + name: "empty allowlist rejected", + mutate: func(c *routerConfig) { c.IngressAuth.AllowedSPIFFEIDs = nil }, + wantErr: "require at least one --ingress-allowed-spiffe-ids", + }, + { + name: "non-SPIFFE ID rejected", + mutate: func(c *routerConfig) { + c.IngressAuth.AllowedSPIFFEIDs = append(c.IngressAuth.AllowedSPIFFEIDs, "https://cluster.local/ns/demo/sa/client") + }, + wantErr: "is not a valid SPIFFE ID", + }, + { + name: "trailing slash rejected", + mutate: func(c *routerConfig) { + c.IngressAuth.AllowedSPIFFEIDs = []string{"spiffe://cluster.local/ns/demo/sa/client/"} + }, + wantErr: "is not a valid SPIFFE ID", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := staticMTLSConfig() + tc.mutate(&cfg) + err := cfg.validate() + if tc.wantErr == "" { + if err != nil { + t.Fatalf("expected valid, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("expected error containing %q, got %v", tc.wantErr, err) + } + }) + } +} + func TestRouterConfigAtenetRouter(t *testing.T) { tests := []struct { name string diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index e56f6417a8..f2bc3e8c71 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -25,6 +25,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "github.com/agent-substrate/substrate/internal/credbundle" ) type dataplaneHealthCheck struct { @@ -77,6 +78,27 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou } xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath) + // validateIngressAuth guarantees the client auth settings whenever a TLS + // listener is enabled, so every TLS listener validates any certificate a + // client presents. Only static-mtls also refuses a client that presents + // none; deprecated-insecure lets it through, as the plaintext listeners do. + if s.cfg.tlsIngressEnabled() { + // Envoy is what reads the bundle, but it would only surface a bad + // one as a listener that never warms. Parse it here so the router + // fails startup instead. + if _, err := credbundle.ParsePool(s.cfg.IngressAuth.ClientCAFile); err != nil { + return fmt.Errorf("loading --ingress-client-ca-file: %w", err) + } + require := s.cfg.IngressAuth.mode() == IngressAuthStaticMTLS + xdsSrv.SetDownstreamClientAuth(s.cfg.IngressAuth.ClientCAFile, s.cfg.IngressAuth.AllowedSPIFFEIDs, require) + slog.InfoContext(ctx, "TLS ingress listeners validate client certificates", + slog.Bool("client_cert_required", require), + slog.String("client_ca_file", s.cfg.IngressAuth.ClientCAFile), + slog.Any("allowed_spiffe_ids", s.cfg.IngressAuth.AllowedSPIFFEIDs)) + } + if s.cfg.IngressAuth.mode() == IngressAuthDeprecatedInsecure { + slog.WarnContext(ctx, "Ingress client authentication is optional: any client that presents no certificate, or uses a plaintext listener, can reach any actor. Set --ingress-auth-mode=static-mtls to require mTLS") + } xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath, s.cfg.UpstreamSpiffePrefix) // The snapshot is a pure function of the configuration applied above, so it diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 8f0973b198..b50dbefdab 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -92,6 +92,11 @@ const ( UpstreamCertSecretName = "upstream_client_cert" UpstreamTrustSecretName = "upstream_trust_bundle" + // IngressClientTrustSecretName is the SDS secret holding the CA bundle the + // TLS ingress listeners verify client certificates against when client + // authentication is on (see SetDownstreamClientAuth). + IngressClientTrustSecretName = "ingress_client_trust_bundle" + // httpProtocolOptionsName is the well-known extension key Envoy looks for in // a cluster's typed_extension_protocol_options. It must match the message's // full proto type name exactly; a typo is silently ignored rather than @@ -209,6 +214,15 @@ type XdsServer struct { // SAN check against the dialed IP fails ("verify SAN list"). upstreamSpiffePrefix string + // Downstream (client-facing) mTLS. When downstreamClientCAPath is set, + // every TLS ingress listener requests a client certificate and rejects + // one that does not chain to it or carry one of + // downstreamAllowedSPIFFEIDs as a URI SAN. downstreamRequireClientCert + // also rejects a client that presents no certificate. + downstreamClientCAPath string + downstreamAllowedSPIFFEIDs []string + downstreamRequireClientCert bool + otlpHost string otlpPort uint32 @@ -334,6 +348,21 @@ func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string) { x.certPath = certPath } +// SetDownstreamClientAuth makes every TLS ingress listener request a client +// certificate and accept only one signed by a CA in caPath whose URI SAN +// exactly matches one of allowedSPIFFEIDs. With require, a client that +// presents no certificate is rejected too; without it, such a client is let +// through unauthenticated. Envoy enforces this during the handshake, so a +// rejected client never reaches ext_proc. It does not touch the plaintext +// listeners (see routerConfig.validateIngressAuth). +func (x *XdsServer) SetDownstreamClientAuth(caPath string, allowedSPIFFEIDs []string, require bool) { + x.mu.Lock() + defer x.mu.Unlock() + x.downstreamClientCAPath = caPath + x.downstreamAllowedSPIFFEIDs = allowedSPIFFEIDs + x.downstreamRequireClientCert = require +} + // otlpDefaultPort is the OTLP/gRPC default port, used when the collector // endpoint names no port. const otlpDefaultPort = "4317" @@ -472,8 +501,9 @@ func (x *XdsServer) UpdateSnapshot() error { } // Listeners - listeners := []types.Resource{ - x.buildListener(), + var listeners []types.Resource + if x.ingressPort > 0 { + listeners = append(listeners, x.buildListener()) } if connectEnabled { listeners = append(listeners, x.buildMainInternalListener()) @@ -491,6 +521,9 @@ func (x *XdsServer) UpdateSnapshot() error { } if needsCert { secrets = append(secrets, x.buildTlsSecret()) + if x.downstreamClientCAPath != "" { + secrets = append(secrets, x.buildIngressClientTrustSecret()) + } } // Mirrors buildUpstreamTransportSocket's gating: the actor-facing socket // exists only when the credential bundle is configured, and it references @@ -1210,7 +1243,13 @@ func (x *XdsServer) buildListener() *listenerv3.Listener { // includes whenever any TLS listener (HTTPS or CONNECT-TLS) is configured. // alpnProtocols, when non-empty, is offered during the handshake; empty // advertises nothing, leaving clients on HTTP/1.1. -func buildDownstreamTlsTransportSocket(alpnProtocols []string) *corev3.TransportSocket { +// +// With client authentication configured (see SetDownstreamClientAuth) the +// socket also requests, and optionally requires, a client certificate, which +// Envoy validates whenever one is presented. As on the upstream side, the CA +// bundle arrives over SDS so it tracks ClusterTrustBundle rotation, while the +// SPIFFE ID matchers stay inline in the default context. +func (x *XdsServer) buildDownstreamTlsTransportSocket(alpnProtocols []string) *corev3.TransportSocket { tlsConfig := &tlsv3.DownstreamTlsContext{ CommonTlsContext: &tlsv3.CommonTlsContext{ AlpnProtocols: alpnProtocols, @@ -1222,6 +1261,29 @@ func buildDownstreamTlsTransportSocket(alpnProtocols []string) *corev3.Transport }, }, } + if x.downstreamClientCAPath != "" { + tlsConfig.RequireClientCertificate = wrapperspb.Bool(x.downstreamRequireClientCert) + sanMatchers := make([]*tlsv3.SubjectAltNameMatcher, 0, len(x.downstreamAllowedSPIFFEIDs)) + for _, id := range x.downstreamAllowedSPIFFEIDs { + sanMatchers = append(sanMatchers, &tlsv3.SubjectAltNameMatcher{ + SanType: tlsv3.SubjectAltNameMatcher_URI, + Matcher: &matcherv3.StringMatcher{ + MatchPattern: &matcherv3.StringMatcher_Exact{Exact: id}, + }, + }) + } + tlsConfig.CommonTlsContext.ValidationContextType = &tlsv3.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &tlsv3.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &tlsv3.CertificateValidationContext{ + MatchTypedSubjectAltNames: sanMatchers, + }, + ValidationContextSdsSecretConfig: &tlsv3.SdsSecretConfig{ + Name: IngressClientTrustSecretName, + SdsConfig: adsConfigSource(), + }, + }, + } + } tlsConfigAny := newAny(tlsConfig) return &corev3.TransportSocket{ Name: "envoy.transport_sockets.tls", @@ -1264,7 +1326,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { }, }, }, - TransportSocket: buildDownstreamTlsTransportSocket(alpn), + TransportSocket: x.buildDownstreamTlsTransportSocket(alpn), }, }, } @@ -1333,7 +1395,7 @@ func (x *XdsServer) buildConnectTerminateTLSListener() *listenerv3.Listener { // No ALPN: CONNECT-TLS clients speak HTTP/1.1 CONNECT // today, and the HTTPS listener's h2 offer deliberately // leaves this listener alone. - TransportSocket: buildDownstreamTlsTransportSocket(nil), + TransportSocket: x.buildDownstreamTlsTransportSocket(nil), }, }, } @@ -1397,17 +1459,30 @@ func (x *XdsServer) buildUpstreamCertSecret() *tlsv3.Secret { // the CA set it read at cluster warm-up and rejects certificates issued by a // rotated CA with "unknown CA". func (x *XdsServer) buildUpstreamTrustSecret() *tlsv3.Secret { + return buildTrustSecret(UpstreamTrustSecretName, x.upstreamTrustBundlePath) +} + +// buildIngressClientTrustSecret serves the CA bundle the TLS ingress listeners +// verify client certificates against. It is watched for the same reason as +// buildUpstreamTrustSecret. +func (x *XdsServer) buildIngressClientTrustSecret() *tlsv3.Secret { + return buildTrustSecret(IngressClientTrustSecretName, x.downstreamClientCAPath) +} + +// buildTrustSecret returns the SDS secret named name, serving the CA bundle at +// path as a validation context. +func buildTrustSecret(name, path string) *tlsv3.Secret { return &tlsv3.Secret{ - Name: UpstreamTrustSecretName, + Name: name, Type: &tlsv3.Secret_ValidationContext{ ValidationContext: &tlsv3.CertificateValidationContext{ TrustedCa: &corev3.DataSource{ Specifier: &corev3.DataSource_Filename{ - Filename: x.upstreamTrustBundlePath, + Filename: path, }, }, WatchedDirectory: &corev3.WatchedDirectory{ - Path: filepath.Dir(x.upstreamTrustBundlePath), + Path: filepath.Dir(path), }, }, }, diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 3794155743..f028070623 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -1501,3 +1501,149 @@ func TestXdsServer_ALPN(t *testing.T) { t.Errorf("HTTPS listener HCM codec = %v, want AUTO so the negotiated protocol is honored", hcm.GetCodecType()) } } + +// snapshotListeners builds the snapshot and returns its listeners by name. +func snapshotListeners(t *testing.T, server *XdsServer) map[string]*listenerv3.Listener { + t.Helper() + if err := server.UpdateSnapshot(); err != nil { + t.Fatalf("UpdateSnapshot failed: %v", err) + } + res, err := server.snapshot.GetSnapshot(NodeID) + if err != nil { + t.Fatalf("Failed to get snapshot: %v", err) + } + snap, ok := res.(*cachev3.Snapshot) + if !ok { + t.Fatalf("Snapshot doesn't conform to type *cachev3.Snapshot, got %T", res) + } + listeners := make(map[string]*listenerv3.Listener) + for name, raw := range snap.GetResources(resourcev3.ListenerType) { + l, ok := raw.(*listenerv3.Listener) + if !ok { + t.Fatalf("Listener '%s' doesn't conform to type *listenerv3.Listener, got %T", name, raw) + } + listeners[name] = l + } + return listeners +} + +// downstreamTlsContext returns the DownstreamTlsContext of a listener's only +// filter chain. +func downstreamTlsContext(t *testing.T, l *listenerv3.Listener) *tlsv3.DownstreamTlsContext { + t.Helper() + chains := l.GetFilterChains() + if len(chains) != 1 { + t.Fatalf("Listener '%s': expected 1 filter chain, got %d", l.GetName(), len(chains)) + } + dtc := &tlsv3.DownstreamTlsContext{} + if err := chains[0].GetTransportSocket().GetTypedConfig().UnmarshalTo(dtc); err != nil { + t.Fatalf("Listener '%s': failed to unmarshal DownstreamTlsContext: %v", l.GetName(), err) + } + return dtc +} + +func TestXdsServer_UpdateSnapshot_DownstreamClientAuth(t *testing.T) { + const ( + certPath = "/run/servicedns.podcert.ate.dev/credential-bundle.pem" + caPath = "/run/podidentity.podcert.ate.dev/trust-bundle.pem" + bundleDir = "/run/podidentity.podcert.ate.dev" + ) + allowed := []string{ + "spiffe://cluster.local/ns/demo/sa/client", + "spiffe://cluster.local/ns/other/sa/client", + } + tlsListeners := []string{IngressHTTPSListener, "connect_terminate_tls"} + + for _, tc := range []struct { + name string + require bool + }{ + {name: "Required", require: true}, + {name: "RequestedNotRequired", require: false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := NewXdsServer(18000) + server.SetConfig(0, 50053, "127.0.0.1") + server.SetConnectPorts(0, 8444) + server.SetTlsConfig(8443, certPath) + server.SetDownstreamClientAuth(caPath, allowed, tc.require) + + listeners := snapshotListeners(t, server) + for _, plaintext := range []string{IngressHTTPListener, "connect_terminate"} { + if _, exists := listeners[plaintext]; exists { + t.Errorf("Plaintext listener '%s' must not be built when its port is 0", plaintext) + } + } + for _, name := range tlsListeners { + l, exists := listeners[name] + if !exists { + t.Fatalf("TLS listener '%s' is missing from snapshot listeners", name) + } + dtc := downstreamTlsContext(t, l) + if got := dtc.GetRequireClientCertificate().GetValue(); got != tc.require { + t.Errorf("Listener '%s': require_client_certificate = %v, want %v", name, got, tc.require) + } + combined := dtc.GetCommonTlsContext().GetCombinedValidationContext() + if combined == nil { + t.Fatalf("Listener '%s': expected a combined validation context", name) + } + if got := combined.GetValidationContextSdsSecretConfig().GetName(); got != IngressClientTrustSecretName { + t.Errorf("Listener '%s': validation SDS secret = '%s', want '%s'", name, got, IngressClientTrustSecretName) + } + if combined.GetValidationContextSdsSecretConfig().GetSdsConfig().GetAds() == nil { + t.Errorf("Listener '%s': expected the validation SDS config to use the ADS config source", name) + } + matchers := combined.GetDefaultValidationContext().GetMatchTypedSubjectAltNames() + if len(matchers) != len(allowed) { + t.Fatalf("Listener '%s': expected %d SAN matchers, got %d", name, len(allowed), len(matchers)) + } + for i, m := range matchers { + if m.GetSanType() != tlsv3.SubjectAltNameMatcher_URI { + t.Errorf("Listener '%s': matcher %d SAN type = %v, want URI", name, i, m.GetSanType()) + } + if got := m.GetMatcher().GetExact(); got != allowed[i] { + t.Errorf("Listener '%s': matcher %d exact = '%s', want '%s'", name, i, got, allowed[i]) + } + } + } + + secrets := snapshotSecrets(t, server) + trust, exists := secrets[IngressClientTrustSecretName] + if !exists { + t.Fatalf("Secret '%s' is missing from snapshot secrets", IngressClientTrustSecretName) + } + validation := trust.GetValidationContext() + if got := validation.GetTrustedCa().GetFilename(); got != caPath { + t.Errorf("Expected trusted CA filename '%s', got '%s'", caPath, got) + } + if got := validation.GetWatchedDirectory().GetPath(); got != bundleDir { + t.Errorf("Expected client CA watched directory '%s', got '%s'", bundleDir, got) + } + }) + } + + t.Run("NotRequiredByDefault", func(t *testing.T) { + server := NewXdsServer(18000) + server.SetConfig(8085, 50053, "127.0.0.1") + server.SetConnectPorts(8081, 8444) + server.SetTlsConfig(8443, certPath) + + listeners := snapshotListeners(t, server) + for _, name := range tlsListeners { + l, exists := listeners[name] + if !exists { + t.Fatalf("TLS listener '%s' is missing from snapshot listeners", name) + } + dtc := downstreamTlsContext(t, l) + if dtc.GetRequireClientCertificate().GetValue() { + t.Errorf("Listener '%s': client certificate required without client auth configured", name) + } + if dtc.GetCommonTlsContext().GetValidationContextType() != nil { + t.Errorf("Listener '%s': unexpected client validation context", name) + } + } + if _, exists := snapshotSecrets(t, server)[IngressClientTrustSecretName]; exists { + t.Errorf("Secret '%s' must not be published without client auth configured", IngressClientTrustSecretName) + } + }) +} diff --git a/docs/threat-model.md b/docs/threat-model.md index 0ccda36af9..a956839504 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -40,7 +40,7 @@ Substrate is an early, fast moving product. It is full of debate and subject to * **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateom` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listener on port 443. * **Actor Routing:** Requests sent to `atenet-router` identify the target Actor with `ate-target-actor: /`. Host and authority values are application metadata and do not select the Actor. -* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads and validates `ate-target-actor`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites the routing header before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header against the Actor currently assigned to that worker. +* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. With `--ingress-auth-mode=static-mtls`, Envoy accepts connections only on its TLS listeners and only from clients presenting a certificate that chains to the configured client CA bundle and carries one of a static list of SPIFFE IDs; any listed client may reach any Actor. The default, `deprecated-insecure`, rejects a TLS client whose certificate fails those checks but accepts one that presents none, and also serves plaintext listeners that accept any client. The ext_proc reads and validates `ate-target-actor`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites the routing header before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header against the Actor currently assigned to that worker. * **Object Storage:** Used to store actor snapshots. * **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. * **Substrate Database:** PostgreSQL. On GCP, PostgreSQL may be Cloud SQL reached through the Cloud SQL Auth Proxy sidecar (see [tools/setup-gcp/cloud-sql.md](../tools/setup-gcp/cloud-sql.md)); that path adds two egress flows from ate-api-server: HTTPS to `sqladmin.googleapis.com:443` and the proxy tunnel to the instance IP on port 3307, both authenticated via IAM and encrypted. diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 800ebf2ef7..e6c3b603d1 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -223,6 +223,24 @@ spec: # CONNECT-tunneled ingress (arbitrary-port support). - "--port-connect=8081" - "--port-connect-tls=8444" + # Client auth on the TLS listeners (--port-https, --port-connect-tls). + # A client certificate must chain to the podidentity + # ClusterTrustBundle, which the envoy container mounts too, and carry + # one of the SPIFFE IDs below. The default --ingress-auth-mode, + # deprecated-insecure, only checks a certificate a client chooses to + # present, and keeps the plaintext listeners open; + # components/router-static-mtls makes the certificate mandatory. + # + # Allowed SPIFFE IDs (exact match), one per client workload: + # + # spiffe://cluster.local/ns/ate-e2e-router-client/sa/router-client + # The e2e suites' router client (see internal/e2e/ingressauth.go). + # + # Add an ID for every in-cluster workload that sends traffic to actors + # through the router. The podidentity signer issues + # spiffe://cluster.local/ns//sa/. + - "--ingress-client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + - "--ingress-allowed-spiffe-ids=spiffe://cluster.local/ns/ate-e2e-router-client/sa/router-client" # Client auth to ateapi (mtls): verify the serving cert against the # servicedns trust bundle and present the podidentity client cert. - "--ateapi-address=k8s:///api.ate-system.svc:443" From f03327bd4aa88e3aea93f26bca0752990e1ee165 Mon Sep 17 00:00:00 2001 From: Shruti Nair Date: Tue, 13 Oct 2026 00:01:35 +0000 Subject: [PATCH 15/20] ateapi: [Part 3] authorize actor lifecycle, egress policy, and worker RPCs with OpenFGA (#2185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Working on https://github.com/agent-substrate/substrate/issues/1563 Fixes https://github.com/agent-substrate/substrate/issues/1700 Adds OpenFGA checks for the remaining actor RPCs (except the identity ones) and for all worker RPCs. These checks are enforced when `--experimental-enable-authz` is set. ### Checks | RPC | Check | Who passes | |---|---|---| | `PauseActor` / `ResumeActor` / `SuspendActor` / `RevertActor` | `can_pause` / `can_resume` / `can_suspend` / `can_revert` on `actor` | atespace editor and above | | `GetActorEgressPolicy` | `can_get_egress_policy` on `actor` | atespace viewer and above | | `Create/Update/DeleteActorEgressPolicy` | `can_create/update/delete_egress_policy` on `actor` | atespace editor and above | | `CreateWorker` / `ListWorkers` | `can_create_worker` / `can_list_workers` on `global:root` | global owner / global viewer | | `GetWorker` / `ListWorkerActorAssignments` | `can_get` / `can_list_actor_assignments` on `worker` | global viewer | | `Update/Delete/DrainWorker` | `can_update` / `can_delete` / `can_drain` on `worker` | global owner | ### Not in this PR - **System component roles (follow-up PR).** With enforcement on, the ateapi callers inside Substrate need grants: - `atenet-router`: `ResumeActor`, plus `ListActors` for its health check; - `atenet-egress`: `GetActor`, `GetActorEgressPolicy`; - `ate-controller`: the worker RPCs. Today the only way to grant them is global owner or AccessPolicy roles. A follow-up will add system roles granted from ateapi's configuration to the components' mTLS SPIFFE IDs, the same way bootstrap owners are granted. Until then, don't turn on `--experimental-enable-authz` in a cluster that relies on these components. - **Identity RPCs** (`MintActorJWT`, `MintActorCertificate`) are handled separately (see #2026). ### Testing - Registry tests for each new RPC, plus fail-closed cases for missing or invalid identifiers. - Interceptor tests against Postgres-backed OpenFGA covering editors, viewers, outsiders, global viewers, global owners and bootstrap owners. Removing the worker → `global:root` contextual link fails the worker cases. - OpenFGA model tests for the new relations (`fga model test`: 6/6 tests, 185/185 checks). --- cmd/ateapi/internal/authz/authorizer.go | 31 +-- cmd/ateapi/internal/authz/authorizer_test.go | 5 + cmd/ateapi/internal/authz/authz_util.go | 46 +++-- cmd/ateapi/internal/authz/interceptor_test.go | 120 ++++++++++++ cmd/ateapi/internal/authz/model.fga | 22 ++- cmd/ateapi/internal/authz/model_test.fga.yaml | 65 ++++++- cmd/ateapi/internal/authz/registry.go | 63 +++++++ cmd/ateapi/internal/authz/registry_test.go | 177 ++++++++++++++++++ 8 files changed, 498 insertions(+), 31 deletions(-) diff --git a/cmd/ateapi/internal/authz/authorizer.go b/cmd/ateapi/internal/authz/authorizer.go index 425a4054cb..ff79c369c6 100644 --- a/cmd/ateapi/internal/authz/authorizer.go +++ b/cmd/ateapi/internal/authz/authorizer.go @@ -40,8 +40,8 @@ type Authorizer struct { } // Check verifies that the principal in ctx has relation on object. -// Structural hierarchy links (global:root as parent_global of every atespace, -// and each atespace as parent_atespace of its actors and actor templates) and +// Structural hierarchy links (global:root as parent_global of every atespace +// and worker, and each atespace as parent_atespace of its actors and actor templates) and // the caller's bootstrap owner grant, if any, are injected as OpenFGA // ContextualTuples at evaluation time rather than persisted in the tuple table. func (a *Authorizer) Check(ctx context.Context, relation, object string) error { @@ -103,19 +103,19 @@ func (a *Authorizer) checkRaw(ctx context.Context, user, relation, object string // tuples in PostgreSQL. // // Why contextual tuples are used instead of storing parent links in the database: -// 1. Deterministic structure: Every `atespace:` unconditionally has -// `global:root` as its `parent_global`, and every `actor:/` -// and `actor_template:/` has `atespace:` as its -// `parent_atespace`. Because these relationships are derived purely from the -// object type/ID, storing a row per resource in the OpenFGA `tuple` table -// would be redundant. -// 2. No write amplification on create: `CreateAtespace`, `CreateActorTemplate`, -// and `CreateActor` can insert their rows without opening an OpenFGA write -// transaction just to link the parent. +// 1. Deterministic structure: Every `atespace:` and `worker:` +// unconditionally has `global:root` as its `parent_global`, and every +// `actor:/` and `actor_template:/` has +// `atespace:` as its `parent_atespace`. Because these +// relationships are derived purely from the object type/ID, storing a row +// per resource in the OpenFGA `tuple` table would be redundant. +// 2. No write amplification on create: `CreateAtespace`, `CreateWorker`, +// `CreateActorTemplate`, and `CreateActor` can insert their rows without +// opening an OpenFGA write transaction just to link the parent. func contextualTuples(object string) []*openfgav1.TupleKey { objectType, id, _ := strings.Cut(object, ":") switch objectType { - case "atespace": + case "atespace", "worker": return []*openfgav1.TupleKey{parentGlobalTuple(object)} case "actor", "actor_template": atespace, _, ok := strings.Cut(id, "/") @@ -132,11 +132,12 @@ func contextualTuples(object string) []*openfgav1.TupleKey { return nil } -// parentGlobalTuple links an atespace object to global:root. -func parentGlobalTuple(atespaceObject string) *openfgav1.TupleKey { +// parentGlobalTuple links a global-scoped object (an atespace or a worker) to +// global:root. +func parentGlobalTuple(object string) *openfgav1.TupleKey { return &openfgav1.TupleKey{ User: GlobalRootObject, Relation: "parent_global", - Object: atespaceObject, + Object: object, } } diff --git a/cmd/ateapi/internal/authz/authorizer_test.go b/cmd/ateapi/internal/authz/authorizer_test.go index 80050c96f3..74bd57ac9f 100644 --- a/cmd/ateapi/internal/authz/authorizer_test.go +++ b/cmd/ateapi/internal/authz/authorizer_test.go @@ -45,6 +45,11 @@ func TestContextualTuples(t *testing.T) { object: AtespaceObject("team-a"), want: []*openfgav1.TupleKey{parentGlobal("atespace:team-a")}, }, + { + name: "worker", + object: WorkerObject("w-1"), + want: []*openfgav1.TupleKey{parentGlobal("worker:w-1")}, + }, { name: "actor", object: ActorObject("team-a", "runner"), diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index 0d92520635..bb4d6c51cd 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -43,20 +43,32 @@ const ( RoleEditor = "editor" RoleViewer = "viewer" - RelationCanCreateAtespace = "can_create_atespace" - RelationCanListAtespaces = "can_list_atespaces" - RelationCanCreateActor = "can_create_actor" - RelationCanListActors = "can_list_actors" - RelationCanCreateActorTemplate = "can_create_actor_template" - RelationCanListActorTemplates = "can_list_actor_templates" - RelationCanGet = "can_get" - RelationCanUseTemplate = "can_use_template" - RelationCanUpdate = "can_update" - RelationCanDelete = "can_delete" - RelationCanCreateAccessPolicy = "can_create_access_policy" - RelationCanGetAccessPolicy = "can_get_access_policy" - RelationCanUpdateAccessPolicy = "can_update_access_policy" - RelationCanDeleteAccessPolicy = "can_delete_access_policy" + RelationCanCreateAtespace = "can_create_atespace" + RelationCanListAtespaces = "can_list_atespaces" + RelationCanCreateActor = "can_create_actor" + RelationCanListActors = "can_list_actors" + RelationCanCreateActorTemplate = "can_create_actor_template" + RelationCanListActorTemplates = "can_list_actor_templates" + RelationCanCreateWorker = "can_create_worker" + RelationCanListWorkers = "can_list_workers" + RelationCanGet = "can_get" + RelationCanUseTemplate = "can_use_template" + RelationCanUpdate = "can_update" + RelationCanDelete = "can_delete" + RelationCanPause = "can_pause" + RelationCanResume = "can_resume" + RelationCanSuspend = "can_suspend" + RelationCanRevert = "can_revert" + RelationCanDrain = "can_drain" + RelationCanListActorAssignments = "can_list_actor_assignments" + RelationCanCreateAccessPolicy = "can_create_access_policy" + RelationCanGetAccessPolicy = "can_get_access_policy" + RelationCanUpdateAccessPolicy = "can_update_access_policy" + RelationCanDeleteAccessPolicy = "can_delete_access_policy" + RelationCanCreateEgressPolicy = "can_create_egress_policy" + RelationCanGetEgressPolicy = "can_get_egress_policy" + RelationCanUpdateEgressPolicy = "can_update_egress_policy" + RelationCanDeleteEgressPolicy = "can_delete_egress_policy" // maxTuplesPerWrite is OpenFGA's default maximum number of tuples allowed in a single Write request. maxTuplesPerWrite = 100 @@ -100,6 +112,12 @@ func AtespaceObject(name string) string { return "atespace:" + objectIDReplacer.Replace(name) } +// WorkerObject formats a worker name as an OpenFGA object string. Workers are +// global-scoped. +func WorkerObject(name string) string { + return "worker:" + objectIDReplacer.Replace(name) +} + // ActorObject formats an actor as an OpenFGA object string. func ActorObject(atespace, name string) string { return "actor:" + atespacedID(atespace, name) diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 9d5cc4fb4d..de80415801 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -434,3 +434,123 @@ func TestUnaryServerInterceptor_ActorAndTemplateChecks(t *testing.T) { }) } } + +func TestUnaryServerInterceptor_LifecycleEgressAndWorkerChecks(t *testing.T) { + ctx := context.Background() + pool := startPostgres(t) + fgaServer, err := NewOpenFGAServer(pool) + if err != nil { + t.Fatalf("NewOpenFGAServer failed: %v", err) + } + t.Cleanup(fgaServer.Close) + // bootstrap-owner is a global owner through configuration only; it has no + // stored tuples. + authorizer, policyManager, err := New(ctx, pool, fgaServer, []string{"bootstrap-owner"}) + if err != nil { + t.Fatalf("New failed: %v", err) + } + for _, g := range []struct{ user, role, object string }{ + {"global-owner", "owner", GlobalRootObject}, + {"global-viewer", "viewer", GlobalRootObject}, + {"editor", "editor", AtespaceObject("team-a")}, + {"viewer", "viewer", AtespaceObject("team-a")}, + {"outsider", "editor", AtespaceObject("team-b")}, + } { + writeTestTuple(t, ctx, pool, policyManager, g.user, g.role, g.object) + } + + type rpcCall struct { + fullMethod string + req any + } + actor := &ateapipb.ObjectRef{Atespace: "team-a", Name: "runner"} + worker := &ateapipb.ObjectRef{Name: "w-1"} + pause := rpcCall{ateapipb.Control_PauseActor_FullMethodName, &ateapipb.PauseActorRequest{Actor: actor}} + resume := rpcCall{ateapipb.Control_ResumeActor_FullMethodName, &ateapipb.ResumeActorRequest{Actor: actor}} + suspend := rpcCall{ateapipb.Control_SuspendActor_FullMethodName, &ateapipb.SuspendActorRequest{Actor: actor}} + revert := rpcCall{ateapipb.Control_RevertActor_FullMethodName, &ateapipb.RevertActorRequest{Actor: actor}} + getEgress := rpcCall{ateapipb.Control_GetActorEgressPolicy_FullMethodName, &ateapipb.GetActorEgressPolicyRequest{Actor: actor}} + createEgress := rpcCall{ateapipb.Control_CreateActorEgressPolicy_FullMethodName, &ateapipb.CreateActorEgressPolicyRequest{Actor: actor}} + updateEgress := rpcCall{ateapipb.Control_UpdateActorEgressPolicy_FullMethodName, &ateapipb.UpdateActorEgressPolicyRequest{Actor: actor}} + deleteEgress := rpcCall{ateapipb.Control_DeleteActorEgressPolicy_FullMethodName, &ateapipb.DeleteActorEgressPolicyRequest{Actor: actor}} + createWorker := rpcCall{ateapipb.Control_CreateWorker_FullMethodName, &ateapipb.CreateWorkerRequest{}} + listWorkers := rpcCall{ateapipb.Control_ListWorkers_FullMethodName, &ateapipb.ListWorkersRequest{}} + getWorker := rpcCall{ateapipb.Control_GetWorker_FullMethodName, &ateapipb.GetWorkerRequest{Worker: worker}} + updateWorker := rpcCall{ateapipb.Control_UpdateWorker_FullMethodName, &ateapipb.UpdateWorkerRequest{Worker: &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: worker.GetName()}, + }}} + deleteWorker := rpcCall{ateapipb.Control_DeleteWorker_FullMethodName, &ateapipb.DeleteWorkerRequest{Worker: worker}} + drainWorker := rpcCall{ateapipb.Control_DrainWorker_FullMethodName, &ateapipb.DrainWorkerRequest{Worker: worker}} + listAssignments := rpcCall{ateapipb.Control_ListWorkerActorAssignments_FullMethodName, &ateapipb.ListWorkerActorAssignmentsRequest{Worker: worker}} + + tests := []struct { + name string + user string + call rpcCall + wantCode codes.Code + }{ + // Lifecycle operations need editor on the actor's atespace. + {"editor pauses actor", "editor", pause, codes.OK}, + {"editor resumes actor", "editor", resume, codes.OK}, + {"editor suspends actor", "editor", suspend, codes.OK}, + {"editor reverts actor", "editor", revert, codes.OK}, + {"viewer cannot pause actor", "viewer", pause, codes.PermissionDenied}, + {"viewer cannot resume actor", "viewer", resume, codes.PermissionDenied}, + {"viewer cannot suspend actor", "viewer", suspend, codes.PermissionDenied}, + {"viewer cannot revert actor", "viewer", revert, codes.PermissionDenied}, + {"editor of another atespace cannot resume actor", "outsider", resume, codes.PermissionDenied}, + {"global viewer cannot suspend actor", "global-viewer", suspend, codes.PermissionDenied}, + {"global owner pauses actor", "global-owner", pause, codes.OK}, + {"bootstrap owner resumes actor", "bootstrap-owner", resume, codes.OK}, + + // Egress policies: viewers read, editors write. + {"viewer gets egress policy", "viewer", getEgress, codes.OK}, + {"global viewer gets egress policy", "global-viewer", getEgress, codes.OK}, + {"outsider cannot get egress policy", "outsider", getEgress, codes.PermissionDenied}, + {"editor creates egress policy", "editor", createEgress, codes.OK}, + {"editor updates egress policy", "editor", updateEgress, codes.OK}, + {"editor deletes egress policy", "editor", deleteEgress, codes.OK}, + {"viewer cannot create egress policy", "viewer", createEgress, codes.PermissionDenied}, + {"viewer cannot update egress policy", "viewer", updateEgress, codes.PermissionDenied}, + {"viewer cannot delete egress policy", "viewer", deleteEgress, codes.PermissionDenied}, + {"global owner updates egress policy", "global-owner", updateEgress, codes.OK}, + {"bootstrap owner deletes egress policy", "bootstrap-owner", deleteEgress, codes.OK}, + + // Workers are global-scoped: global viewers read, global owners write. + {"global viewer lists workers", "global-viewer", listWorkers, codes.OK}, + {"global viewer gets worker", "global-viewer", getWorker, codes.OK}, + {"global viewer lists worker assignments", "global-viewer", listAssignments, codes.OK}, + {"global viewer cannot create worker", "global-viewer", createWorker, codes.PermissionDenied}, + {"global viewer cannot update worker", "global-viewer", updateWorker, codes.PermissionDenied}, + {"global viewer cannot delete worker", "global-viewer", deleteWorker, codes.PermissionDenied}, + {"global viewer cannot drain worker", "global-viewer", drainWorker, codes.PermissionDenied}, + {"global owner creates worker", "global-owner", createWorker, codes.OK}, + {"global owner updates worker", "global-owner", updateWorker, codes.OK}, + {"global owner deletes worker", "global-owner", deleteWorker, codes.OK}, + {"global owner drains worker", "global-owner", drainWorker, codes.OK}, + {"bootstrap owner creates worker", "bootstrap-owner", createWorker, codes.OK}, + {"bootstrap owner drains worker", "bootstrap-owner", drainWorker, codes.OK}, + {"bootstrap owner gets worker", "bootstrap-owner", getWorker, codes.OK}, + {"atespace editor cannot list workers", "editor", listWorkers, codes.PermissionDenied}, + {"atespace editor cannot get worker", "editor", getWorker, codes.PermissionDenied}, + {"atespace editor cannot drain worker", "editor", drainWorker, codes.PermissionDenied}, + } + + interceptor := UnaryServerInterceptor(authorizer, true) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + userCtx := principal.InjectContext(ctx, principal.PrincipalInfo{ID: tc.user, Kind: principal.KindJWT}) + handlerCalled := false + _, err := interceptor(userCtx, tc.call.req, &grpc.UnaryServerInfo{FullMethod: tc.call.fullMethod}, func(context.Context, any) (any, error) { + handlerCalled = true + return "ok", nil + }) + if apierror.Code(err) != tc.wantCode { + t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", apierror.Code(err), tc.wantCode, err) + } + if wantHandler := tc.wantCode == codes.OK; handlerCalled != wantHandler { + t.Fatalf("handlerCalled = %v, want %v", handlerCalled, wantHandler) + } + }) + } +} diff --git a/cmd/ateapi/internal/authz/model.fga b/cmd/ateapi/internal/authz/model.fga index 95092357dc..deaea1e641 100644 --- a/cmd/ateapi/internal/authz/model.fga +++ b/cmd/ateapi/internal/authz/model.fga @@ -25,19 +25,32 @@ type user # Physical host node running atelet type node -# Worker pod running ateom +# Worker pod running ateom (global-scoped) type worker relations + define parent_global: [global] define host_node: [node] + define can_get: viewer from parent_global + define can_list_actor_assignments: viewer from parent_global + define can_update: owner from parent_global + define can_delete: owner from parent_global + define can_drain: owner from parent_global + # Global Scope (singleton: there will only ever be one instance, e.g. 'global:root') type global relations define owner: [user] define viewer: [user] or owner + # TODO: add system component roles (atenet-router, atenet-egress, + # ate-controller), granted from ateapi's configuration to their mTLS + # SPIFFE IDs, so they don't need global owner to call ateapi. + define can_create_atespace: owner define can_list_atespaces: viewer + define can_create_worker: owner + define can_list_workers: viewer define can_get: viewer # Listing across all atespaces (List RPCs with an empty atespace). @@ -99,6 +112,12 @@ type actor define can_update: editor from parent_atespace define can_delete: editor from parent_atespace + # Egress policy: one per actor, so its permissions live on the actor. + define can_get_egress_policy: viewer from parent_atespace + define can_create_egress_policy: editor from parent_atespace + define can_update_egress_policy: editor from parent_atespace + define can_delete_egress_policy: editor from parent_atespace + # --- Hardened System & Infrastructure Operations --- # Node Restriction: Only the node hosting the scheduled worker can mint ateom actor credentials @@ -108,5 +127,6 @@ type actor define can_resume: [user] or editor from parent_atespace # Lifecycle Operations + define can_pause: [user] or editor from parent_atespace define can_suspend: [user] or editor from parent_atespace define can_revert: [user] or editor from parent_atespace diff --git a/cmd/ateapi/internal/authz/model_test.fga.yaml b/cmd/ateapi/internal/authz/model_test.fga.yaml index 8f35493cf0..191f65bbbb 100644 --- a/cmd/ateapi/internal/authz/model_test.fga.yaml +++ b/cmd/ateapi/internal/authz/model_test.fga.yaml @@ -50,6 +50,9 @@ tuples: - user: node:node-2 relation: host_node object: worker:pod-2 + - user: global:root + relation: parent_global + object: worker:pod-1 # Actor Instance (actor:runner-1 in atespace:space-1 scheduled on worker:pod-1) - user: atespace:space-1 @@ -80,6 +83,8 @@ tests: owner: true viewer: true can_create_atespace: true + can_create_worker: true + can_list_workers: true can_list_actors: true can_list_actor_templates: true can_create_access_policy: true @@ -110,6 +115,8 @@ tests: owner: false viewer: true can_create_atespace: false + can_create_worker: false + can_list_workers: true can_list_actors: true can_list_actor_templates: true can_create_access_policy: false @@ -246,40 +253,69 @@ tests: can_get: true can_update: true can_delete: true + can_pause: true can_resume: true can_suspend: true can_revert: true + can_get_egress_policy: true + can_create_egress_policy: true + can_update_egress_policy: true + can_delete_egress_policy: true - user: user:bob # Editor object: actor:runner-1 assertions: can_get: true can_update: true can_delete: true + can_pause: true can_resume: true can_suspend: true can_revert: true + can_get_egress_policy: true + can_create_egress_policy: true + can_update_egress_policy: true + can_delete_egress_policy: true - user: user:dave # Viewer object: actor:runner-1 assertions: can_get: true can_update: false can_delete: false + can_pause: false can_resume: false can_suspend: false can_revert: false + can_get_egress_policy: true + can_create_egress_policy: false + can_update_egress_policy: false + can_delete_egress_policy: false - user: user:frank # Cluster Viewer object: actor:runner-1 assertions: can_get: true can_update: false can_delete: false + can_pause: false can_resume: false can_suspend: false can_revert: false - - user: user:sa-router # Ingress router proxy + can_get_egress_policy: true + can_create_egress_policy: false + can_update_egress_policy: false + can_delete_egress_policy: false + - user: user:eve # External + object: actor:runner-1 + assertions: + can_get: false + can_pause: false + can_get_egress_policy: false + can_update_egress_policy: false + - user: user:sa-router # Ingress router proxy can only resume object: actor:runner-1 assertions: can_resume: true + can_pause: false + can_get_egress_policy: false - user: user:sa-runner-1 # Workload identity can suspend and revert object: actor:runner-1 assertions: @@ -296,3 +332,30 @@ tests: object: actor:runner-1 assertions: can_mint_ateom_actor_credential: false + + - name: 6. Worker Operations (Inherited from Global) + check: + - user: user:carol # Global owner + object: worker:pod-1 + assertions: + can_get: true + can_list_actor_assignments: true + can_update: true + can_delete: true + can_drain: true + - user: user:frank # Global viewer + object: worker:pod-1 + assertions: + can_get: true + can_list_actor_assignments: true + can_update: false + can_delete: false + can_drain: false + - user: user:alice # Atespace owner has no worker access + object: worker:pod-1 + assertions: + can_get: false + can_list_actor_assignments: false + can_update: false + can_delete: false + can_drain: false diff --git a/cmd/ateapi/internal/authz/registry.go b/cmd/ateapi/internal/authz/registry.go index f469105e7f..edc2a331f2 100644 --- a/cmd/ateapi/internal/authz/registry.go +++ b/cmd/ateapi/internal/authz/registry.go @@ -81,6 +81,13 @@ func onAtespaceOrGlobal(relation, atespace string, fldPath *field.Path) ([]check return onAtespace(relation, atespace, fldPath) } +func onWorker(relation, name string, fldPath *field.Path) ([]check, field.ErrorList) { + if errs := validateName(name, fldPath); len(errs) > 0 { + return nil, errs + } + return []check{{relation: relation, object: WorkerObject(name)}}, nil +} + // resourceRef identifies an atespaced resource. Both *ateapipb.ObjectRef and // *ateapipb.ResourceMetadata satisfy it, including as nil pointers. type resourceRef interface { @@ -210,4 +217,60 @@ var defaultRPCPermissions = map[string]rpcRule{ ateapipb.Control_DeleteActor_FullMethodName: rule(checksOf(func(r *ateapipb.DeleteActorRequest) ([]check, field.ErrorList) { return onActor(RelationCanDelete, r.GetActor(), field.NewPath("actor")) })), + + // TODO: grant the system components that call the RPCs below + // (atenet-router: ResumeActor; atenet-egress: GetActor, + // GetActorEgressPolicy; ate-controller: worker RPCs) through system roles + // in model.fga. + + // Actor lifecycle. + ateapipb.Control_PauseActor_FullMethodName: rule(checksOf(func(r *ateapipb.PauseActorRequest) ([]check, field.ErrorList) { + return onActor(RelationCanPause, r.GetActor(), field.NewPath("actor")) + })), + ateapipb.Control_ResumeActor_FullMethodName: rule(checksOf(func(r *ateapipb.ResumeActorRequest) ([]check, field.ErrorList) { + return onActor(RelationCanResume, r.GetActor(), field.NewPath("actor")) + })), + ateapipb.Control_SuspendActor_FullMethodName: rule(checksOf(func(r *ateapipb.SuspendActorRequest) ([]check, field.ErrorList) { + return onActor(RelationCanSuspend, r.GetActor(), field.NewPath("actor")) + })), + ateapipb.Control_RevertActor_FullMethodName: rule(checksOf(func(r *ateapipb.RevertActorRequest) ([]check, field.ErrorList) { + return onActor(RelationCanRevert, r.GetActor(), field.NewPath("actor")) + })), + + // Actor egress policies. + ateapipb.Control_GetActorEgressPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.GetActorEgressPolicyRequest) ([]check, field.ErrorList) { + return onActor(RelationCanGetEgressPolicy, r.GetActor(), field.NewPath("actor")) + })), + ateapipb.Control_CreateActorEgressPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.CreateActorEgressPolicyRequest) ([]check, field.ErrorList) { + return onActor(RelationCanCreateEgressPolicy, r.GetActor(), field.NewPath("actor")) + })), + ateapipb.Control_UpdateActorEgressPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.UpdateActorEgressPolicyRequest) ([]check, field.ErrorList) { + return onActor(RelationCanUpdateEgressPolicy, r.GetActor(), field.NewPath("actor")) + })), + ateapipb.Control_DeleteActorEgressPolicy_FullMethodName: rule(checksOf(func(r *ateapipb.DeleteActorEgressPolicyRequest) ([]check, field.ErrorList) { + return onActor(RelationCanDeleteEgressPolicy, r.GetActor(), field.NewPath("actor")) + })), + + // Workers. + ateapipb.Control_CreateWorker_FullMethodName: rule(checksOf(func(*ateapipb.CreateWorkerRequest) ([]check, field.ErrorList) { + return onGlobal(RelationCanCreateWorker) + })), + ateapipb.Control_ListWorkers_FullMethodName: rule(checksOf(func(*ateapipb.ListWorkersRequest) ([]check, field.ErrorList) { + return onGlobal(RelationCanListWorkers) + })), + ateapipb.Control_GetWorker_FullMethodName: rule(checksOf(func(r *ateapipb.GetWorkerRequest) ([]check, field.ErrorList) { + return onWorker(RelationCanGet, r.GetWorker().GetName(), field.NewPath("worker", "name")) + })), + ateapipb.Control_UpdateWorker_FullMethodName: rule(checksOf(func(r *ateapipb.UpdateWorkerRequest) ([]check, field.ErrorList) { + return onWorker(RelationCanUpdate, r.GetWorker().GetMetadata().GetName(), field.NewPath("worker", "metadata", "name")) + })), + ateapipb.Control_DeleteWorker_FullMethodName: rule(checksOf(func(r *ateapipb.DeleteWorkerRequest) ([]check, field.ErrorList) { + return onWorker(RelationCanDelete, r.GetWorker().GetName(), field.NewPath("worker", "name")) + })), + ateapipb.Control_DrainWorker_FullMethodName: rule(checksOf(func(r *ateapipb.DrainWorkerRequest) ([]check, field.ErrorList) { + return onWorker(RelationCanDrain, r.GetWorker().GetName(), field.NewPath("worker", "name")) + })), + ateapipb.Control_ListWorkerActorAssignments_FullMethodName: rule(checksOf(func(r *ateapipb.ListWorkerActorAssignmentsRequest) ([]check, field.ErrorList) { + return onWorker(RelationCanListActorAssignments, r.GetWorker().GetName(), field.NewPath("worker", "name")) + })), } diff --git a/cmd/ateapi/internal/authz/registry_test.go b/cmd/ateapi/internal/authz/registry_test.go index e76d084ad3..225d916c3a 100644 --- a/cmd/ateapi/internal/authz/registry_test.go +++ b/cmd/ateapi/internal/authz/registry_test.go @@ -29,6 +29,7 @@ func TestDefaultRPCPermissions(t *testing.T) { actorMeta := &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "runner"} localTemplate := &ateapipb.ObjectRef{Atespace: "team-a", Name: "tmpl"} sharedTemplate := &ateapipb.ObjectRef{Atespace: "shared", Name: "tmpl"} + workerRef := &ateapipb.ObjectRef{Name: "w-1"} tests := []struct { name string @@ -139,6 +140,104 @@ func TestDefaultRPCPermissions(t *testing.T) { req: &ateapipb.DeleteActorRequest{Actor: actorRef}, want: []check{{RelationCanDelete, "actor:team-a/runner"}}, }, + + // Actor lifecycle. + { + name: "PauseActor", + fullMethod: ateapipb.Control_PauseActor_FullMethodName, + req: &ateapipb.PauseActorRequest{Actor: actorRef}, + want: []check{{RelationCanPause, "actor:team-a/runner"}}, + }, + { + name: "ResumeActor", + fullMethod: ateapipb.Control_ResumeActor_FullMethodName, + req: &ateapipb.ResumeActorRequest{Actor: actorRef}, + want: []check{{RelationCanResume, "actor:team-a/runner"}}, + }, + { + name: "SuspendActor", + fullMethod: ateapipb.Control_SuspendActor_FullMethodName, + req: &ateapipb.SuspendActorRequest{Actor: actorRef}, + want: []check{{RelationCanSuspend, "actor:team-a/runner"}}, + }, + { + name: "RevertActor", + fullMethod: ateapipb.Control_RevertActor_FullMethodName, + req: &ateapipb.RevertActorRequest{Actor: actorRef}, + want: []check{{RelationCanRevert, "actor:team-a/runner"}}, + }, + + // Actor egress policies. + { + name: "GetActorEgressPolicy", + fullMethod: ateapipb.Control_GetActorEgressPolicy_FullMethodName, + req: &ateapipb.GetActorEgressPolicyRequest{Actor: actorRef}, + want: []check{{RelationCanGetEgressPolicy, "actor:team-a/runner"}}, + }, + { + name: "CreateActorEgressPolicy", + fullMethod: ateapipb.Control_CreateActorEgressPolicy_FullMethodName, + req: &ateapipb.CreateActorEgressPolicyRequest{Actor: actorRef}, + want: []check{{RelationCanCreateEgressPolicy, "actor:team-a/runner"}}, + }, + { + name: "UpdateActorEgressPolicy", + fullMethod: ateapipb.Control_UpdateActorEgressPolicy_FullMethodName, + req: &ateapipb.UpdateActorEgressPolicyRequest{Actor: actorRef}, + want: []check{{RelationCanUpdateEgressPolicy, "actor:team-a/runner"}}, + }, + { + name: "DeleteActorEgressPolicy", + fullMethod: ateapipb.Control_DeleteActorEgressPolicy_FullMethodName, + req: &ateapipb.DeleteActorEgressPolicyRequest{Actor: actorRef}, + want: []check{{RelationCanDeleteEgressPolicy, "actor:team-a/runner"}}, + }, + + // Workers. + { + name: "CreateWorker", + fullMethod: ateapipb.Control_CreateWorker_FullMethodName, + req: &ateapipb.CreateWorkerRequest{}, + want: []check{{RelationCanCreateWorker, GlobalRootObject}}, + }, + { + name: "ListWorkers", + fullMethod: ateapipb.Control_ListWorkers_FullMethodName, + req: &ateapipb.ListWorkersRequest{}, + want: []check{{RelationCanListWorkers, GlobalRootObject}}, + }, + { + name: "GetWorker", + fullMethod: ateapipb.Control_GetWorker_FullMethodName, + req: &ateapipb.GetWorkerRequest{Worker: workerRef}, + want: []check{{RelationCanGet, "worker:w-1"}}, + }, + { + name: "UpdateWorker", + fullMethod: ateapipb.Control_UpdateWorker_FullMethodName, + req: &ateapipb.UpdateWorkerRequest{Worker: &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: "w-1"}, + }}, + want: []check{{RelationCanUpdate, "worker:w-1"}}, + }, + { + name: "DeleteWorker", + fullMethod: ateapipb.Control_DeleteWorker_FullMethodName, + req: &ateapipb.DeleteWorkerRequest{Worker: workerRef}, + want: []check{{RelationCanDelete, "worker:w-1"}}, + }, + { + name: "DrainWorker", + fullMethod: ateapipb.Control_DrainWorker_FullMethodName, + req: &ateapipb.DrainWorkerRequest{Worker: workerRef}, + want: []check{{RelationCanDrain, "worker:w-1"}}, + }, + { + name: "ListWorkerActorAssignments", + fullMethod: ateapipb.Control_ListWorkerActorAssignments_FullMethodName, + req: &ateapipb.ListWorkerActorAssignmentsRequest{Worker: workerRef}, + want: []check{{RelationCanListActorAssignments, "worker:w-1"}}, + }, } for _, tc := range tests { @@ -296,6 +395,84 @@ func TestDefaultRPCPermissions_MalformedIdentifierFailsClosed(t *testing.T) { req: &ateapipb.DeleteActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "team:a", Name: "runner"}}, wantField: "actor.atespace", }, + { + name: "PauseActor without an actor", + fullMethod: ateapipb.Control_PauseActor_FullMethodName, + req: &ateapipb.PauseActorRequest{}, + wantField: "actor.atespace", + }, + { + name: "ResumeActor with an invalid name", + fullMethod: ateapipb.Control_ResumeActor_FullMethodName, + req: &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "Runner"}}, + wantField: "actor.name", + }, + { + name: "SuspendActor without a name", + fullMethod: ateapipb.Control_SuspendActor_FullMethodName, + req: &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "team-a"}}, + wantField: "actor.name", + }, + { + name: "RevertActor with an invalid atespace", + fullMethod: ateapipb.Control_RevertActor_FullMethodName, + req: &ateapipb.RevertActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "team/a", Name: "runner"}}, + wantField: "actor.atespace", + }, + { + name: "GetActorEgressPolicy without an atespace", + fullMethod: ateapipb.Control_GetActorEgressPolicy_FullMethodName, + req: &ateapipb.GetActorEgressPolicyRequest{Actor: &ateapipb.ObjectRef{Name: "runner"}}, + wantField: "actor.atespace", + }, + { + name: "CreateActorEgressPolicy without an actor", + fullMethod: ateapipb.Control_CreateActorEgressPolicy_FullMethodName, + req: &ateapipb.CreateActorEgressPolicyRequest{}, + wantField: "actor.atespace", + }, + { + name: "UpdateActorEgressPolicy with an invalid name", + fullMethod: ateapipb.Control_UpdateActorEgressPolicy_FullMethodName, + req: &ateapipb.UpdateActorEgressPolicyRequest{Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "a b"}}, + wantField: "actor.name", + }, + { + name: "DeleteActorEgressPolicy without a name", + fullMethod: ateapipb.Control_DeleteActorEgressPolicy_FullMethodName, + req: &ateapipb.DeleteActorEgressPolicyRequest{Actor: &ateapipb.ObjectRef{Atespace: "team-a"}}, + wantField: "actor.name", + }, + { + name: "GetWorker without a worker", + fullMethod: ateapipb.Control_GetWorker_FullMethodName, + req: &ateapipb.GetWorkerRequest{}, + wantField: "worker.name", + }, + { + name: "UpdateWorker without metadata", + fullMethod: ateapipb.Control_UpdateWorker_FullMethodName, + req: &ateapipb.UpdateWorkerRequest{Worker: &ateapipb.Worker{}}, + wantField: "worker.metadata.name", + }, + { + name: "DeleteWorker with an invalid name", + fullMethod: ateapipb.Control_DeleteWorker_FullMethodName, + req: &ateapipb.DeleteWorkerRequest{Worker: &ateapipb.ObjectRef{Name: "w/1"}}, + wantField: "worker.name", + }, + { + name: "DrainWorker with an over-long name", + fullMethod: ateapipb.Control_DrainWorker_FullMethodName, + req: &ateapipb.DrainWorkerRequest{Worker: &ateapipb.ObjectRef{Name: strings.Repeat("w", 64)}}, + wantField: "worker.name", + }, + { + name: "ListWorkerActorAssignments without a name", + fullMethod: ateapipb.Control_ListWorkerActorAssignments_FullMethodName, + req: &ateapipb.ListWorkerActorAssignmentsRequest{Worker: &ateapipb.ObjectRef{}}, + wantField: "worker.name", + }, } for _, tc := range tests { From b8c275788e4c59212737423150520fe9b2ae4396 Mon Sep 17 00:00:00 2001 From: ILLUM1N0X Date: Tue, 13 Oct 2026 02:19:59 +0000 Subject: [PATCH 16/20] Move the OIDC JWT verifier to internal/ so binaries other than ate-api-server can use it (#2519) Part of #777 This is the first, mechanical step: later PRs add a `MintAteomActorJWT` RPC on `WorkerService`, have the snapshot plugin's control sidecar exchange that JWT for a downscoped GCS token scoped to the actor's own prefix plus its source tag, and have the node plugin transfer with that token. The design, including why downscoped tokens rather than the signed URLs the issue suggests, is in the design doc that accompanies those PRs. The snapshot plugin's control sidecar is about to verify actor JWTs before exchanging them for downscoped object-store credentials. The verifier in `oidcjwt` lived under `cmd/ateapi/internal`, where only ate-api-server can import it. This moves it to `internal/oidcjwt` so other binaries in the module can share it, keeping JWT verification on the one standard-library implementation. Pure move: the four files are unchanged (`git` records them as renames), ate-api-server's import path is updated, and the pointer in `AGENTS.md` follows the package. No behavior change. - [x] Tests pass (`go test -race ./internal/oidcjwt/... ./cmd/ateapi/...`; the `cmd/ateapi` postgres password-logging test only fails locally with `PGPASSWORD` set in the shell, on `main` as well) - [x] Appropriate changes to documentation are included in the PR (`AGENTS.md`) --- AGENTS.md | 2 +- cmd/ateapi/main.go | 2 +- {cmd/ateapi/internal => internal}/oidcjwt/http.go | 0 {cmd/ateapi/internal => internal}/oidcjwt/http_test.go | 0 {cmd/ateapi/internal => internal}/oidcjwt/oidcjwt.go | 0 {cmd/ateapi/internal => internal}/oidcjwt/oidcjwt_test.go | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename {cmd/ateapi/internal => internal}/oidcjwt/http.go (100%) rename {cmd/ateapi/internal => internal}/oidcjwt/http_test.go (100%) rename {cmd/ateapi/internal => internal}/oidcjwt/oidcjwt.go (100%) rename {cmd/ateapi/internal => internal}/oidcjwt/oidcjwt_test.go (100%) diff --git a/AGENTS.md b/AGENTS.md index 6a79db448c..1e5634bce0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,7 +102,7 @@ The following is what Substrate currently offers. Keep this up to date when updating AGENTS.md. - **Workload Isolation**: The project uses `gVisor` (`runsc`) for sandboxing and security isolation of workloads on pods. -- **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`cmd/ateapi/internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. +- **JWTs and JWKs**: Signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and JWK set publishing (`internal/oidcdiscovery`) are built on the standard library's crypto packages. Extend those packages rather than adding go-jose or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses. - **Ingress Authentication**: `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a SPIFFE certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids`; that list is the whole authorization policy, so any listed client may reach any Actor. The default, `deprecated-insecure`, checks a certificate only when a client presents one, and also serves plaintext listeners that accept any client. Install with it via `ate-setup --ingress-auth-mode=static-mtls` (`manifests/ate-install/components/router-static-mtls`); the allowlist lives in `manifests/ate-install/atenet-router.yaml`. For future plans for security, reference `docs/roadmap.md`. diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 41a1687b5c..6d342a1b59 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -30,7 +30,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/apiauthn" "github.com/agent-substrate/substrate/cmd/ateapi/internal/authz" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/oidcjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/atepg" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workerservice" @@ -40,6 +39,7 @@ import ( "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/oidcdiscovery" + "github.com/agent-substrate/substrate/internal/oidcjwt" "github.com/agent-substrate/substrate/internal/pluginsocket" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" diff --git a/cmd/ateapi/internal/oidcjwt/http.go b/internal/oidcjwt/http.go similarity index 100% rename from cmd/ateapi/internal/oidcjwt/http.go rename to internal/oidcjwt/http.go diff --git a/cmd/ateapi/internal/oidcjwt/http_test.go b/internal/oidcjwt/http_test.go similarity index 100% rename from cmd/ateapi/internal/oidcjwt/http_test.go rename to internal/oidcjwt/http_test.go diff --git a/cmd/ateapi/internal/oidcjwt/oidcjwt.go b/internal/oidcjwt/oidcjwt.go similarity index 100% rename from cmd/ateapi/internal/oidcjwt/oidcjwt.go rename to internal/oidcjwt/oidcjwt.go diff --git a/cmd/ateapi/internal/oidcjwt/oidcjwt_test.go b/internal/oidcjwt/oidcjwt_test.go similarity index 100% rename from cmd/ateapi/internal/oidcjwt/oidcjwt_test.go rename to internal/oidcjwt/oidcjwt_test.go From 2e3814a03aac70168442cf4d02e7cf5e0ac68d8e Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Tue, 13 Oct 2026 03:28:28 +0000 Subject: [PATCH 17/20] Rename the snapshot-plugin sidecar, image and flag to objectstore-plugin (#2540) The plugin now serves sandbox assets as well as snapshots, and its API is already `objectstore.v1`, so the name `snapshot-plugin` no longer describes it. This renames the binary, sidecar, sockets and flag to match. The `objectstore.v1` proto package and RPC names are unchanged. - `cmd/snapshot-plugin` moves to `cmd/objectstore-plugin`; the atelet and ate-api-server manifests and the kind overlays follow. - `--snapshot-plugin-socket` becomes `--objectstore-plugin-socket` on atelet and ate-api-server, with no alias. - Go identifiers, log and error text, `docs/architecture.md` and `hack/verify/ateapi-deps.sh` now say "object-store plugin". #### Breaking change The `cmd/snapshot-plugin` image is now `cmd/objectstore-plugin`, the `snapshot-plugin` sidecar container and volume in atelet and ate-api-server are now `objectstore-plugin`, the sockets move from `/run/snapshot-plugin` to `/run/objectstore-plugin`, and the `--snapshot-plugin-socket` flag on atelet and ate-api-server is now `--objectstore-plugin-socket`. The sidecars' trace `service.name` changes from `snapshot-plugin-` to `objectstore-plugin-`. Overlays that patch the `snapshot-plugin` container, pass the old flag or filter on the old service name must be updated. Needs the `breaking-change` label. Tested with `go build ./...`, `go test -race` on cmd/ateapi, cmd/atelet, cmd/objectstore-plugin, cmd/ate-setup, internal/objectstoreplugin and internal/pluginsocket, and `hack/verify/ateapi-deps.sh`. The base, kind, kind-agentgateway and agentgateway kustomize renders differ from before only by the rename. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- cmd/ate-setup/internal/images/images.go | 2 +- .../controlapi/actor_template_test.go | 2 +- cmd/ateapi/internal/controlapi/service.go | 10 +++--- cmd/ateapi/internal/controlapi/workflow.go | 16 ++++----- .../internal/controlapi/workflow_delete.go | 2 +- ...go => workflow_objectstore_plugin_test.go} | 6 ++-- .../internal/controlapi/workflow_revert.go | 2 +- .../internal/controlapi/workflow_suspend.go | 2 +- .../controlapi/workflow_suspend_test.go | 4 +-- .../internal/controlapi/workflow_tag.go | 4 +-- .../controlapi/workflow_testutil_test.go | 4 +-- cmd/ateapi/main.go | 18 +++++----- cmd/ateapi/main_test.go | 2 +- cmd/atelet/internal/ateletpath/ateletpath.go | 2 +- cmd/atelet/main.go | 34 +++++++++---------- cmd/atelet/sandbox_assets_plugin_test.go | 4 +-- cmd/atelet/snapshot_transfer.go | 10 +++--- cmd/atelet/snapshot_transfer_test.go | 18 +++++----- .../main.go | 14 ++++---- .../main_test.go | 4 +-- docs/architecture.md | 2 +- hack/verify/ateapi-deps.sh | 2 +- internal/objectstoreplugin/node.go | 2 +- .../objectstoreplugintest.go | 2 +- manifests/ate-install/ate-api-server.yaml | 26 +++++++------- manifests/ate-install/atelet.yaml | 31 +++++++++-------- .../kind/atelet/kustomization.yaml | 4 +-- manifests/ate-install/kind/kustomization.yaml | 4 +-- 28 files changed, 117 insertions(+), 116 deletions(-) rename cmd/ateapi/internal/controlapi/{workflow_snapshot_plugin_test.go => workflow_objectstore_plugin_test.go} (91%) rename cmd/{snapshot-plugin => objectstore-plugin}/main.go (91%) rename cmd/{snapshot-plugin => objectstore-plugin}/main_test.go (95%) diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index dd4e97f6eb..73902271e6 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -47,8 +47,8 @@ var Components = []string{ "cmd/ateom-gvisor", "cmd/ateom-microvm", "cmd/credential-provider/kubernetes-secrets", + "cmd/objectstore-plugin", "cmd/podcertcontroller", - "cmd/snapshot-plugin", "demos/counter", "demos/egress", "demos/multi-template/fspersist", diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 6e0f28385a..a9b6dbbfc3 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -249,7 +249,7 @@ func TestDeleteActorTemplate(t *testing.T) { tagRef := resources.TagRefFromTag(tag) tagURI := mustReservedTagSnapshotURI(t, tag) objects.PutSnapshot(t, tagURI, "manifest.json") - svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} + svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStorePlugin: objectstoreplugintest.ControlClient(objects)} // The handler must request AnyState to clean up an active golden actor. mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 4e5db34875..0c3ae2fc85 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -49,7 +49,7 @@ type RPCService struct { instruments *Instruments mu sync.RWMutex volumePlugins map[string]volume.VolumePluginControlPlane - snapshotPlugin objectstorev1.ControlProviderClient + objectStorePlugin objectstorev1.ControlProviderClient actorJWTIssuer string actorIDJWTPool localjwtauthority.Pool @@ -68,7 +68,7 @@ type VolumePluginRegistry interface { // // instruments may be nil; the record helpers no-op. // -// snapshotPlugin may be nil, which leaves external snapshots in place instead +// objectStorePlugin may be nil, which leaves external snapshots in place instead // of copying and releasing them. Only tests that never reach those steps pass // nil; ate-api always builds one. // @@ -83,7 +83,7 @@ func NewRPCService( instruments *Instruments, egressGatewayAddress string, volumePlugins map[string]volume.VolumePluginControlPlane, - snapshotPlugin objectstorev1.ControlProviderClient, + objectStorePlugin objectstorev1.ControlProviderClient, actorJWTIssuer string, actorIDJWTPool localjwtauthority.Pool, actorIDCAPool localca.Pool, @@ -98,12 +98,12 @@ func NewRPCService( dialer: dialer, instruments: instruments, volumePlugins: volumePlugins, - snapshotPlugin: snapshotPlugin, + objectStorePlugin: objectStorePlugin, actorJWTIssuer: actorJWTIssuer, actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, snapshotPlugin) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, objectStorePlugin) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index dbcb0bdd78..97d0b46209 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -110,12 +110,12 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry - snapshotPlugin objectstorev1.ControlProviderClient + objectStorePlugin objectstorev1.ControlProviderClient } // NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. // -// snapshotPlugin may be nil, which leaves external snapshots in place instead +// objectStorePlugin may be nil, which leaves external snapshots in place instead // of copying and releasing them. Only tests that never reach those steps pass // nil; ate-api always builds one. func NewActorWorkflow( @@ -127,7 +127,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, - snapshotPlugin objectstorev1.ControlProviderClient, + objectStorePlugin objectstorev1.ControlProviderClient, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -139,21 +139,21 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, - snapshotPlugin: snapshotPlugin, + objectStorePlugin: objectStorePlugin, } } // cleanupSnapshot deletes every object under prefix through the control -// snapshot plugin. +// object-store plugin. func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { - _, err := w.snapshotPlugin.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) + _, err := w.objectStorePlugin.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) return pluginsocket.CallError(err) } // copySnapshot copies every object under src to dst through the control -// snapshot plugin. +// object-store plugin. func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { - _, err := w.snapshotPlugin.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) + _, err := w.objectStorePlugin.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) return pluginsocket.CallError(err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index ac787425c2..90ce7d507b 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -375,7 +375,7 @@ func (w *ActorWorkflow) ensureExternalSnapshotsReleased(ctx context.Context, act ctx, done := stepSpan(ctx, "ReleaseExternalSnapshots") defer func() { err = done(err) }() - if w.snapshotPlugin == nil { + if w.objectStorePlugin == nil { markSkipped(ctx, "no object store configured") return nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go b/cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go similarity index 91% rename from cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go rename to cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go index 3344061492..35f011e889 100644 --- a/cmd/ateapi/internal/controlapi/workflow_snapshot_plugin_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_objectstore_plugin_test.go @@ -39,9 +39,9 @@ func (p failingControlPlugin) CopySnapshot(context.Context, *objectstorev1.CopyS return nil, p.err } -// A control snapshot plugin that cannot be reached fails the request with +// A control object-store plugin that cannot be reached fails the request with // Unavailable, which a client retries; other plugin errors stay Internal. -func TestSnapshotPluginErrorCodeToClient(t *testing.T) { +func TestObjectStorePluginErrorCodeToClient(t *testing.T) { prefix, err := resources.ParseStoragePrefix("gs://bucket/root/atespaces/team-a/actors/uid1/snapshots/snap1") if err != nil { t.Fatal(err) @@ -55,7 +55,7 @@ func TestSnapshotPluginErrorCodeToClient(t *testing.T) { {codes.FailedPrecondition, codes.Internal}, } { t.Run(tc.pluginCode.String(), func(t *testing.T) { - w := &ActorWorkflow{snapshotPlugin: failingControlPlugin{err: status.Error(tc.pluginCode, "plugin failed")}} + w := &ActorWorkflow{objectStorePlugin: failingControlPlugin{err: status.Error(tc.pluginCode, "plugin failed")}} for name, call := range map[string]func(context.Context) error{ "cleanup": func(ctx context.Context) error { return w.cleanupSnapshot(ctx, prefix) }, "copy": func(ctx context.Context) error { return w.copySnapshot(ctx, prefix, prefix) }, diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index 00da969f91..3675131ccd 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -202,7 +202,7 @@ func (w *ActorWorkflow) ensureInProgressSnapshotDiscarded(ctx context.Context, a inProgressSnap := findLatestDurableSnapshot(actor.GetStatus(), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) inProgress := inProgressSnap.GetDurableSnapshot().GetObject().GetSnapshotUri() switch { - case w.snapshotPlugin == nil: + case w.objectStorePlugin == nil: markSkipped(ctx, "no object store configured") return nil case inProgress == "": diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index c4cf2aabd8..99c89fea5c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -456,7 +456,7 @@ func (w *ActorWorkflow) releaseReplacedSnapshot(ctx context.Context, actor *atea defer func() { err = done(err) }() switch { - case w.snapshotPlugin == nil: + case w.objectStorePlugin == nil: markSkipped(ctx, "no object store configured") return nil case previous == "": diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 33e7125443..43f24b38e9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -457,7 +457,7 @@ func TestEnsureSuspendedFinalized_ReleasesReplacedSnapshot(t *testing.T) { var errObjectStore = errors.New("object storage is unavailable") // isObjectStoreErr reports whether err came from errObjectStore. The object -// store sits behind the snapshot plugin's gRPC boundary, which carries the +// store sits behind the object-store plugin's gRPC boundary, which carries the // message but not the error value, so errors.Is cannot match it. func isObjectStoreErr(err error) bool { return err != nil && strings.Contains(err.Error(), errObjectStore.Error()) @@ -547,7 +547,7 @@ func TestEnsureSuspendedFinalized_KeepsReplacedSnapshotOnConflict(t *testing.T) } }) - w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, snapshotPlugin: objectstoreplugintest.ControlClient(objects)} + w := &ActorWorkflow{store: &conflictingUpdateStore{Interface: persistence}, objectStorePlugin: objectstoreplugintest.ControlClient(objects)} if _, err := w.ensureSuspendedFinalized(ctx, actorRef); apierror.Code(err) != codes.Aborted { t.Fatalf("ensureSuspendedFinalized = %v, want code Aborted", err) } diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index 742fcfe74f..0311d70449 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -158,7 +158,7 @@ func (w *ActorWorkflow) ensureTagSnapshotReleased(ctx context.Context, tag *atea ctx, done := stepSpan(ctx, "ReleaseTagSnapshot") defer func() { err = done(err) }() - if w.snapshotPlugin == nil { + if w.objectStorePlugin == nil { markSkipped(ctx, "no object store configured") return nil } @@ -322,7 +322,7 @@ func (w *ActorWorkflow) ensureTagSnapshotCopied(ctx context.Context, tag *ateapi ctx, done := stepSpan(ctx, "CopyTagSnapshot") defer func() { err = done(err) }() - if w.snapshotPlugin == nil { + if w.objectStorePlugin == nil { markSkipped(ctx, "no object store configured") return nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index 2f7bbc4aa6..52971405cb 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -42,7 +42,7 @@ func testWorkerUID(podName string) string { // unit tests never reach (worker cache, atelet dialer, k8s clients) are nil, // so a step that unexpectedly executes against them fails the test loudly. // External snapshots go to an in-memory object store through the control -// snapshot plugin. +// object-store plugin. func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplName string) *ActorWorkflow { t.Helper() storetest.MustCreateAtespace(t, context.Background(), st, tmplAtespace) @@ -66,7 +66,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, snapshotPlugin: objectstoreplugintest.ControlClient(objects)}, objects + return &ActorWorkflow{store: persistence, objectStorePlugin: objectstoreplugintest.ControlClient(objects)}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 6d342a1b59..d1896c2b3b 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -96,7 +96,7 @@ var ( drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") - snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/control.sock", "Unix socket of the control snapshot plugin that external snapshots are deleted and copied through.") + objectStorePluginSocket = pflag.String("objectstore-plugin-socket", "/run/objectstore-plugin/control.sock", "Unix socket of the control object-store plugin that external snapshots are deleted and copied through.") templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) @@ -119,7 +119,7 @@ func main() { serverboot.Fatal(ctx, "Invalid PostgreSQL configuration", err) } if err := rejectStorageEnv(); err != nil { - serverboot.Fatal(ctx, "Storage settings moved to the snapshot-plugin sidecar", err) + serverboot.Fatal(ctx, "Storage settings moved to the objectstore-plugin sidecar", err) } slog.InfoContext(ctx, "ateapi starting", slog.String("version", version.Version)) if *templateResyncInterval < minResyncInterval { @@ -259,18 +259,18 @@ func main() { serverboot.Fatal(ctx, "Failed to create metric instruments", err) } - snapshotPluginConn, err := pluginsocket.Dial(*snapshotPluginSocket, pluginsocket.ReadyWait) + objectStorePluginConn, err := pluginsocket.Dial(*objectStorePluginSocket, pluginsocket.ReadyWait) if err != nil { - serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) + serverboot.Fatal(ctx, "Failed to set up the object-store plugin client", err) } - defer snapshotPluginConn.Close() + defer objectStorePluginConn.Close() // A plugin that never serves would fail every call. Fail at startup // instead, giving the sidecar time to come up. readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) - err = pluginsocket.WaitReady(readyCtx, snapshotPluginConn) + err = pluginsocket.WaitReady(readyCtx, objectStorePluginConn) cancelReady() if err != nil { - serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + serverboot.Fatal(ctx, "Object-store plugin is not serving", err) } volPlugins := make(map[string]volume.VolumePluginControlPlane) @@ -296,7 +296,7 @@ func main() { instruments, *defaultEgressGatewayAddress, volPlugins, - objectstorev1.NewControlProviderClient(snapshotPluginConn), + objectstorev1.NewControlProviderClient(objectStorePluginConn), *actorJWTIssuer, actorIDJWTAuthorityPool, actorIDCAPool, @@ -432,7 +432,7 @@ func loadFlagsFromEnv() error { // sets AWS_* in every container. func rejectStorageEnv() error { if v, ok := os.LookupEnv("ATE_STORAGE_BACKEND"); ok { - return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on ate-api-server, which no longer reads it; set the storage backend on its snapshot-plugin sidecar", v) + return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on ate-api-server, which no longer reads it; set the storage backend on its objectstore-plugin sidecar", v) } return nil } diff --git a/cmd/ateapi/main_test.go b/cmd/ateapi/main_test.go index 9c03aab650..54546caae9 100644 --- a/cmd/ateapi/main_test.go +++ b/cmd/ateapi/main_test.go @@ -326,7 +326,7 @@ func TestRejectStorageEnv(t *testing.T) { os.Setenv("ATE_STORAGE_BACKEND", *tc.value) } err := rejectStorageEnv() - if got := err != nil && strings.Contains(err.Error(), "snapshot-plugin sidecar"); got != tc.want { + if got := err != nil && strings.Contains(err.Error(), "objectstore-plugin sidecar"); got != tc.want { t.Errorf("rejectStorageEnv() = %v, want error naming the sidecar: %t", err, tc.want) } }) diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index aa80821d7d..335f05c748 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -31,7 +31,7 @@ var ( ImageCacheDir = filepath.Join(nodepath.BasePath, "image-cache") // SnapshotScratchDir holds short-lived directories for snapshot manifests - // moved through the snapshot plugin. It lives under BasePath, which the + // moved through the object-store plugin. It lives under BasePath, which the // node plugin shares with atelet. SnapshotScratchDir = filepath.Join(nodepath.BasePath, "snapshot-scratch") ) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index a829fff68a..e7a2859a92 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -114,7 +114,7 @@ var ( drainDelay = pflag.Duration("drain-delay", 0, "How long to keep accepting new RPCs after SIGTERM before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 5*time.Minute, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") - snapshotPluginSocket = pflag.String("snapshot-plugin-socket", "/run/snapshot-plugin/node.sock", "Unix socket of the node object-store plugin that external snapshots are fetched and uploaded through, and every sandbox asset is fetched through. A gs:// asset the plugin cannot serve is then read anonymously.") + objectStorePluginSocket = pflag.String("objectstore-plugin-socket", "/run/objectstore-plugin/node.sock", "Unix socket of the node object-store plugin that external snapshots are fetched and uploaded through, and every sandbox asset is fetched through. A gs:// asset the plugin cannot serve is then read anonymously.") ) var _ imagecache.CandidateKeychain = (*credentialprovider.Keychain)(nil) @@ -284,27 +284,27 @@ func main() { go trustBundles.Informer().Run(stopCh) cache.WaitForCacheSync(stopCh, trustBundles.Informer().HasSynced) - snapshotPluginConn, err := pluginsocket.Dial(*snapshotPluginSocket, pluginsocket.ReadyWait) + objectStorePluginConn, err := pluginsocket.Dial(*objectStorePluginSocket, pluginsocket.ReadyWait) if err != nil { - serverboot.Fatal(ctx, "Failed to set up the snapshot plugin client", err) + serverboot.Fatal(ctx, "Failed to set up the object-store plugin client", err) } - defer snapshotPluginConn.Close() + defer objectStorePluginConn.Close() // A plugin that never serves would fail every call. Fail at startup // instead, giving the sidecar time to come up. readyCtx, cancelReady := context.WithTimeout(ctx, time.Minute) - err = pluginsocket.WaitReady(readyCtx, snapshotPluginConn) + err = pluginsocket.WaitReady(readyCtx, objectStorePluginConn) cancelReady() if err != nil { - serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + serverboot.Fatal(ctx, "Object-store plugin is not serving", err) } wmService := NewService( ctx, ateomDialer, wrappedAnonGCS, - objectstorev1.NewNodeProviderClient(snapshotPluginConn), - objectstorev1.NewAssetProviderClient(snapshotPluginConn), - *snapshotPluginSocket, + objectstorev1.NewNodeProviderClient(objectStorePluginConn), + objectstorev1.NewAssetProviderClient(objectStorePluginConn), + *objectStorePluginSocket, imageCache, instruments, volPlugins, @@ -415,12 +415,12 @@ func ateomHerderServerOptions(creds credentials.TransportCredentials) []grpc.Ser } // rejectStorageEnv fails if ATE_STORAGE_BACKEND is set on atelet, which no -// longer reads it: object storage settings belong on the snapshot-plugin +// longer reads it: object storage settings belong on the objectstore-plugin // sidecar. Only this variable is a signal: EKS pod identity sets AWS_* in // every container. func rejectStorageEnv() error { if v, ok := os.LookupEnv("ATE_STORAGE_BACKEND"); ok { - return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on atelet, which no longer reads it; set the storage backend on its snapshot-plugin sidecar", v) + return fmt.Errorf("ATE_STORAGE_BACKEND=%q is set on atelet, which no longer reads it; set the storage backend on its objectstore-plugin sidecar", v) } return nil } @@ -475,8 +475,8 @@ type AteomHerder struct { imageCache *imagecache.Store // anonGCSClient reads gs:// assets the plugin cannot serve, without credentials. anonGCSClient objectstorage.ObjectStorage - // snapshotPlugin moves external snapshot files between the node and storage. - snapshotPlugin objectstorev1.NodeProviderClient + // objectStorePlugin moves external snapshot files between the node and storage. + objectStorePlugin objectstorev1.NodeProviderClient // assetPlugin fetches every sandbox asset first; anonGCSClient reads a gs:// // asset it cannot serve. assetPlugin objectstorev1.AssetProviderClient @@ -499,7 +499,7 @@ func NewService( ctx context.Context, ateomDialer *AteomDialer, anonGCSClient objectstorage.ObjectStorage, - snapshotPlugin objectstorev1.NodeProviderClient, + objectStorePlugin objectstorev1.NodeProviderClient, assetPlugin objectstorev1.AssetProviderClient, pluginSocket string, imageCache *imagecache.Store, @@ -512,7 +512,7 @@ func NewService( ateomDialer: ateomDialer, imageCache: imageCache, anonGCSClient: anonGCSClient, - snapshotPlugin: snapshotPlugin, + objectStorePlugin: objectStorePlugin, assetPlugin: assetPlugin, pluginSocket: pluginSocket, snapshotScratchDir: ateletpath.SnapshotScratchDir, @@ -772,7 +772,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe switch req.GetType() { case ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL: // CheckpointWorkload has already deleted the workload, so a retried - // Checkpoint cannot redo this upload. It waits for the snapshot plugin + // Checkpoint cannot redo this upload. It waits for the object-store plugin // as long as ctx allows, and a plugin error reaches the control plane as // Internal, which crashes the actor. // TODO(#362): Because we do not cache the external snapshot files when upload fails, we have to mark the Actor as CRASHED. @@ -913,7 +913,7 @@ func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletp } // uploadSnapshot uploads rec's snapshot files from srcDir to uri through the -// snapshot plugin, then the marshaled manifest. The manifest goes last, in its +// object-store plugin, then the marshaled manifest. The manifest goes last, in its // own call: its presence is the commit marker — readers assume every file it // lists is already present. A crash mid-upload thus leaves only orphaned // files, never a manifest pointing at files that never landed; retries diff --git a/cmd/atelet/sandbox_assets_plugin_test.go b/cmd/atelet/sandbox_assets_plugin_test.go index d26993e3ab..c72b25980b 100644 --- a/cmd/atelet/sandbox_assets_plugin_test.go +++ b/cmd/atelet/sandbox_assets_plugin_test.go @@ -49,7 +49,7 @@ import ( const ( testAssetBucket = "cluster-bucket" - testPluginSock = "/run/snapshot-plugin/node.sock" + testPluginSock = "/run/objectstore-plugin/node.sock" ) // bucketStorage is an in-memory object store keyed by "bucket/object" that @@ -659,7 +659,7 @@ func TestRejectAteletStorageEnv(t *testing.T) { } for _, v := range []string{"s3", "gcs", ""} { os.Setenv("ATE_STORAGE_BACKEND", v) - if err := rejectStorageEnv(); err == nil || !strings.Contains(err.Error(), "snapshot-plugin sidecar") { + if err := rejectStorageEnv(); err == nil || !strings.Contains(err.Error(), "objectstore-plugin sidecar") { t.Errorf("rejectStorageEnv() with %q = %v, want an error naming the sidecar", v, err) } } diff --git a/cmd/atelet/snapshot_transfer.go b/cmd/atelet/snapshot_transfer.go index 065b70a2ae..e106921a3e 100644 --- a/cmd/atelet/snapshot_transfer.go +++ b/cmd/atelet/snapshot_transfer.go @@ -28,10 +28,10 @@ import ( ) // External snapshots move between the node and storage only through the -// snapshot plugin. These helpers adapt the plugin's file-oriented API to the +// object-store plugin. These helpers adapt the plugin's file-oriented API to the // manifest bytes atelet works with. -// pluginCallError replaces a snapshot plugin's error with one that keeps its +// pluginCallError replaces an object-store plugin's error with one that keeps its // gRPC code and drops its message. The message may quote a signed URL or other // credential, and atelet's errors reach ate-api-server, its RPC log and the // actor's crash message, so only the code and the redacted snapshot URI are @@ -45,7 +45,7 @@ func pluginCallError(op, snapshotURI string, err error) error { if code == codes.Unknown { code = status.FromContextError(err).Code() // a context error that is not a status } - return status.Errorf(code, "snapshot plugin %s of %s failed (%s)", op, redactURLString(snapshotURI), code) + return status.Errorf(code, "object-store plugin %s of %s failed (%s)", op, redactURLString(snapshotURI), code) } // fetchSnapshotFiles downloads the named snapshot files into dstDir. @@ -53,7 +53,7 @@ func (s *AteomHerder) fetchSnapshotFiles(ctx context.Context, snapshotURI, dstDi if len(files) == 0 { return nil } - _, err := s.snapshotPlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ + _, err := s.objectStorePlugin.FetchSnapshot(ctx, &objectstorev1.FetchSnapshotRequest{ SnapshotUri: snapshotURI, WritePath: dstDir, Files: files, @@ -70,7 +70,7 @@ func (s *AteomHerder) uploadSnapshotFiles(ctx context.Context, snapshotURI, srcD if len(files) == 0 { return nil } - _, err := s.snapshotPlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ + _, err := s.objectStorePlugin.UploadSnapshot(ctx, &objectstorev1.UploadSnapshotRequest{ SnapshotUri: snapshotURI, LocalPath: srcDir, Files: files, diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 7d5ab06dde..5ef23cb368 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -54,7 +54,7 @@ func newPluginHerder(t *testing.T, store objectstorage.ObjectStorage) *AteomHerd t.Fatal(err) } return &AteomHerder{ - snapshotPlugin: objectstoreplugintest.NodeClient(plugin), + objectStorePlugin: objectstoreplugintest.NodeClient(plugin), snapshotScratchDir: t.TempDir(), } } @@ -163,12 +163,12 @@ func (p failingNodePlugin) UploadSnapshot(context.Context, *objectstorev1.Upload return nil, p.err } -// A snapshot plugin that cannot be reached reaches ate-api-server as +// An object-store plugin that cannot be reached reaches ate-api-server as // Unavailable, which it retries, rather than Internal, which crashes the // actor. The upload after CheckpointWorkload is the exception: a retried // Checkpoint cannot redo it, so it stays Internal. Other plugin errors reach // ate-api-server as Internal. -func TestSnapshotPluginErrorCodeThroughAtelet(t *testing.T) { +func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { uri, err := resources.ParseSnapshotURI(pausedSnapshotURI) if err != nil { t.Fatal(err) @@ -209,7 +209,7 @@ func TestSnapshotPluginErrorCodeThroughAtelet(t *testing.T) { want = codes.Unavailable } s := &AteomHerder{ - snapshotPlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, + objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "plugin failed")}, snapshotScratchDir: t.TempDir(), } // atelet's server interceptor decides the code ate-api-server sees. @@ -223,11 +223,11 @@ func TestSnapshotPluginErrorCodeThroughAtelet(t *testing.T) { } } -// A snapshot plugin's status text can quote a signed URL or other credential. +// An object-store plugin's status text can quote a signed URL or other credential. // Neither it nor the snapshot URI's query reaches atelet's RPC log or the // error ate-api-server receives: only the plugin's code and the snapshot URI // as scheme://host/path do, and callers still read the code. -func TestSnapshotPluginErrorKeepsCredentialsOut(t *testing.T) { +func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { const ( secret = "SECRET-SIG" signedURI = "s3://bucket/snapshots/a?sig=" + secret @@ -265,7 +265,7 @@ func TestSnapshotPluginErrorKeepsCredentialsOut(t *testing.T) { t.Run(pluginCode.String()+"/"+tr.name, func(t *testing.T) { logs := captureLogs(t) s := &AteomHerder{ - snapshotPlugin: failingNodePlugin{err: status.Error(pluginCode, "get "+signedURI+": AccountKey="+secret)}, + objectStorePlugin: failingNodePlugin{err: status.Error(pluginCode, "get "+signedURI+": AccountKey="+secret)}, snapshotScratchDir: t.TempDir(), } _, err := ateinterceptors.InternalServerUnaryInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/test"}, @@ -379,7 +379,7 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { t.Fatalf("WaitReady = %v", err) } s := &AteomHerder{ - snapshotPlugin: objectstorev1.NewNodeProviderClient(conn), + objectStorePlugin: objectstorev1.NewNodeProviderClient(conn), snapshotScratchDir: t.TempDir(), } rec := sandboxAssetsRecord{ @@ -440,7 +440,7 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { // snapshot upload uses to tell a missing snapshot from a failed probe. func TestFetchManifestKeepsNotFound(t *testing.T) { s := &AteomHerder{ - snapshotPlugin: failingNodePlugin{err: status.Error(codes.NotFound, "no manifest")}, + objectStorePlugin: failingNodePlugin{err: status.Error(codes.NotFound, "no manifest")}, snapshotScratchDir: t.TempDir(), } _, err := s.fetchManifest(context.Background(), pausedSnapshotURI) diff --git a/cmd/snapshot-plugin/main.go b/cmd/objectstore-plugin/main.go similarity index 91% rename from cmd/snapshot-plugin/main.go rename to cmd/objectstore-plugin/main.go index ff2a67ea06..c311666bee 100644 --- a/cmd/snapshot-plugin/main.go +++ b/cmd/objectstore-plugin/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Command snapshot-plugin serves the object-store plugin API on GCS or S3 over +// Command objectstore-plugin serves the object-store plugin API on GCS or S3 over // a Unix socket. It runs as a sidecar: "node" next to atelet, serving snapshot // transfers and sandbox assets, and "control" next to ate-api-server. The // backend is chosen by ATE_STORAGE_BACKEND ("s3", or GCS by default) with the @@ -46,7 +46,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) -const usage = `usage: snapshot-plugin [flags]` +const usage = `usage: objectstore-plugin [flags]` func main() { if len(os.Args) < 2 { @@ -57,7 +57,7 @@ func main() { flags := pflag.NewFlagSet(mode, pflag.ExitOnError) socket := flags.String("socket", "", "Unix socket to serve on, or to check in healthcheck mode") root := flags.String("root", nodepath.BasePath, "node mode: the only directory tree local snapshot and asset files may be read from or written to") - assetStagingDir := flags.String("asset-staging-dir", "/var/lib/snapshot-plugin/asset-staging", "node mode: directory sandbox assets are downloaded into and verified before they are copied below --root; must not be below --root or readable by the caller") + assetStagingDir := flags.String("asset-staging-dir", "/var/lib/objectstore-plugin/asset-staging", "node mode: directory sandbox assets are downloaded into and verified before they are copied below --root; must not be below --root or readable by the caller") assetStagingCapacity := flags.String("asset-staging-capacity", "8Gi", "node mode: the most sandbox asset downloads in --asset-staging-dir may hold at once, as a Kubernetes quantity; downloads wait for space, and an asset larger than this is refused") timeout := flags.Duration("timeout", 5*time.Second, "healthcheck mode: how long to wait for the plugin to report that it is serving") _ = flags.Parse(os.Args[2:]) @@ -72,13 +72,13 @@ func main() { // shell or grpc_health_probe, so the binary checks itself. if mode == "healthcheck" { if err := healthcheck(ctx, *socket, *timeout); err != nil { - serverboot.Fatal(ctx, "Snapshot plugin is not serving", err) + serverboot.Fatal(ctx, "Object-store plugin is not serving", err) } return } tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ - ServiceName: "snapshot-plugin-" + mode, + ServiceName: "objectstore-plugin-" + mode, Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), }) if err != nil { @@ -133,11 +133,11 @@ func main() { defer stop() go func() { <-sigCtx.Done() - slog.InfoContext(ctx, "Draining snapshot plugin") + slog.InfoContext(ctx, "Draining object-store plugin") srv.GracefulStop() }() - slog.InfoContext(ctx, "Serving snapshot plugin", slog.String("mode", mode), slog.String("socket", *socket)) + slog.InfoContext(ctx, "Serving object-store plugin", slog.String("mode", mode), slog.String("socket", *socket)) if err := srv.Serve(lis); err != nil { serverboot.Fatal(ctx, "Serve failed", err) } diff --git a/cmd/snapshot-plugin/main_test.go b/cmd/objectstore-plugin/main_test.go similarity index 95% rename from cmd/snapshot-plugin/main_test.go rename to cmd/objectstore-plugin/main_test.go index 2f35272cfc..07b9cd1f9b 100644 --- a/cmd/snapshot-plugin/main_test.go +++ b/cmd/objectstore-plugin/main_test.go @@ -67,7 +67,7 @@ func TestInstallStagingCapacityFitsVolume(t *testing.T) { } var capacityArg string for _, c := range ds.Spec.Template.Spec.InitContainers { - if c.Name != "snapshot-plugin" { + if c.Name != "objectstore-plugin" { continue } for _, a := range c.Args { @@ -77,7 +77,7 @@ func TestInstallStagingCapacityFitsVolume(t *testing.T) { } } if capacityArg == "" { - t.Fatal("the snapshot-plugin sidecar does not set --asset-staging-capacity") + t.Fatal("the objectstore-plugin sidecar does not set --asset-staging-capacity") } capacity, err := parseCapacity(capacityArg) if err != nil { diff --git a/docs/architecture.md b/docs/architecture.md index 3bc6e90c8a..83eeaa90f3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -327,7 +327,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov * **Lifecycle Management**: The `ateom` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for micro-VMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.) - * **Storage Mover**: External snapshots move through a *snapshot plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/snapshot-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own. + * **Storage Mover**: External snapshots move through an *object-store plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/objectstore-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own. ### Sandbox Classes diff --git a/hack/verify/ateapi-deps.sh b/hack/verify/ateapi-deps.sh index d41874be5d..64093cc4c6 100755 --- a/hack/verify/ateapi-deps.sh +++ b/hack/verify/ateapi-deps.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# ate-api-server moves snapshots through the snapshot plugin sidecar, so it +# ate-api-server moves snapshots through the object-store plugin sidecar, so it # must not link the object-store SDKs. set -o errexit -o nounset -o pipefail diff --git a/internal/objectstoreplugin/node.go b/internal/objectstoreplugin/node.go index bef2923ca7..99df8b87fc 100644 --- a/internal/objectstoreplugin/node.go +++ b/internal/objectstoreplugin/node.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package objectstoreplugin implements the snapshot plugin API on GCS or S3 +// Package objectstoreplugin implements the object-store plugin API on GCS or S3 // object storage, using the plugin's own credentials. // // Snapshot files are stored as zstd-compressed objects named .zstd diff --git a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go index bc3441ac6a..c7012105fb 100644 --- a/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go +++ b/internal/objectstoreplugin/objectstoreplugintest/objectstoreplugintest.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package objectstoreplugintest provides snapshot plugin clients that call the +// Package objectstoreplugintest provides object-store plugin clients that call the // object-store plugins directly, without a socket, for tests that drive atelet // or ate-api-server through the real plugin. package objectstoreplugintest diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 2e325ff40c..168fd63596 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -83,25 +83,25 @@ spec: # the readinessProbe below reaching NotReady, plus propagation) + # --drain-timeout. terminationGracePeriodSeconds: 40 - # The control snapshot plugin deletes and copies external snapshots for - # ate-api-server, over a Unix socket in a pod-local emptyDir. A native + # The control object-store plugin deletes and copies external snapshots + # for ate-api-server, over a Unix socket in a pod-local emptyDir. A native # sidecar starts before ate-api-server and stops after it. ate-api-server # starts only once the startup probe sees the plugin serving. initContainers: - - name: snapshot-plugin - image: ko://github.com/agent-substrate/substrate/cmd/snapshot-plugin + - name: objectstore-plugin + image: ko://github.com/agent-substrate/substrate/cmd/objectstore-plugin restartPolicy: Always args: - control - - --socket=/run/snapshot-plugin/control.sock + - --socket=/run/objectstore-plugin/control.sock # The plugin listens on a Unix socket, which a gRPC probe cannot # reach, and the image has no shell, so the binary checks itself. startupProbe: exec: command: - - /ko-app/snapshot-plugin + - /ko-app/objectstore-plugin - healthcheck - - --socket=/run/snapshot-plugin/control.sock + - --socket=/run/objectstore-plugin/control.sock periodSeconds: 2 failureThreshold: 30 securityContext: @@ -122,8 +122,8 @@ spec: - configMapRef: name: ate-otel-config volumeMounts: - - name: snapshot-plugin - mountPath: /run/snapshot-plugin + - name: objectstore-plugin + mountPath: /run/objectstore-plugin containers: - name: ate-api-server image: ko://github.com/agent-substrate/substrate/cmd/ateapi @@ -149,7 +149,7 @@ spec: # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s - --drain-timeout=15s - - --snapshot-plugin-socket=/run/snapshot-plugin/control.sock + - --objectstore-plugin-socket=/run/objectstore-plugin/control.sock env: - name: POD_NAME valueFrom: @@ -185,8 +185,8 @@ spec: name: ate-api-server-secret-envvars optional: true volumeMounts: - - name: snapshot-plugin - mountPath: /run/snapshot-plugin + - name: objectstore-plugin + mountPath: /run/objectstore-plugin # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" @@ -252,7 +252,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: - - name: snapshot-plugin + - name: objectstore-plugin emptyDir: {} - name: "servicedns" projected: diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 725d9696ac..02c4cc4ce9 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -147,20 +147,21 @@ spec: - key: ate.dev/sandboxClass operator: Exists effect: NoSchedule - # The node snapshot plugin moves external snapshots between this node and - # object storage for atelet, over a Unix socket in a pod-local emptyDir. + # The node object-store plugin moves external snapshots between this + # node and object storage and fetches sandbox assets for atelet, over a + # Unix socket in a pod-local emptyDir. # A native sidecar (restartPolicy: Always) starts before atelet and stops # after it, so uploads still in flight during atelet's drain finish. # atelet starts only once the startup probe sees the plugin serving. initContainers: - - name: snapshot-plugin - image: ko://github.com/agent-substrate/substrate/cmd/snapshot-plugin + - name: objectstore-plugin + image: ko://github.com/agent-substrate/substrate/cmd/objectstore-plugin restartPolicy: Always args: - node - - --socket=/run/snapshot-plugin/node.sock + - --socket=/run/objectstore-plugin/node.sock - --root=/var/lib/ate - - --asset-staging-dir=/var/lib/snapshot-plugin/asset-staging + - --asset-staging-dir=/var/lib/objectstore-plugin/asset-staging # atelet's 8 GiB per-asset cap, so any asset fits. Each download # reserves its object's size; ones past the capacity wait their turn, # so the asset-staging volume below never fills up. @@ -170,9 +171,9 @@ spec: startupProbe: exec: command: - - /ko-app/snapshot-plugin + - /ko-app/objectstore-plugin - healthcheck - - --socket=/run/snapshot-plugin/node.sock + - --socket=/run/objectstore-plugin/node.sock periodSeconds: 2 failureThreshold: 30 # Reads and writes snapshot and asset files under the /var/lib/ate @@ -203,13 +204,13 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ate - - name: snapshot-plugin - mountPath: /run/snapshot-plugin + - name: objectstore-plugin + mountPath: /run/objectstore-plugin # Sandbox assets are downloaded here and verified against their sha256 # before the plugin copies them below /var/lib/ate. Only this container # mounts it, so atelet never sees bytes it did not pin. - name: asset-staging - mountPath: /var/lib/snapshot-plugin/asset-staging + mountPath: /var/lib/objectstore-plugin/asset-staging containers: - name: atelet image: ko://github.com/agent-substrate/substrate/cmd/atelet @@ -227,7 +228,7 @@ spec: # terminationGracePeriodSeconds above. - --drain-delay=0s - --drain-timeout=5m - - --snapshot-plugin-socket=/run/snapshot-plugin/node.sock + - --objectstore-plugin-socket=/run/objectstore-plugin/node.sock # atelet does no mounts, netlink, device, or namespace operations (those # live in the ateom worker pod) — it only reads/writes the # /var/lib/ate hostPath as root. CAP_DAC_OVERRIDE is for one thing: @@ -300,8 +301,8 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ate - - name: snapshot-plugin - mountPath: /run/snapshot-plugin + - name: objectstore-plugin + mountPath: /run/objectstore-plugin - name: podidentity mountPath: /run/podidentity.podcert.ate.dev - name: servicedns-ca @@ -321,7 +322,7 @@ spec: mountPath: /run/image-credential-provider/config.yaml readOnly: true volumes: - - name: snapshot-plugin + - name: objectstore-plugin emptyDir: {} # Holds asset downloads until they are verified. The plugin keeps them # within --asset-staging-capacity; the limit adds 1Gi for filesystem diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 089cd1454e..b4160dbce6 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -56,11 +56,11 @@ patches: $patch: delete - name: image-credential-provider-config $patch: delete - # The snapshot plugin transfers atelet's external snapshots and + # The object-store plugin transfers atelet's external snapshots and # fetches its private sandbox assets (the micro-VM assets # stage-to-rustfs.sh uploads), so it points at rustfs. initContainers: - - name: snapshot-plugin + - name: objectstore-plugin env: - name: ATE_STORAGE_BACKEND value: s3 diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index f124e8ff9b..f97ad60d2c 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -43,7 +43,7 @@ resources: # shared by every component via envFrom -- putting always_on there would pin the # router and the control plane to 100% too, undoing their per-component ratios. # -# The storage settings point ate-api-server's snapshot plugin at the same +# The storage settings point ate-api-server's object-store plugin at the same # rustfs the ./atelet overlay points atelet's at: both ends of an external # snapshot's life have to agree on where it lives. patches: @@ -62,7 +62,7 @@ patches: - name: OTEL_TRACES_SAMPLER value: parentbased_always_on initContainers: - - name: snapshot-plugin + - name: objectstore-plugin env: - name: ATE_STORAGE_BACKEND value: s3 From 96dffb287919da3b198402c07e1fc68b7b61c90d Mon Sep 17 00:00:00 2001 From: cloudonly Date: Tue, 13 Oct 2026 05:39:53 +0000 Subject: [PATCH 18/20] Report missing Kind and control-plane prerequisites before deployment (#2403) On a new workstation, a Kind demo install reports only a missing kubeconfig context. On a cluster without the control plane, it can create the demo Namespace and WorkerPool before failing while waiting for a Deployment that no controller can create. Check Docker and the resolved kubeconfig context before Kind deployments, with instructions to restore kubeconfig or create the cluster and local registry. Demo and benchmark commands check for `deployment/ate-controller` before creating resources. Permission and connection errors retain their original cause. The checks live in `ate-setup`, so both native commands and the shell shims use them, including when a demo flag precedes `--deploy-ate-system`. The controller check establishes that it is installed; it does not replace workload rollout checks. Help and delete commands do not require the new deployment checks. Cluster recovery guidance states that the creation script replaces an existing cluster with the same name. Fixes #2400. Validation: - Regression tests first failed on the original implementation. Tests cover native commands and Kind shims, missing context and Docker, demo and benchmark prerequisites, action order, installed controllers, Forbidden responses, resolved configuration, malformed kubeconfig, and help. - `go test -race ./cmd/ate-setup/...` passed on macOS arm64. - The original isolated reproduction passes with the fix; the missing-controller scenario makes no resource writes. - `make verify` passed on Linux amd64 with Go 1.27.1: all 145 test packages passed with the race detector, and all 14 verification scripts passed. - The Linux `ate-setup` binary built successfully from the final commit. Generated files and the Git working tree remained clean. --- cmd/ate-setup/commands.md | 13 + cmd/ate-setup/internal/cmd/demo.go | 3 + cmd/ate-setup/internal/cmd/root.go | 14 + cmd/ate-setup/internal/config/kindcheck.go | 64 +++++ cmd/ate-setup/internal/steps/benchmarks.go | 3 + cmd/ate-setup/internal/steps/preflight.go | 51 ++++ cmd/ate-setup/preflight_test.go | 288 +++++++++++++++++++++ 7 files changed, 436 insertions(+) create mode 100644 cmd/ate-setup/internal/config/kindcheck.go create mode 100644 cmd/ate-setup/internal/steps/preflight.go create mode 100644 cmd/ate-setup/preflight_test.go diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 5131b60c4c..0b40317b16 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -43,6 +43,19 @@ a pre-scan pass, so they may appear anywhere on its command line. Both have an environment equivalent, read when the flag is absent: `ATE_IMAGE_REPO` and `ATE_IMAGE_TAG`. +Kind deployments check that Docker is available and the resolved kubeconfig +context exists before doing any install work. If the context is missing, the +error explains how to export kubeconfig for an existing cluster or create the +cluster and local registry with `hack/create-kind-cluster.sh`. That script +replaces an existing cluster with the same name. + +Deploy the control plane before deploying demos or benchmarks. These commands +check for `deployment/ate-controller` in the configured system namespace before +creating workload resources. An existing Deployment passes this check; normal +rollout waits still diagnose workloads that cannot become ready. The shell shim +runs actions in argument order, so put `--deploy-ate-system` before demo or +benchmark deployment flags when combining them on one command line. + ## Installing a release Without `--image-repo`, `ate-setup` builds every image from the checkout with diff --git a/cmd/ate-setup/internal/cmd/demo.go b/cmd/ate-setup/internal/cmd/demo.go index 1c7ad4825f..e3dc080deb 100644 --- a/cmd/ate-setup/internal/cmd/demo.go +++ b/cmd/ate-setup/internal/cmd/demo.go @@ -56,6 +56,9 @@ func init() { Short: demo.Description(), Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + if err := env.RequireControlPlane(cmd.Context()); err != nil { + return err + } return demo.Deploy(cmd.Context(), env) }, } diff --git a/cmd/ate-setup/internal/cmd/root.go b/cmd/ate-setup/internal/cmd/root.go index 674d02eaaa..2b1fd8995f 100644 --- a/cmd/ate-setup/internal/cmd/root.go +++ b/cmd/ate-setup/internal/cmd/root.go @@ -112,6 +112,11 @@ is present (skipped for --kind, and by NO_DEV_ENV=1).`, if err := cfg.EnsureClusterCredentials(cmd.Context()); err != nil { return err } + if cfg.Kind && isDeployCommand(cmd) { + if err := cfg.CheckKindCluster(cmd.Context()); err != nil { + return err + } + } env, err = steps.NewEnv(cfg) if err != nil { return err @@ -130,6 +135,15 @@ is present (skipped for --kind, and by NO_DEV_ENV=1).`, }, } +func isDeployCommand(cmd *cobra.Command) bool { + for c := cmd; c != nil; c = c.Parent() { + if c == deployCmd { + return true + } + } + return false +} + // recordRun writes what this run used, so the next one can reproduce it by // pointing --config at the file. Recording is best effort: an install that // worked must not be reported as failed because a directory was read-only. diff --git a/cmd/ate-setup/internal/config/kindcheck.go b/cmd/ate-setup/internal/config/kindcheck.go new file mode 100644 index 0000000000..cd7219956c --- /dev/null +++ b/cmd/ate-setup/internal/config/kindcheck.go @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "k8s.io/client-go/tools/clientcmd" +) + +// CheckKindCluster checks local deployment prerequisites without creating or +// replacing a cluster. It uses the same kubeconfig loading rules as the clients. +func (c *Config) CheckKindCluster(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + output, err := exec.CommandContext(ctx, "docker", "info").CombinedOutput() + if err != nil { + if ctx.Err() != nil { + err = ctx.Err() + } + return fmt.Errorf("cannot access Docker; start Docker and check `docker info` before deploying to Kind: %w\n%s", err, strings.TrimSpace(string(output))) + } + + rules := clientcmd.NewDefaultClientConfigLoadingRules() + rules.ExplicitPath = c.Kubeconfig + config, err := rules.Load() + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("while reading kubeconfig: %w", err) + } + name := c.Context + if err == nil { + if name == "" { + name = config.CurrentContext + } + if _, exists := config.Contexts[name]; exists { + return nil + } + } + + cluster := c.Resolved().String("kindCluster.name") + return fmt.Errorf("Kubernetes context %q was not found. If Kind cluster %q already exists, restore its kubeconfig with:\n"+ + " hack/kind.sh export kubeconfig --name=%q\n"+ + "Otherwise create the cluster and local registry with:\n"+ + " KIND_CLUSTER_NAME=%q hack/create-kind-cluster.sh\n"+ + "The creation script replaces an existing cluster with the same name. For a custom context, check --context and --kubeconfig instead", + name, cluster, cluster, cluster) +} diff --git a/cmd/ate-setup/internal/steps/benchmarks.go b/cmd/ate-setup/internal/steps/benchmarks.go index 6244460b40..abc8d6e920 100644 --- a/cmd/ate-setup/internal/steps/benchmarks.go +++ b/cmd/ate-setup/internal/steps/benchmarks.go @@ -65,6 +65,9 @@ func (e *Env) DeployBenchmarks(ctx context.Context, opts BenchmarkOptions) error if err := opts.Validate(); err != nil { return err } + if err := e.RequireControlPlane(ctx); err != nil { + return err + } log.Stepf("deploy_benchmarks (worker_count=%d, sandbox_class=%s)", opts.WorkerCount, opts.SandboxClass) // The microvm SandboxConfig lives outside the default set installed by diff --git a/cmd/ate-setup/internal/steps/preflight.go b/cmd/ate-setup/internal/steps/preflight.go new file mode 100644 index 0000000000..cdf8534ce7 --- /dev/null +++ b/cmd/ate-setup/internal/steps/preflight.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RequireControlPlane refuses workload installs before they create resources +// that only ate-controller can reconcile into Deployments. +func (e *Env) RequireControlPlane(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err := e.Kube.Typed.AppsV1().Deployments(e.Namespace()).Get(ctx, "ate-controller", metav1.GetOptions{}) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("while checking deployment/ate-controller in namespace %s: %w", e.Namespace(), err) + } + flags := "" + if e.Cfg.Kind { + flags += " --kind" + } + if e.Cfg.Context != "" { + flags += fmt.Sprintf(" --context=%q", e.Cfg.Context) + } + if e.Cfg.Kubeconfig != "" { + flags += fmt.Sprintf(" --kubeconfig=%q", e.Cfg.Kubeconfig) + } + return fmt.Errorf("demos and benchmarks require deployment/ate-controller in namespace %s: %w\n"+ + "Deploy the control plane first, using the same installation settings:\n"+ + " go run ./cmd/ate-setup%s deploy ate-system --credential-provider='{\"name\":\"k8s.io\"}'", e.Namespace(), err, flags) +} diff --git a/cmd/ate-setup/preflight_test.go b/cmd/ate-setup/preflight_test.go new file mode 100644 index 0000000000..a81d142761 --- /dev/null +++ b/cmd/ate-setup/preflight_test.go @@ -0,0 +1,288 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/cmd" +) + +// Each invocation gets its own command tree and flags, including invocations +// made by the real shell shims. The test executable runs the installer without +// rebuilding it or contacting a real cluster. +func TestInstallerPreflightProcess(t *testing.T) { + if os.Getenv("ATE_SETUP_PREFLIGHT_TEST") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + cmd.Root().SetArgs(os.Args[i+1:]) + cmd.Execute() + os.Exit(0) + } + } + t.Fatal("missing installer arguments") +} + +func runPreflightInstaller(t *testing.T, kubeconfig, dockerStatus string, shim bool, args ...string) (string, int) { + t.Helper() + bin := t.TempDir() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for name, source := range map[string]string{ + "docker": "#!/bin/sh\necho called >> \"$PREFLIGHT_DOCKER_LOG\"\nexit \"$PREFLIGHT_DOCKER_STATUS\"\n", + "go": `#!/bin/sh +if [ "$*" = "env GOARCH" ]; then + echo amd64 +elif [ "$1" = run ] && [ "$2" = ./cmd/ate-setup ]; then + shift 2 + exec "$PREFLIGHT_EXECUTABLE" -test.run='^TestInstallerPreflightProcess$' -- "$@" +else + echo "unexpected go invocation: $*" >&2 + exit 1 +fi +`, + } { + if err := os.WriteFile(filepath.Join(bin, name), []byte(source), 0o755); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + process := exec.CommandContext(ctx, executable, append([]string{"-test.run=^TestInstallerPreflightProcess$", "--"}, args...)...) + if shim { + process = exec.CommandContext(ctx, "bash", append([]string{"hack/install-ate-kind.sh"}, args...)...) + } + process.Dir = repoRoot(t) + process.Env = []string{ + "PATH=" + bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "KUBECONFIG=" + kubeconfig, + "NO_DEV_ENV=true", + "ATE_NO_REPORT_DEFAULTS=1", + "ATE_SETUP_PREFLIGHT_TEST=1", + "PREFLIGHT_EXECUTABLE=" + executable, + "PREFLIGHT_DOCKER_STATUS=" + dockerStatus, + "PREFLIGHT_DOCKER_LOG=" + filepath.Join(bin, "docker.log"), + } + output, err := process.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("installer did not fail promptly: %v\n%s", ctx.Err(), output) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return string(output), exitErr.ExitCode() + } + if err != nil { + t.Fatalf("running installer: %v\n%s", err, output) + } + return string(output), 0 +} + +func preflightKubeconfig(t *testing.T, server string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config") + contents := "apiVersion: v1\nkind: Config\nclusters: []\ncontexts: []\nusers: []\n" + if server != "" { + contents = fmt.Sprintf(`apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: %s +contexts: +- name: kind-kind + context: + cluster: local + user: local +users: +- name: local + user: {} +current-context: kind-kind +`, server) + } + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestKindInstallerMissingContext(t *testing.T) { + for _, shim := range []bool{false, true} { + t.Run(fmt.Sprintf("shim=%t", shim), func(t *testing.T) { + args := []string{"--kind", "deploy", "demo", "counter"} + if shim { + args = []string{"--deploy-demo-counter"} + } + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "0", shim, args...) + if code == 0 || !strings.Contains(out, "create-kind-cluster.sh") || !strings.Contains(out, "export kubeconfig") { + t.Fatalf("exit=%d, want missing-context recovery guidance\n%s", code, out) + } + }) + } +} + +func TestKindInstallerDockerUnavailable(t *testing.T) { + out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "1", false, + "--kind", "deploy", "demo", "counter") + if code == 0 || !strings.Contains(out, "Docker") || !strings.Contains(out, "docker info") { + t.Fatalf("exit=%d, want Docker readiness guidance\n%s", code, out) + } +} + +func TestInstallerRequiresControlPlaneBeforeDemoOrBenchmark(t *testing.T) { + for _, tc := range []struct { + name string + shim bool + args []string + }{ + {"native demo", false, []string{"deploy", "demo", "counter"}}, + {"native benchmark", false, []string{"deploy", "benchmarks"}}, + {"kind demo", true, []string{"--deploy-demo-autoscaled-workerpool"}}, + {"kind benchmark", true, []string{"--deploy-benchmarks"}}, + {"demo before system", true, []string{"--deploy-demo-counter", "--deploy-ate-system"}}, + } { + t.Run(tc.name, func(t *testing.T) { + var mu sync.Mutex + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests = append(requests, r.Method+" "+r.URL.Path) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"NotFound","code":404,"message":"deployments.apps ate-controller not found"}`) + })) + defer server.Close() + out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", tc.shim, tc.args...) + if code == 0 || !strings.Contains(out, "deploy ate-system") || !strings.Contains(out, "credential-provider") { + t.Errorf("exit=%d, want control-plane install guidance\n%s", code, out) + } + mu.Lock() + defer mu.Unlock() + want := "GET /apis/apps/v1/namespaces/ate-system/deployments/ate-controller" + if len(requests) != 1 || requests[0] != want { + t.Errorf("requests=%v, want only %q before refusing deployment", requests, want) + } + }) + } +} + +func TestKindInstallerHelpNeedsNoCluster(t *testing.T) { + for _, shim := range []bool{false, true} { + args := []string{"--kind", "deploy", "demo", "counter", "--help"} + if shim { + args = []string{"--help"} + } + out, code := runPreflightInstaller(t, preflightKubeconfig(t, ""), "1", shim, args...) + if code != 0 || !strings.Contains(out, "Usage:") { + t.Fatalf("shim=%t exit=%d, want help without prerequisites\n%s", shim, code, out) + } + } +} + +func TestInstallerControlPlaneCheckPreservesErrorsAndAllowsInstalledSystem(t *testing.T) { + for _, installed := range []bool{false, true} { + t.Run(fmt.Sprintf("installed=%t", installed), func(t *testing.T) { + var mu sync.Mutex + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests = append(requests, r.Method+" "+r.URL.Path) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if installed && strings.HasSuffix(r.URL.Path, "/deployments/ate-controller") { + _, _ = fmt.Fprint(w, `{"kind":"Deployment","apiVersion":"apps/v1","metadata":{"name":"ate-controller","namespace":"ate-system"}}`) + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"Forbidden","code":403,"message":"fixture access denied"}`) + })) + defer server.Close() + out, code := runPreflightInstaller(t, preflightKubeconfig(t, server.URL), "0", false, "deploy", "demo", "counter") + want := "fixture access denied" + if installed { + want = "demo-counter_deploy" + } + if code == 0 || !strings.Contains(out, want) || strings.Contains(out, "Deploy the control plane first") { + t.Errorf("exit=%d, want original API error without missing-install guidance\n%s", code, out) + } + mu.Lock() + defer mu.Unlock() + if len(requests) == 0 || requests[0] != "GET /apis/apps/v1/namespaces/ate-system/deployments/ate-controller" { + t.Fatalf("requests=%v, want control-plane check first", requests) + } + if installed && len(requests) < 2 { + t.Errorf("requests=%v, want demo to proceed after the control-plane check", requests) + } + if !installed && len(requests) != 1 { + t.Errorf("requests=%v, want permission error to stop the install", requests) + } + for _, request := range requests { + if !strings.HasPrefix(request, "GET ") { + t.Errorf("unexpected mutation: %s", request) + } + } + }) + } +} + +func TestKindInstallerUsesResolvedContextAndClusterName(t *testing.T) { + document := filepath.Join(t.TempDir(), "install.yaml") + if err := os.WriteFile(document, []byte("apiVersion: install.ate.dev/v1alpha1\nkind: SubstrateInstall\ncontext: selected-by-config\nkindCluster:\n name: local-dev\n"), 0o600); err != nil { + t.Fatal(err) + } + out, code := runPreflightInstaller(t, preflightKubeconfig(t, "http://127.0.0.1:1"), "0", false, + "--kind", "--config", document, "deploy", "demo", "counter") + for _, want := range []string{`context "selected-by-config"`, `export kubeconfig --name="local-dev"`, `KIND_CLUSTER_NAME="local-dev"`, "replaces an existing cluster"} { + if code == 0 || !strings.Contains(out, want) { + t.Errorf("exit=%d, want %q in resolved-context guidance\n%s", code, want, out) + } + } +} + +func TestKindInstallerMissingOrMalformedKubeconfig(t *testing.T) { + for _, malformed := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "config") + want := "create-kind-cluster.sh" + if malformed { + if err := os.WriteFile(path, []byte("contexts: [\n"), 0o600); err != nil { + t.Fatal(err) + } + want = "while reading kubeconfig" + } + out, code := runPreflightInstaller(t, path, "0", false, "--kind", "deploy", "demo", "counter") + if code == 0 || !strings.Contains(out, want) { + t.Errorf("malformed=%t exit=%d, want %q\n%s", malformed, code, want, out) + } + if malformed && strings.Contains(out, "create-kind-cluster.sh") { + t.Errorf("malformed kubeconfig was misreported as a missing cluster\n%s", out) + } + } +} From 760962b2438b62e82cd9556194edec0a2610370f Mon Sep 17 00:00:00 2001 From: ktsoator Date: Tue, 13 Oct 2026 06:16:56 +0000 Subject: [PATCH 19/20] fix(objectstorage): handle download reader close errors (#2446) Return the reader's close error when the download succeeds, and preserve the original download error when both fail. Add regression tests with valid and truncated zstd streams. Refs #2445 Tested with `go test -race ./pkg/objectstorage -count=1`. --- pkg/objectstorage/objects.go | 2 +- pkg/objectstorage/objects_test.go | 48 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/pkg/objectstorage/objects.go b/pkg/objectstorage/objects.go index 5f0cef3fe7..7c37fbbfc0 100644 --- a/pkg/objectstorage/objects.go +++ b/pkg/objectstorage/objects.go @@ -389,7 +389,7 @@ func fetchFromGCSWithZstd(ctx context.Context, client ObjectStorage, gsURL strin } defer func() { if closeErr := rc.Close(); closeErr != nil { - if err != nil { + if err == nil { err = closeErr } else { slog.InfoContext(ctx, "Dropped error from rc.Close", slog.Any("err", closeErr)) diff --git a/pkg/objectstorage/objects_test.go b/pkg/objectstorage/objects_test.go index 67e309e376..53ae63fdd7 100644 --- a/pkg/objectstorage/objects_test.go +++ b/pkg/objectstorage/objects_test.go @@ -17,6 +17,7 @@ package objectstorage import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -47,6 +48,53 @@ func (s *memStore) GetObject(_ context.Context, bucket, object string) (io.ReadC return io.NopCloser(bytes.NewReader(b)), nil } +type closeErrorReader struct { + io.ReadCloser + closeErr error +} + +func (r *closeErrorReader) Close() error { + return errors.Join(r.ReadCloser.Close(), r.closeErr) +} + +type closeErrorStore struct { + *memStore + closeErr error +} + +func (s *closeErrorStore) GetObject(ctx context.Context, bucket, object string) (io.ReadCloser, error) { + r, err := s.memStore.GetObject(ctx, bucket, object) + if err != nil { + return nil, err + } + return &closeErrorReader{ReadCloser: r, closeErr: s.closeErr}, nil +} + +func TestFetchFromGCSWithZstdCloseError(t *testing.T) { + var compressed bytes.Buffer + if _, err := writeContent(&compressed, bytes.NewBufferString("snapshot payload")); err != nil { + t.Fatal(err) + } + closeErr := errors.New("object reader close failed") + for _, tc := range []struct { + name string + data []byte + wantErr error + }{ + {name: "successful decode", data: compressed.Bytes(), wantErr: closeErr}, + {name: "truncated object", data: compressed.Bytes()[:compressed.Len()-1], wantErr: io.ErrUnexpectedEOF}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &closeErrorStore{memStore: newMemStore(), closeErr: closeErr} + store.m["bucket/snapshot"] = tc.data + err := fetchFromGCSWithZstd(context.Background(), store, "gs://bucket/snapshot", io.Discard) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("fetchFromGCSWithZstd() error = %v, want %v", err, tc.wantErr) + } + }) + } +} + // streamingMemStore is a memStore that advertises streaming PutObject support, so // sendZstd takes the pipe (compress∥upload overlap) path used for GCS // instead of staging a seekable temp file. From be3aa280fad168326e73f75954f746fafc2953c4 Mon Sep 17 00:00:00 2001 From: anishgang Date: Tue, 13 Oct 2026 06:18:25 +0000 Subject: [PATCH 20/20] Benchmark external volumes with the DurDir load (#2057) Rebases #1074 (Add benchmarking for storage, by @elijahrb) onto main and runs it with the DurDir load. Replaces #1074. - Adds the `glutton-storage` ActorTemplate, which mounts an external CSI volume as glutton's data dir. `workloads/deploy.sh --storage-class-name NAME` deploys it, and `deploy_locust.sh` passes the flag through. Without a StorageClass it is not deployed, so clusters with no CSI driver don't wait on its golden snapshot. - The DurDir load drives it with `--durdir-template glutton-storage`. DurDir already resumes an actor, writes a file, checks the file's checksum after every resume, and suspends the actor again, so the storage benchmark needs no load of its own. DurDir does not count crashed actors under `CrashCount` yet. - glutton gets two opt-in flags. Only `glutton-storage` sets them, so the baseline and DurDir numbers don't change. `--sync-writes` fsyncs each `WriteDisk` file. `--readyz-require-writable-data-dir` fails `/readyz` until the data dir is writable, so a resume waits for the volume mount. - `csi-nfs-sc` is now defined in one file, `hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml`. `ate-setup setup csi nfs` and `hack/setup-csi-nfs-kind.sh` both apply it instead of each carrying a copy, so the e2e tests, demos and benchmarks use the same class. A test checks this. To benchmark on NFS, install with `hack/install-ate.sh --setup-csi=nfs` and pass `--storage-class-name csi-nfs-sc`. - The Locust scripts take `LOCUST_IMAGE` or `KO_DOCKER_REPO`, so they work on Kind with a local registry, and `LOCUST_IMAGE_PULL_POLICY` (default `Always`). The orchestrator and `tests.yaml` don't change. Letting a test set a StorageClass or apply extra manifests moves to a follow-up PR, where the test API can be worked out. Testing: unit tests for the glutton flags and the shared StorageClass. `go test -race` on the changed packages, golangci-lint, shellcheck and the boilerplate check pass. Not run end to end on a cluster. - [x] Tests pass --------- Co-authored-by: Elijah Rodriguez-Beltran --- benchmarking/README.md | 11 +++ benchmarking/deploy_locust.sh | 8 ++ benchmarking/locust/build_and_push.sh | 12 ++- benchmarking/locust/deploy.sh | 18 +++- benchmarking/locust/manifests/locust.yaml | 12 +-- benchmarking/workloads/deploy.sh | 32 +++++- .../glutton-storage-template.yaml.tmpl | 62 ++++++++++++ cmd/ate-setup/internal/steps/csi.go | 19 +--- cmd/ate-setup/internal/steps/csi_test.go | 44 +++++++++ cmd/benchmarking/glutton/main.go | 11 ++- hack/setup-csi-nfs-kind.sh | 17 +--- .../deploy/example/storageclass-nfs.yaml | 21 ++++ internal/benchmarking/glutton/server.go | 24 ++++- internal/benchmarking/glutton/server_test.go | 97 +++++++++++++++++++ internal/benchmarking/glutton/service.go | 29 +++++- 15 files changed, 363 insertions(+), 54 deletions(-) create mode 100644 benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl create mode 100644 hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml diff --git a/benchmarking/README.md b/benchmarking/README.md index 154aad7737..015964bc58 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -13,6 +13,12 @@ scenario ladder, read [observability.md](observability.md). > [!IMPORTANT] > Source the environment configuration file (e.g., `source .ate-dev-env.sh`) > first so `PROJECT_ID`, `BUCKET_NAME`, etc. are set. +> +> **For Local Kind Clusters**: Set `KO_DOCKER_REPO` to the local registry so images are pulled without remote cloud credentials: +> ```bash +> export KO_DOCKER_REPO="localhost:5001" +> export BUCKET_NAME="ate-snapshots" +> ``` Note that deploying the benchmarks does not run them. You must visit Locust's web UI to start a test. @@ -113,6 +119,11 @@ and state restoration latency when a durable directory is attached to the actor. * `--durdir-template`: ActorTemplate name: * `glutton-durdir-data` (default): Attaches a durable data directory without memory snapshot restore. * `glutton-durdir-full`: Attaches a durable data directory and performs a full memory snapshot restore. + * `glutton-storage`: Attaches an external CSI volume instead of a durable + directory. `deploy_locust.sh` and `workloads/deploy.sh` deploy it only + when `--storage-class-name` names an existing StorageClass. To run it on + NFS, install with `hack/install-ate.sh --setup-csi=nfs` and pass + `--storage-class-name csi-nfs-sc`, the class the e2e tests also use. #### DurDir Reported Metrics diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index 1154693599..a51d1e6ee5 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -33,6 +33,7 @@ OTLP_ENDPOINT="" ACTOR_MEMORY="" # Empty leaves the WorkerPool pods unsized. WORKER_MEMORY="" +STORAGE_CLASS_NAME="" WAIT_TIMEOUT_SECS="" usage() { @@ -50,6 +51,8 @@ usage() { echo " benchmark ActorTemplates (default: 256Mi, the microvm minimum)." echo " --worker-memory SIZE Forwarded to workloads/deploy.sh. Memory request and limit" echo " for each WorkerPool pod (default: unset, the pod is unsized)." + echo " --storage-class-name S Forwarded to workloads/deploy.sh. StorageClass for the glutton-storage" + echo " external volume; setting it deploys glutton-storage." echo " --wait-timeout SECONDS Forwarded to workloads/deploy.sh. The timeout in seconds for" echo " waiting for the ateom workers to be ready (default: 300)" echo " --skip-build Skip locust image build/push (use the existing :latest image)" @@ -80,6 +83,8 @@ while [[ "$#" -gt 0 ]]; do --actor-memory=*) ACTOR_MEMORY="${1#*=}" ;; --worker-memory) shift; WORKER_MEMORY="$1" ;; --worker-memory=*) WORKER_MEMORY="${1#*=}" ;; + --storage-class-name) shift; STORAGE_CLASS_NAME="$1" ;; + --storage-class-name=*) STORAGE_CLASS_NAME="${1#*=}" ;; --wait-timeout) shift; WAIT_TIMEOUT_SECS="$1" ;; --wait-timeout=*) WAIT_TIMEOUT_SECS="${1#*=}" ;; --skip-build) SKIP_BUILD=1 ;; @@ -121,6 +126,9 @@ if [[ "${action}" == "deploy" ]]; then if [[ -n "${WORKER_MEMORY}" ]]; then workload_args+=(--worker-memory "${WORKER_MEMORY}") fi + if [[ -n "${STORAGE_CLASS_NAME}" ]]; then + workload_args+=(--storage-class-name "${STORAGE_CLASS_NAME}") + fi if [[ -n "${WAIT_TIMEOUT_SECS}" ]]; then workload_args+=(--wait-timeout "${WAIT_TIMEOUT_SECS}") fi diff --git a/benchmarking/locust/build_and_push.sh b/benchmarking/locust/build_and_push.sh index b4b4c5b68f..5fda8a6041 100755 --- a/benchmarking/locust/build_and_push.sh +++ b/benchmarking/locust/build_and_push.sh @@ -24,12 +24,18 @@ if [[ -f .ate-dev-env.sh ]]; then source .ate-dev-env.sh fi -if [ -z "${PROJECT_ID:-}" ]; then - echo "Error: PROJECT_ID environment variable must be set." >&2 +if [[ -z "${PROJECT_ID:-}" && -z "${KO_DOCKER_REPO:-}" && -z "${LOCUST_IMAGE:-}" ]]; then + echo "Error: PROJECT_ID or KO_DOCKER_REPO environment variable must be set." >&2 exit 1 fi -IMAGE="us-docker.pkg.dev/${PROJECT_ID}/gcr.io/ate-images/locust-test:latest" +if [[ -n "${LOCUST_IMAGE:-}" ]]; then + IMAGE="${LOCUST_IMAGE}" +elif [[ -n "${KO_DOCKER_REPO:-}" ]]; then + IMAGE="${KO_DOCKER_REPO}/locust-test:latest" +else + IMAGE="us-docker.pkg.dev/${PROJECT_ID}/gcr.io/ate-images/locust-test:latest" +fi # Target platform must match the cluster's nodes, not the build host. PLATFORM="${LOCUST_IMAGE_PLATFORM:-linux/amd64}" diff --git a/benchmarking/locust/deploy.sh b/benchmarking/locust/deploy.sh index 19d4939695..09b2f3fc96 100755 --- a/benchmarking/locust/deploy.sh +++ b/benchmarking/locust/deploy.sh @@ -24,11 +24,23 @@ if [[ -f .ate-dev-env.sh ]]; then source .ate-dev-env.sh fi -if [ -z "${PROJECT_ID:-}" ]; then - echo "Error: PROJECT_ID environment variable must be set." >&2 +if [[ -z "${PROJECT_ID:-}" && -z "${KO_DOCKER_REPO:-}" && -z "${LOCUST_IMAGE:-}" ]]; then + echo "Error: PROJECT_ID or KO_DOCKER_REPO environment variable must be set." >&2 exit 1 fi +if [[ -z "${LOCUST_IMAGE:-}" ]]; then + if [[ -n "${KO_DOCKER_REPO:-}" ]]; then + LOCUST_IMAGE="${KO_DOCKER_REPO}/locust-test:latest" + else + LOCUST_IMAGE="us-docker.pkg.dev/${PROJECT_ID}/gcr.io/ate-images/locust-test:latest" + fi +fi + +LOCUST_IMAGE_PULL_POLICY="${LOCUST_IMAGE_PULL_POLICY:-Always}" + +export LOCUST_IMAGE LOCUST_IMAGE_PULL_POLICY + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MANIFEST="${SCRIPT_DIR}/manifests/locust.yaml" @@ -67,7 +79,7 @@ deploy() { kubectl -n benchmarking create configmap agentsession-script \ --from-file="script.yaml=${AGENTSESSION_SCRIPT}" --dry-run=client -o yaml | kubectl apply -f - fi - echo "Deploying Locust load (PROJECT_ID=${PROJECT_ID}, user_class=${BENCHMARK_USER_CLASS}, agentsession_script=${AGENTSESSION_SCRIPT:-})..." + echo "Deploying Locust load (image=${LOCUST_IMAGE}, pull_policy=${LOCUST_IMAGE_PULL_POLICY}, user_class=${BENCHMARK_USER_CLASS}, agentsession_script=${AGENTSESSION_SCRIPT:-})..." envsubst < "${MANIFEST}" | kubectl apply -f - } diff --git a/benchmarking/locust/manifests/locust.yaml b/benchmarking/locust/manifests/locust.yaml index 2affc71e6d..7c7c0534e7 100644 --- a/benchmarking/locust/manifests/locust.yaml +++ b/benchmarking/locust/manifests/locust.yaml @@ -46,8 +46,8 @@ spec: spec: containers: - name: locust-master - image: us-docker.pkg.dev/${PROJECT_ID}/gcr.io/ate-images/locust-test:latest - imagePullPolicy: Always + image: ${LOCUST_IMAGE} + imagePullPolicy: ${LOCUST_IMAGE_PULL_POLICY} args: - "-m" - "locust" @@ -86,8 +86,8 @@ spec: memory: "512Mi" # Python and boomer workers do not play nicely together, leaving this in for reference. # - name: locust-python-worker - # image: us-docker.pkg.dev/${PROJECT_ID}/gcr.io/ate-images/locust-test:latest - # imagePullPolicy: Always + # image: ${LOCUST_IMAGE} + # imagePullPolicy: ${LOCUST_IMAGE_PULL_POLICY} # args: # - "-m" # - "locust" @@ -122,8 +122,8 @@ spec: - name: boomer-worker # Same image as the master/worker — the boomer-worker Go binary is # baked in at /app/boomer-worker by benchmarking/locust/Dockerfile. - image: us-docker.pkg.dev/${PROJECT_ID}/gcr.io/ate-images/locust-test:latest - imagePullPolicy: Always + image: ${LOCUST_IMAGE} + imagePullPolicy: ${LOCUST_IMAGE_PULL_POLICY} command: ["/app/boomer-worker"] args: - "--prometheus-addr=:8001" diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 675eb588f7..ff5623c98f 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -34,7 +34,9 @@ POOL_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl" # The benchmark ActorTemplates: -template.yaml.tmpl each, created # through the ate API in the benchmark-workloads atespace. WORKLOAD_TEMPLATES # overrides the default set — the usermem and kernelmem templates (for the -# matching locust tests) are not deployed by default. +# matching locust tests) are not deployed by default. The glutton-storage +# template is added when a StorageClass is set, because it needs a CSI driver +# that not every cluster has. read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full}" if [[ ! -f "${POOL_MANIFEST}" ]]; then @@ -53,6 +55,9 @@ ACTOR_MEMORY="256Mi" # and limit. Empty leaves the pod unsized. The limit is also the memory the # worker reports as its actor capacity. WORKER_MEMORY="" +# The StorageClass for the glutton-storage external volume. Empty means the +# template is not deployed. +STORAGE_CLASS_NAME="${STORAGE_CLASS_NAME:-}" # The address to which an instrumented actor container sends its telemetry. # --otlp-endpoint sets it. Without the flag, resolve_otlp_endpoint reads the # address that the control plane uses. @@ -74,6 +79,8 @@ usage() { echo " the smallest size microvm admits)" echo " --worker-memory SIZE Memory request and limit for each WorkerPool pod" echo " (default: unset, the pod is unsized)" + echo " --storage-class-name NAME StorageClass for the glutton-storage external volume. Setting it" + echo " deploys glutton-storage (default: not deployed). Env: STORAGE_CLASS_NAME." echo " --otlp-endpoint URL The address to which an instrumented actor container" echo " sends telemetry (default: the endpoint in the" echo " ate-otel-config ConfigMap)" @@ -145,6 +152,7 @@ substitute() { -e "s|\${ACTOR_MEMORY}|${ACTOR_MEMORY}|g" \ -e "s|\${WORKER_RESOURCES}|${worker_resources}|g" \ -e "s|\${SWEPERF_IMAGE}|${SWEPERF_IMAGE:-}|g" \ + -e "s|\${STORAGE_CLASS_NAME}|${STORAGE_CLASS_NAME}|g" \ "${manifest}" } @@ -195,7 +203,7 @@ wait_templates_ready() { deploy() { resolve_otlp_endpoint - echo "Deploying workloads (worker_count=${WORKER_COUNT}, actor_memory=${ACTOR_MEMORY}, worker_memory=${WORKER_MEMORY:-unset}, otlp_endpoint=${OTLP_ENDPOINT})..." + echo "Deploying workloads (worker_count=${WORKER_COUNT}, actor_memory=${ACTOR_MEMORY}, worker_memory=${WORKER_MEMORY:-unset}, storage_class=${STORAGE_CLASS_NAME:-none}, otlp_endpoint=${OTLP_ENDPOINT})..." substitute "${POOL_MANIFEST}" | hack/run-tool.sh ko apply -f - echo "Waiting for worker pool to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." kubectl wait --for=create deployment/benchmark-ateom \ @@ -289,6 +297,13 @@ while [[ "$#" -gt 0 ]]; do --worker-memory=*) WORKER_MEMORY="${1#*=}" ;; + --storage-class-name) + shift + STORAGE_CLASS_NAME="$1" + ;; + --storage-class-name=*) + STORAGE_CLASS_NAME="${1#*=}" + ;; --wait-timeout) shift WAIT_TIMEOUT_SECS="$1" @@ -322,6 +337,19 @@ if ! [[ "${WAIT_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then exit 1 fi +# glutton-storage joins the default set only when it has a StorageClass. Delete +# always includes it, because deleting a template that does not exist is a +# no-op. +if [[ -z "${WORKLOAD_TEMPLATES:-}" ]] \ + && { [[ -n "${STORAGE_CLASS_NAME}" ]] || [[ "${action}" == "delete" ]]; }; then + TEMPLATES+=(glutton-storage) +fi + +if [[ "${action}" == "deploy" && -z "${STORAGE_CLASS_NAME}" && " ${TEMPLATES[*]} " == *" glutton-storage "* ]]; then + echo "Error: the glutton-storage template needs --storage-class-name" >&2 + exit 1 +fi + if [[ "${action}" == "deploy" ]]; then build_kubectl_ate deploy diff --git a/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl new file mode 100644 index 0000000000..67812376f6 --- /dev/null +++ b/benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl @@ -0,0 +1,62 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ActorTemplate (protojson-shaped ateapipb.ActorTemplate) for the glutton-storage +# benchmark workload, created through the ate API by workloads/deploy.sh. +# Attaches an external volume backed by ${STORAGE_CLASS_NAME}. The DurDir load +# runs it with --durdir-template glutton-storage. +metadata: + atespace: benchmark-workloads + name: glutton-storage +workerSelector: + matchLabels: + workload: benchmark-ateom +volumes: +- name: external-storage + externalVolumeTemplate: + capacity: 1Gi + storageClassName: ${STORAGE_CLASS_NAME} +containers: +- name: glutton-storage + image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton + command: + - "/ko-app/glutton" + - "--grpc-listen-addr=:80" + - "--metrics-listen-addr=:9090" + - "--data-dir=/mnt/storage" + - "--sync-writes" + - "--readyz-require-writable-data-dir" + - "--mode=http" + volumeMounts: + - name: external-storage + mountPath: /mnt/storage + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "${OTLP_ENDPOINT}" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "service.name=glutton-storage-actor" + wakeupProbe: + httpGet: + path: /readyz + port: 80 +resources: + limits: + - name: memory + quantity: ${ACTOR_MEMORY} +snapshotConfig: + onCommit: SNAPSHOT_CONTENT_SCOPE_FULL + storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-storage/ +sandboxConfig: + sandboxClass: ${SANDBOX_CLASS_ENUM} + configName: ${SANDBOX_CONFIG_NAME} diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index 786b5e064b..4dcf0914a8 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -39,23 +39,6 @@ import ( // they mount inside it are visible to atelet. const ateomHostDir = nodepath.BasePath -// csiNFSStorageClass is the StorageClass the external volume demos provision -// from. -const csiNFSStorageClass = ` -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-nfs-sc -provisioner: nfs.csi.k8s.io -parameters: - server: nfs-server.default.svc.cluster.local - share: / -reclaimPolicy: Delete -volumeBindingMode: Immediate -mountOptions: - - nfsvers=4.1 -` - // csiHostpathControllerService fronts the hostpath driver's socat sidecar, // which bridges the TCP endpoint atelet dials to the driver's unix socket. The // CSIDriverConfig applied later names this Service in both its @@ -410,7 +393,7 @@ spec: // The driver resolves the server's DNS name at provisioning time, so the // StorageClass can be created before the NFS server has an address. log.Infof("Creating the csi-nfs-sc StorageClass...") - if err := e.applyInline(ctx, csiNFSStorageClass); err != nil { + if err := e.Kube.ApplyPath(ctx, deployDir+"/example/storageclass-nfs.yaml"); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/csi_test.go b/cmd/ate-setup/internal/steps/csi_test.go index c22a1c1efe..aded1b9905 100644 --- a/cmd/ate-setup/internal/steps/csi_test.go +++ b/cmd/ate-setup/internal/steps/csi_test.go @@ -17,10 +17,14 @@ package steps import ( "context" "os" + "path/filepath" "strings" "testing" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" ) // setupCSIWithoutACluster runs SetupCSI against an Env holding no cluster @@ -123,6 +127,46 @@ func TestTheHostpathControllerServiceIsCreatedBeforeTheDriver(t *testing.T) { } } +// The e2e tests, the demos and the benchmarks all provision from csi-nfs-sc, so +// both NFS setups apply the one manifest that defines it rather than each +// carrying a copy that can drift. The step wants a cluster, so which file it +// applies is pinned in the source. +func TestTheNFSSetupsApplyTheSharedStorageClass(t *testing.T) { + const manifest = "hack/third_party/csi-driver-nfs/deploy/example/storageclass-nfs.yaml" + root := repoRoot(t) + + objs, err := kube.LoadPath(filepath.Join(root, manifest)) + if err != nil { + t.Fatalf("loading %s: %v", manifest, err) + } + if len(objs) != 1 || objs[0].GetKind() != "StorageClass" || objs[0].GetName() != "csi-nfs-sc" { + t.Fatalf("%s holds %d object(s), want only the csi-nfs-sc StorageClass", manifest, len(objs)) + } + if p, _, _ := unstructured.NestedString(objs[0].Object, "provisioner"); p != "nfs.csi.k8s.io" { + t.Errorf("csi-nfs-sc provisioner = %q, want nfs.csi.k8s.io", p) + } + + goSetup := functionBody(t, "csi.go", "func (e *Env) setupCSINFS(") + if !strings.Contains(goSetup, `deployDir+"/example/storageclass-nfs.yaml"`) { + t.Errorf("setupCSINFS no longer applies %s", manifest) + } + script, err := os.ReadFile(filepath.Join(root, "hack", "setup-csi-nfs-kind.sh")) + if err != nil { + t.Fatalf("reading the shell setup: %v", err) + } + if !strings.Contains(string(script), manifest) { + t.Errorf("hack/setup-csi-nfs-kind.sh no longer applies %s", manifest) + } + for name, src := range map[string]string{ + "setupCSINFS": goSetup, + "hack/setup-csi-nfs-kind.sh": string(script), + } { + if strings.Contains(src, "kind: StorageClass") { + t.Errorf("%s defines a StorageClass inline; apply %s instead", name, manifest) + } + } +} + // Both spellings of each driver selection have to keep selecting the same // thing: --setup-csi was a boolean in the shell installer, and the scripts and // dev-env files that still pass true or false have to keep working. diff --git a/cmd/benchmarking/glutton/main.go b/cmd/benchmarking/glutton/main.go index 9c01fb99b6..901d175789 100644 --- a/cmd/benchmarking/glutton/main.go +++ b/cmd/benchmarking/glutton/main.go @@ -36,6 +36,8 @@ var ( metricsListenAddr = pflag.String("metrics-listen-addr", ":9090", "Address and port the Prometheus metrics server should listen on.") dataDir = pflag.String("data-dir", "", "Directory under which WriteDisk files are stored. Required.") mode = pflag.String("mode", glutton.ModeGRPC, "Wire protocol for the main listener: grpc (default) or http.") + syncWrites = pflag.Bool("sync-writes", false, "Fsync each WriteDisk file before returning, so the write includes the flush to the backing volume.") + writableReadyz = pflag.Bool("readyz-require-writable-data-dir", false, "Fail /readyz until --data-dir is writable, so a resume waits for an external volume mount.") showVersion = pflag.Bool("version", false, "Print version and exit.") ) @@ -73,7 +75,14 @@ func main() { serverboot.Fatal(ctx, "Failed to create data directory", fmt.Errorf("%s: %w", *dataDir, err)) } - svc, err := glutton.New(*dataDir) + var opts []glutton.Option + if *syncWrites { + opts = append(opts, glutton.WithSyncWrites()) + } + if *writableReadyz { + opts = append(opts, glutton.WithWritableReadyz()) + } + svc, err := glutton.New(*dataDir, opts...) if err != nil { serverboot.Fatal(ctx, "Failed to construct glutton service", err) } diff --git a/hack/setup-csi-nfs-kind.sh b/hack/setup-csi-nfs-kind.sh index c05af9c40c..68a41e011b 100755 --- a/hack/setup-csi-nfs-kind.sh +++ b/hack/setup-csi-nfs-kind.sh @@ -96,23 +96,8 @@ EOF # 6. Create NFS StorageClass (pointing to the in-cluster NFS server) # We wait for the NFS server service to get an IP first, but we can use its DNS name # since kube-dns should resolve it. The CSI driver will resolve it when provisioning. -# The share is "/" because hack/csi/nfs-server.yaml exports a single directory with -# fsid=0, which NFSv4 presents as the pseudo-root. echo "Creating csi-nfs-sc StorageClass..." -cat <