From 49ee2fe484cb8a7b042ba4939139dbb9e050ec07 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 15 Oct 2026 17:06:12 +0200 Subject: [PATCH 1/2] feat(credential-provider): grant namespaces selected by label The Kubernetes credential provider grants each atespace a fixed list of namespaces, read once at startup. A platform that creates a namespace per tenant at run time, and runs each tenant's actors in the atespace named after it, cannot follow those namespaces with a fixed list. A namespace policy can now carry an ownNamespaceSelector, a Kubernetes label selector, in place of atespace and allowedNamespaces. Each namespace the selector matches may be read by the atespace of the same name and by no other, so one tenant's actors never read another tenant's Secrets. The provider watches namespaces and matches each request against the namespace's labels at that moment: a namespace created, labeled, unlabeled or deleted after startup is granted or refused without a restart. A secretSelector narrows a selected grant as it narrows a listed one. An empty selector, or an ownNamespaceSelector combined with atespace or allowedNamespaces, fails loading. The provider starts the namespace watch only when a policy selects namespaces, so a policy file without one parses and behaves as before. The provider's ClusterRole gains list and watch on namespaces, and /statusz lists the namespaces the selectors match now. Signed-off-by: Jose Armesto --- .../kubernetes-secrets/main.go | 3 + .../kubernetes-secrets/nsauthz.go | 195 ++++++++++-- .../kubernetes-secrets/nsauthz_test.go | 279 ++++++++++++++++++ .../kubernetes-secrets/statusz.go | 5 +- docs/egress-credential-injection.md | 25 +- .../k8s-credential-provider.yaml | 4 + .../namespace-policy.yaml | 7 +- 7 files changed, 485 insertions(+), 33 deletions(-) create mode 100644 cmd/credential-provider/kubernetes-secrets/nsauthz_test.go diff --git a/cmd/credential-provider/kubernetes-secrets/main.go b/cmd/credential-provider/kubernetes-secrets/main.go index 544fff4ceb..c05f8898bb 100644 --- a/cmd/credential-provider/kubernetes-secrets/main.go +++ b/cmd/credential-provider/kubernetes-secrets/main.go @@ -111,6 +111,9 @@ func run(ctx context.Context) error { return fmt.Errorf("namespace policy: %w", err) } slog.InfoContext(ctx, "loaded namespace authorization policy", slog.String("file", *nsPolicyFile)) + if err := nsAuth.WatchNamespaces(ctx, client); err != nil { + return fmt.Errorf("namespace policy: %w", err) + } creds, err := buildServerCreds(ctx) if err != nil { diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz.go b/cmd/credential-provider/kubernetes-secrets/nsauthz.go index 2b7cee3115..624b60fc38 100644 --- a/cmd/credential-provider/kubernetes-secrets/nsauthz.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz.go @@ -15,11 +15,20 @@ package main import ( + "context" "fmt" "os" + "slices" "sort" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" "sigs.k8s.io/yaml" ) @@ -33,6 +42,13 @@ type namespacePolicyFile struct { type atespaceNamespacePolicy struct { Atespace string `json:"atespace"` AllowedNamespaces []string `json:"allowedNamespaces"` + // OwnNamespaceSelector, when set, takes the place of atespace and + // allowedNamespaces: every namespace whose labels it matches may be read + // by the atespace of the same name, and by no other. The namespaces are + // matched against their labels at the time of each request, so one that is + // created, labeled or unlabeled after startup is granted or refused + // without a restart. It must name at least one requirement. + OwnNamespaceSelector *metav1.LabelSelector `json:"ownNamespaceSelector,omitempty"` // SecretSelector, when set, narrows the grant to Secrets in every allowed // namespace that carry all of its labels. // @@ -67,6 +83,13 @@ type grant struct { labels map[string]string } +// selectorGrant is the grant a namespace selector gives in each namespace it +// matches. +type selectorGrant struct { + selector labels.Selector + grant grant +} + // NamespaceAuthorizer decides whether an atespace may resolve secrets in a given // Kubernetes namespace. It is default-deny: an atespace absent from the mapping // can resolve nothing. @@ -76,6 +99,11 @@ type NamespaceAuthorizer struct { // when any one of them admits it. Within a grant the criteria are AND-ed, // so every label in one selector must be present. allowed map[string]map[string][]grant + // ownNamespace holds the ownNamespaceSelector policies, OR-ed with allowed. + ownNamespace []selectorGrant + // namespaces is the cache the selectors are matched against. + // WatchNamespaces sets it; until then those policies grant nothing. + namespaces corelisters.NamespaceLister } // LoadNamespaceAuthorizer reads the YAML policy file at path and builds an @@ -96,50 +124,167 @@ func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { } // newNamespaceAuthorizer builds an authorizer over a parsed policy file, -// validating that each grant names an atespace. +// validating that each grant names an atespace or selects namespaces. func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, error) { allowed := make(map[string]map[string][]grant) + var ownNamespace []selectorGrant for i, p := range file.Policies { + if p.OwnNamespaceSelector != nil { + if p.Atespace != "" || len(p.AllowedNamespaces) != 0 { + return nil, fmt.Errorf("namespace policy %d: ownNamespaceSelector cannot be combined with atespace or allowedNamespaces", i) + } + g, err := newGrant(i, p.SecretSelector) + if err != nil { + return nil, err + } + sg, err := newSelectorGrant(i, p.OwnNamespaceSelector, g) + if err != nil { + return nil, err + } + ownNamespace = append(ownNamespace, sg) + continue + } if p.Atespace == "" { return nil, fmt.Errorf("namespace policy %d: atespace is required", i) } - // A malformed label can never match, so it would narrow the grant to - // nothing and look like the policy was simply ignored. Fail loading - // instead. - for k, v := range p.SecretSelector.labels() { - if len(validation.IsQualifiedName(k)) != 0 { - return nil, fmt.Errorf("namespace policy %d: invalid label key %q", i, k) - } - if len(validation.IsValidLabelValue(v)) != 0 { - return nil, fmt.Errorf("namespace policy %d: invalid label value %q for key %q", i, v, k) - } + g, err := newGrant(i, p.SecretSelector) + if err != nil { + return nil, err } set := allowed[p.Atespace] if set == nil { set = make(map[string][]grant) allowed[p.Atespace] = set } - // Each policy keeps its own grant. Merging them into one would AND the - // selectors of separate policies together, and would let a narrow policy - // take away the namespace a broad one granted. - g := grant{labels: map[string]string{}} - for k, v := range p.SecretSelector.labels() { - g.labels[k] = v - } for _, ns := range p.AllowedNamespaces { set[ns] = append(set[ns], g) } } - return &NamespaceAuthorizer{allowed: allowed}, nil + return &NamespaceAuthorizer{allowed: allowed, ownNamespace: ownNamespace}, nil +} + +// newSelectorGrant gives grant g of policy i in the namespaces its selector +// matches. +func newSelectorGrant(i int, ls *metav1.LabelSelector, g grant) (selectorGrant, error) { + selector, err := metav1.LabelSelectorAsSelector(ls) + if err != nil { + return selectorGrant{}, fmt.Errorf("namespace policy %d: invalid ownNamespaceSelector: %w", i, err) + } + // An empty selector matches every namespace. Ask for that by label. + if selector.Empty() { + return selectorGrant{}, fmt.Errorf("namespace policy %d: ownNamespaceSelector must name at least one requirement", i) + } + return selectorGrant{selector: selector, grant: g}, nil +} + +// newGrant builds the grant of policy i. Each policy keeps its own grant. +// Merging them into one would AND the selectors of separate policies together, +// and would let a narrow policy take away the namespace a broad one granted. +func newGrant(i int, s *secretSelector) (grant, error) { + g := grant{labels: map[string]string{}} + for k, v := range s.labels() { + // A malformed label can never match, so it would narrow the grant to + // nothing and look like the policy was simply ignored. Fail loading + // instead. + if len(validation.IsQualifiedName(k)) != 0 { + return grant{}, fmt.Errorf("namespace policy %d: invalid label key %q", i, k) + } + if len(validation.IsValidLabelValue(v)) != 0 { + return grant{}, fmt.Errorf("namespace policy %d: invalid label value %q for key %q", i, v, k) + } + g.labels[k] = v + } + return g, nil } -// Grants returns the loaded policy as atespace → sorted namespaces +// namespaceSyncTimeout bounds the first list of namespaces, so a provider that +// cannot list them, for want of RBAC say, exits instead of never serving. +const namespaceSyncTimeout = time.Minute + +// WatchNamespaces starts the namespace cache ownNamespaceSelector policies are +// matched against, for as long as ctx lives, and waits for its first list. +// It does nothing when no policy selects namespaces, so a policy file without +// one needs no access to namespaces. +func (a *NamespaceAuthorizer) WatchNamespaces(ctx context.Context, client kubernetes.Interface) error { + if len(a.ownNamespace) == 0 { + return nil + } + factory := informers.NewSharedInformerFactory(client, 0) + namespaces := factory.Core().V1().Namespaces() + informer := namespaces.Informer() + factory.Start(ctx.Done()) + syncCtx, cancel := context.WithTimeout(ctx, namespaceSyncTimeout) + defer cancel() + if !cache.WaitForCacheSync(syncCtx.Done(), informer.HasSynced) { + return fmt.Errorf("namespace cache did not sync within %s", namespaceSyncTimeout) + } + a.namespaces = namespaces.Lister() + return nil +} + +// grantsFor returns the grants atespace holds in namespace: those the policy +// lists and, for the namespace of the atespace's own name, those of the +// ownNamespaceSelector policies it matches now. The cache holds the labels of +// its last watch event: while it cannot watch, a label change is not seen. +func (a *NamespaceAuthorizer) grantsFor(atespace, namespace string) []grant { + if a == nil { + return nil + } + grants := slices.Clip(a.allowed[atespace][namespace]) + if atespace == namespace { + grants = append(grants, a.matching(namespace, a.ownNamespace)...) + } + return grants +} + +// matching returns the grants of those selectors the namespace's labels match +// now. A namespace the cache does not hold matches none. +func (a *NamespaceAuthorizer) matching(namespace string, selectors []selectorGrant) []grant { + if a.namespaces == nil || len(selectors) == 0 { + return nil + } + ns, err := a.namespaces.Get(namespace) + if err != nil { + return nil + } + set := labels.Set(ns.GetLabels()) + var grants []grant + for _, sg := range selectors { + if sg.selector.Matches(set) { + grants = append(grants, sg.grant) + } + } + return grants +} + +// Grants returns the policy in force as atespace → sorted namespaces, +// including each namespace a namespace selector matches now. func (a *NamespaceAuthorizer) Grants() map[string][]string { if a == nil { return nil } - out := make(map[string][]string, len(a.allowed)) + sets := make(map[string]map[string]bool, len(a.allowed)) + add := func(atespace, namespace string) { + if sets[atespace] == nil { + sets[atespace] = make(map[string]bool) + } + sets[atespace][namespace] = true + } for atespace, namespaces := range a.allowed { + for ns := range namespaces { + add(atespace, ns) + } + } + if a.namespaces != nil { + all, _ := a.namespaces.List(labels.Everything()) + for _, ns := range all { + if len(a.matching(ns.Name, a.ownNamespace)) > 0 { + add(ns.Name, ns.Name) + } + } + } + out := make(map[string][]string, len(sets)) + for atespace, namespaces := range sets { list := make([]string, 0, len(namespaces)) for ns := range namespaces { list = append(list, ns) @@ -154,11 +299,7 @@ func (a *NamespaceAuthorizer) Grants() map[string][]string { // deny: an atespace absent from the mapping, or a namespace not in its list, is // refused. func (a *NamespaceAuthorizer) Allowed(atespace, namespace string) bool { - set, ok := a.allowed[atespace] - if !ok { - return false - } - return len(set[namespace]) > 0 + return len(a.grantsFor(atespace, namespace)) > 0 } // AllowedSecret reports whether atespace may resolve a Secret with these @@ -166,7 +307,7 @@ func (a *NamespaceAuthorizer) Allowed(atespace, namespace string) bool { // including one that does not exist: pass nil labels to ask that question // before reading anything from Kubernetes. func (a *NamespaceAuthorizer) AllowedSecret(atespace, namespace string, labels map[string]string) bool { - for _, g := range a.allowed[atespace][namespace] { + for _, g := range a.grantsFor(atespace, namespace) { if len(g.labels) == 0 { return true } diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go b/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go new file mode 100644 index 0000000000..b6019b98e3 --- /dev/null +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go @@ -0,0 +1,279 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "go.ate.dev/substrate/pkg/proto/credproviderpb" +) + +const tenantLabel = "example.com/tenant" + +// tenantPolicy is the policy file of a platform whose tenants each run their +// actors in the atespace named after their namespace, next to a static grant. +const tenantPolicy = `policies: +- atespace: platform + allowedNamespaces: [platform] +- ownNamespaceSelector: + matchExpressions: + - key: example.com/tenant + operator: Exists +` + +func namespace(name string, labels map[string]string) *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}} +} + +func tokenSecret(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "model-key", Namespace: ns}, + Data: map[string][]byte{"token": []byte("key-of-" + ns)}, + } +} + +// watchedClient returns a fake clientset whose namespace watch is known to be +// open once the returned channel closes: the fake drops what changes between +// an informer's list and its watch. +func watchedClient(objects ...runtime.Object) (*fake.Clientset, <-chan struct{}) { + client := fake.NewClientset(objects...) + started := make(chan struct{}) + var once sync.Once + client.PrependWatchReactor("namespaces", func(action k8stesting.Action) (bool, watch.Interface, error) { + w, err := client.Tracker().Watch(action.GetResource(), action.GetNamespace()) + if err == nil { + once.Do(func() { close(started) }) + } + return true, w, err + }) + return client, started +} + +// loadPolicy loads a policy file the way the provider does at startup. +func loadPolicy(t *testing.T, policy string) *NamespaceAuthorizer { + t.Helper() + path := filepath.Join(t.TempDir(), "policy.yaml") + if err := os.WriteFile(path, []byte(policy), 0o600); err != nil { + t.Fatal(err) + } + authz, err := LoadNamespaceAuthorizer(path) + if err != nil { + t.Fatalf("LoadNamespaceAuthorizer: %v", err) + } + return authz +} + +func fetch(srv *Server, atespace, ns string) (string, codes.Code) { + resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/" + ns + "/model-key/token", + ActorSpiffeId: "spiffe://substrate-actor.local/actor/" + atespace + "/agent", + }) + return string(resp.GetOpaqueBytes()), status.Code(err) +} + +// tenantFixture is a provider over a cluster of tenant namespaces: org-a and +// org-b labeled as tenants, platform and org-c not, and a +// Secret in each of them and in org-d, which does not exist yet. +type tenantFixture struct { + client *fake.Clientset + authz *NamespaceAuthorizer + srv *Server +} + +func newTenantFixture(t *testing.T, policy string) *tenantFixture { + t.Helper() + tenant := map[string]string{tenantLabel: "a"} + client, watchStarted := watchedClient( + namespace("platform", nil), namespace("org-a", tenant), namespace("org-b", map[string]string{tenantLabel: "b"}), + namespace("org-c", nil), + tokenSecret("platform"), tokenSecret("org-a"), tokenSecret("org-b"), tokenSecret("org-c"), tokenSecret("org-d"), + ) + authz := loadPolicy(t, policy) + if err := authz.WatchNamespaces(t.Context(), client); err != nil { + t.Fatalf("WatchNamespaces: %v", err) + } + select { + case <-watchStarted: + case <-time.After(10 * time.Second): + t.Fatal("the namespace watch never opened") + } + return &tenantFixture{client: client, authz: authz, srv: NewServer(client, authz)} +} + +func (f *tenantFixture) expect(t *testing.T, atespace, ns string, want codes.Code) { + t.Helper() + value, code := fetch(f.srv, atespace, ns) + if code != want { + t.Fatalf("atespace %s reading %s: code %v, want %v", atespace, ns, code, want) + } + if want == codes.OK && value != "key-of-"+ns { + t.Fatalf("atespace %s reading %s: got %q", atespace, ns, value) + } +} + +func (f *tenantFixture) eventually(t *testing.T, atespace, ns string, want codes.Code) { + t.Helper() + if err := wait.PollUntilContextTimeout(t.Context(), 10*time.Millisecond, 10*time.Second, true, func(context.Context) (bool, error) { + _, code := fetch(f.srv, atespace, ns) + return code == want, nil + }); err != nil { + t.Fatalf("atespace %s reading %s never answered %v: %v", atespace, ns, want, err) + } +} + +func (f *tenantFixture) relabel(t *testing.T, name string, labels map[string]string) { + t.Helper() + if _, err := f.client.CoreV1().Namespaces().Update(t.Context(), namespace(name, labels), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } +} + +// A namespace the selector matches is readable by the atespace of its own name +// and by no other, and the match follows the namespace's labels after startup. +func TestFetchSecretOwnNamespaceSelector(t *testing.T) { + f := newTenantFixture(t, tenantPolicy) + client, authz := f.client, f.authz + expect, eventually, relabel := f.expect, f.eventually, f.relabel + + t.Run("a matched namespace reads its own Secrets", func(t *testing.T) { + expect(t, "org-a", "org-a", codes.OK) + }) + t.Run("a matched namespace cannot read another tenant's", func(t *testing.T) { + before := len(client.Actions()) + expect(t, "org-a", "org-b", codes.PermissionDenied) + if slices.ContainsFunc(client.Actions()[before:], func(a k8stesting.Action) bool { return a.GetResource().Resource == "secrets" }) { + t.Fatal("a cross-tenant request reached the Secrets API") + } + }) + t.Run("a static atespace gains no matched namespace", func(t *testing.T) { + expect(t, "platform", "platform", codes.OK) + expect(t, "platform", "org-a", codes.PermissionDenied) + }) + t.Run("an unlabeled namespace is not granted", func(t *testing.T) { + expect(t, "org-c", "org-c", codes.PermissionDenied) + }) + t.Run("a namespace labeled after startup is granted", func(t *testing.T) { + relabel(t, "org-c", map[string]string{tenantLabel: "c"}) + eventually(t, "org-c", "org-c", codes.OK) + if got := authz.Grants()["org-c"]; !slices.Equal(got, []string{"org-c"}) { + t.Fatalf("Grants()[org-c] = %v, want [org-c]", got) + } + }) + t.Run("a namespace created after startup is granted", func(t *testing.T) { + if _, err := client.CoreV1().Namespaces().Create(t.Context(), namespace("org-d", map[string]string{tenantLabel: "d"}), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + eventually(t, "org-d", "org-d", codes.OK) + }) + t.Run("a namespace whose label is removed is refused", func(t *testing.T) { + relabel(t, "org-a", nil) + eventually(t, "org-a", "org-a", codes.PermissionDenied) + }) + t.Run("a deleted namespace is refused", func(t *testing.T) { + if err := client.CoreV1().Namespaces().Delete(t.Context(), "org-b", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + eventually(t, "org-b", "org-b", codes.PermissionDenied) + }) +} + +// A policy file without a namespace selector must not need RBAC on +// namespaces: the provider would otherwise fail to start on every install +// that does not grant it. +func TestWatchNamespacesWithoutSelectorReadsNothing(t *testing.T) { + client := fake.NewClientset() + client.PrependReactor("*", "namespaces", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("forbidden") + }) + authz := loadPolicy(t, "policies:\n- atespace: team-a\n allowedNamespaces: [ns1]\n") + if err := authz.WatchNamespaces(t.Context(), client); err != nil { + t.Fatalf("WatchNamespaces: %v", err) + } + if actions := client.Actions(); len(actions) != 0 { + t.Fatalf("a policy without a namespace selector called the API: %v", actions) + } +} + +func TestLoadNamespaceAuthorizerNamespaceSelectors(t *testing.T) { + for _, tc := range []struct { + name, policy string + }{ + {"with atespace", "policies:\n- atespace: team-a\n ownNamespaceSelector: {matchLabels: {t: a}}\n"}, + {"with allowedNamespaces", "policies:\n- allowedNamespaces: [ns1]\n ownNamespaceSelector: {matchLabels: {t: a}}\n"}, + {"empty selector", "policies:\n- ownNamespaceSelector: {}\n"}, + {"unknown operator", "policies:\n- ownNamespaceSelector: {matchExpressions: [{key: t, operator: Present}]}\n"}, + {"invalid key", "policies:\n- ownNamespaceSelector: {matchLabels: {'not a key': a}}\n"}, + {"misspelled field", "policies:\n- ownNamespaceSelector: {matchLabel: {t: a}}\n"}, + {"invalid secretSelector", "policies:\n- ownNamespaceSelector: {matchLabels: {t: a}}\n secretSelector: {matchLabels: {'not a key': v}}\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "policy.yaml") + if err := os.WriteFile(path, []byte(tc.policy), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadNamespaceAuthorizer(path); err == nil { + t.Fatal("policy accepted, want an error") + } + }) + } +} + +// A secretSelector narrows a selected namespace's grant as it narrows a +// listed one. +func TestFetchSecretNamespaceSelectorNarrowsBySecretLabel(t *testing.T) { + const narrow = " secretSelector:\n matchLabels:\n example.com/credential: \"true\"\n" + for _, tc := range []struct{ name, policy, atespace string }{ + {"ownNamespaceSelector", tenantPolicy + narrow, "org-a"}, + } { + t.Run(tc.name, func(t *testing.T) { + labeled := tokenSecret("org-a") + labeled.Labels = map[string]string{"example.com/credential": "true"} + other := tokenSecret("org-a") + other.Name = "ca-pool" + client := fake.NewClientset(namespace("org-a", map[string]string{tenantLabel: "a"}), labeled, other) + authz := loadPolicy(t, tc.policy) + if err := authz.WatchNamespaces(t.Context(), client); err != nil { + t.Fatalf("WatchNamespaces: %v", err) + } + srv := NewServer(client, authz) + if _, code := fetch(srv, tc.atespace, "org-a"); code != codes.OK { + t.Fatalf("labeled Secret: code %v, want OK", code) + } + if _, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/org-a/ca-pool/token", + ActorSpiffeId: "spiffe://substrate-actor.local/actor/" + tc.atespace + "/agent", + }); status.Code(err) != codes.PermissionDenied { + t.Fatalf("unlabeled Secret: %v, want PermissionDenied", err) + } + }) + } +} diff --git a/cmd/credential-provider/kubernetes-secrets/statusz.go b/cmd/credential-provider/kubernetes-secrets/statusz.go index df586ff021..b510feca20 100644 --- a/cmd/credential-provider/kubernetes-secrets/statusz.go +++ b/cmd/credential-provider/kubernetes-secrets/statusz.go @@ -16,7 +16,8 @@ // serving an HTML dashboard by default and JSON under ?format=json (or // Accept: application/json). It shows the provider's configuration as the // running process holds it — notably the authorization policy loaded at -// startup, which the ConfigMap on disk may have since drifted from. Names +// startup, which the ConfigMap on disk may have since drifted from, with the +// namespaces its namespace selectors match now. Names // and configuration only, never secret values. package main @@ -88,7 +89,7 @@ provider: {{.ProviderName}}
required caller SAN: {{.InjectorSAN}}
args: {{.Args}}

-

Namespace policy (as loaded at startup)

+

Namespace policy (as loaded at startup, namespace selectors as matched now)

{{if .PolicyGrants}} diff --git a/docs/egress-credential-injection.md b/docs/egress-credential-injection.md index 5177ac18fa..8c3a84f233 100644 --- a/docs/egress-credential-injection.md +++ b/docs/egress-credential-injection.md @@ -158,7 +158,7 @@ it alongside the gateway. The install applies: * the provider's ServiceAccount, its Deployment and its Service in `ate-system`; * a ClusterRole and ClusterRoleBinding that let it read Secrets in every - namespace; + namespace, and list and watch namespaces; * the `k8s-credential-provider-namespace-policy` ConfigMap, only when it is absent (see below); and * a NetworkPolicy that admits only the egress gateway's pods @@ -190,8 +190,27 @@ kubectl -n ate-system edit configmap k8s-credential-provider-namespace-policy - ns1 ``` -The provider loads the policy **once at startup** and does not yet reload it, -so after editing, restart the provider: +A policy can carry an `ownNamespaceSelector` (a Kubernetes label selector) in +place of `atespace` and `allowedNamespaces`. Each namespace it matches may be +read by the atespace of the same name, and by no other, which suits a platform +that creates one namespace and one atespace per tenant at run time: + +```yaml + namespace-policy.yaml: | + policies: + - ownNamespaceSelector: + matchExpressions: + - key: example.com/tenant + operator: Exists +``` + +The provider watches namespaces only when a policy has one, and matches each +request against the namespace's current labels, so a namespace created, +labeled, unlabeled or deleted later is granted or refused without a restart. +A `secretSelector` narrows a selected grant as it narrows a listed one. + +The provider loads the policy file itself **once at startup** and does not yet +reload it, so after editing, restart the provider: ```bash kubectl -n ate-system rollout restart deployment/k8s-credential-provider diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 90de2c24c7..11fd5e975f 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -33,6 +33,10 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] +# Watched only when a namespace policy has an ownNamespaceSelector. +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/manifests/egress-credential-injection/namespace-policy.yaml b/manifests/egress-credential-injection/namespace-policy.yaml index e90b9c6c32..c53ca0118a 100644 --- a/manifests/egress-credential-injection/namespace-policy.yaml +++ b/manifests/egress-credential-injection/namespace-policy.yaml @@ -13,7 +13,8 @@ # limitations under the License. # The atespace→namespace authorization policy k8s-credential-provider enforces: an actor's -# atespace may only resolve Secrets whose URI namespace is listed for it here. +# atespace may only resolve Secrets whose URI namespace is listed for it here, +# or whose namespace an ownNamespaceSelector matches and is named after it. # Enforcement is default-deny — an atespace absent from this file resolves # nothing. The install creates this ConfigMap only when it is absent, granting # nothing; the provider needs it to start. @@ -33,4 +34,8 @@ data: # - atespace: team-a # allowedNamespaces: # - ns1 + # - ownNamespaceSelector: + # matchExpressions: + # - key: example.com/tenant + # operator: Exists policies: [] From 5d7bdf3d4e3fadb4694f0346307af0c68bcdee16 Mon Sep 17 00:00:00 2001 From: QuentinBisson Date: Mon, 19 Oct 2026 12:45:01 +0200 Subject: [PATCH 2/2] refactor(credential-provider): simplify namespace selector matching Pin that combining static and selector grants never writes into the policy's shared slice. --- .../kubernetes-secrets/nsauthz.go | 26 ++-- .../kubernetes-secrets/nsauthz_test.go | 117 +++++++++++------- 2 files changed, 83 insertions(+), 60 deletions(-) diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz.go b/cmd/credential-provider/kubernetes-secrets/nsauthz.go index 624b60fc38..f6cc9840d8 100644 --- a/cmd/credential-provider/kubernetes-secrets/nsauthz.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz.go @@ -114,9 +114,9 @@ func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { return nil, fmt.Errorf("reading namespace policy file %q: %w", path, err) } var file namespacePolicyFile - // Strict: a typo like matchLabel, or an unsupported field like - // matchExpressions, must fail rather than silently parse as an empty - // selector, which would grant every Secret in the namespace. + // Strict: a typo like matchLabel, or a field secretSelector does not + // support like matchExpressions, must fail rather than silently parse as + // an empty selector, which would grant every Secret in the namespace. if err := yaml.UnmarshalStrict(data, &file); err != nil { return nil, fmt.Errorf("parsing namespace policy file %q: %w", path, err) } @@ -227,20 +227,20 @@ func (a *NamespaceAuthorizer) WatchNamespaces(ctx context.Context, client kubern // ownNamespaceSelector policies it matches now. The cache holds the labels of // its last watch event: while it cannot watch, a label change is not seen. func (a *NamespaceAuthorizer) grantsFor(atespace, namespace string) []grant { - if a == nil { - return nil - } + // Clipped so the append below allocates instead of writing into the + // policy's own slice, which concurrent requests share. grants := slices.Clip(a.allowed[atespace][namespace]) if atespace == namespace { - grants = append(grants, a.matching(namespace, a.ownNamespace)...) + grants = append(grants, a.matching(namespace)...) } return grants } -// matching returns the grants of those selectors the namespace's labels match -// now. A namespace the cache does not hold matches none. -func (a *NamespaceAuthorizer) matching(namespace string, selectors []selectorGrant) []grant { - if a.namespaces == nil || len(selectors) == 0 { +// matching returns the grants of the ownNamespaceSelector policies the +// namespace's labels match now. A namespace the cache does not hold matches +// none. +func (a *NamespaceAuthorizer) matching(namespace string) []grant { + if a.namespaces == nil || len(a.ownNamespace) == 0 { return nil } ns, err := a.namespaces.Get(namespace) @@ -249,7 +249,7 @@ func (a *NamespaceAuthorizer) matching(namespace string, selectors []selectorGra } set := labels.Set(ns.GetLabels()) var grants []grant - for _, sg := range selectors { + for _, sg := range a.ownNamespace { if sg.selector.Matches(set) { grants = append(grants, sg.grant) } @@ -278,7 +278,7 @@ func (a *NamespaceAuthorizer) Grants() map[string][]string { if a.namespaces != nil { all, _ := a.namespaces.List(labels.Everything()) for _, ns := range all { - if len(a.matching(ns.Name, a.ownNamespace)) > 0 { + if len(a.matching(ns.Name)) > 0 { add(ns.Name, ns.Name) } } diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go b/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go index b6019b98e3..32618d6dba 100644 --- a/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz_test.go @@ -92,9 +92,13 @@ func loadPolicy(t *testing.T, policy string) *NamespaceAuthorizer { return authz } -func fetch(srv *Server, atespace, ns string) (string, codes.Code) { - resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{ - Uri: "ate-secret://k8s.io/default/" + ns + "/model-key/token", +func fetch(t *testing.T, srv *Server, atespace, ns string) (string, codes.Code) { + return fetchNamed(t, srv, atespace, ns, "model-key") +} + +func fetchNamed(t *testing.T, srv *Server, atespace, ns, name string) (string, codes.Code) { + resp, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/" + ns + "/" + name + "/token", ActorSpiffeId: "spiffe://substrate-actor.local/actor/" + atespace + "/agent", }) return string(resp.GetOpaqueBytes()), status.Code(err) @@ -131,7 +135,7 @@ func newTenantFixture(t *testing.T, policy string) *tenantFixture { func (f *tenantFixture) expect(t *testing.T, atespace, ns string, want codes.Code) { t.Helper() - value, code := fetch(f.srv, atespace, ns) + value, code := fetch(t, f.srv, atespace, ns) if code != want { t.Fatalf("atespace %s reading %s: code %v, want %v", atespace, ns, code, want) } @@ -143,7 +147,7 @@ func (f *tenantFixture) expect(t *testing.T, atespace, ns string, want codes.Cod func (f *tenantFixture) eventually(t *testing.T, atespace, ns string, want codes.Code) { t.Helper() if err := wait.PollUntilContextTimeout(t.Context(), 10*time.Millisecond, 10*time.Second, true, func(context.Context) (bool, error) { - _, code := fetch(f.srv, atespace, ns) + _, code := fetch(t, f.srv, atespace, ns) return code == want, nil }); err != nil { t.Fatalf("atespace %s reading %s never answered %v: %v", atespace, ns, want, err) @@ -161,48 +165,46 @@ func (f *tenantFixture) relabel(t *testing.T, name string, labels map[string]str // and by no other, and the match follows the namespace's labels after startup. func TestFetchSecretOwnNamespaceSelector(t *testing.T) { f := newTenantFixture(t, tenantPolicy) - client, authz := f.client, f.authz - expect, eventually, relabel := f.expect, f.eventually, f.relabel t.Run("a matched namespace reads its own Secrets", func(t *testing.T) { - expect(t, "org-a", "org-a", codes.OK) + f.expect(t, "org-a", "org-a", codes.OK) }) t.Run("a matched namespace cannot read another tenant's", func(t *testing.T) { - before := len(client.Actions()) - expect(t, "org-a", "org-b", codes.PermissionDenied) - if slices.ContainsFunc(client.Actions()[before:], func(a k8stesting.Action) bool { return a.GetResource().Resource == "secrets" }) { + before := len(f.client.Actions()) + f.expect(t, "org-a", "org-b", codes.PermissionDenied) + if slices.ContainsFunc(f.client.Actions()[before:], func(a k8stesting.Action) bool { return a.GetResource().Resource == "secrets" }) { t.Fatal("a cross-tenant request reached the Secrets API") } }) t.Run("a static atespace gains no matched namespace", func(t *testing.T) { - expect(t, "platform", "platform", codes.OK) - expect(t, "platform", "org-a", codes.PermissionDenied) + f.expect(t, "platform", "platform", codes.OK) + f.expect(t, "platform", "org-a", codes.PermissionDenied) }) t.Run("an unlabeled namespace is not granted", func(t *testing.T) { - expect(t, "org-c", "org-c", codes.PermissionDenied) + f.expect(t, "org-c", "org-c", codes.PermissionDenied) }) t.Run("a namespace labeled after startup is granted", func(t *testing.T) { - relabel(t, "org-c", map[string]string{tenantLabel: "c"}) - eventually(t, "org-c", "org-c", codes.OK) - if got := authz.Grants()["org-c"]; !slices.Equal(got, []string{"org-c"}) { + f.relabel(t, "org-c", map[string]string{tenantLabel: "c"}) + f.eventually(t, "org-c", "org-c", codes.OK) + if got := f.authz.Grants()["org-c"]; !slices.Equal(got, []string{"org-c"}) { t.Fatalf("Grants()[org-c] = %v, want [org-c]", got) } }) t.Run("a namespace created after startup is granted", func(t *testing.T) { - if _, err := client.CoreV1().Namespaces().Create(t.Context(), namespace("org-d", map[string]string{tenantLabel: "d"}), metav1.CreateOptions{}); err != nil { + if _, err := f.client.CoreV1().Namespaces().Create(t.Context(), namespace("org-d", map[string]string{tenantLabel: "d"}), metav1.CreateOptions{}); err != nil { t.Fatal(err) } - eventually(t, "org-d", "org-d", codes.OK) + f.eventually(t, "org-d", "org-d", codes.OK) }) t.Run("a namespace whose label is removed is refused", func(t *testing.T) { - relabel(t, "org-a", nil) - eventually(t, "org-a", "org-a", codes.PermissionDenied) + f.relabel(t, "org-a", nil) + f.eventually(t, "org-a", "org-a", codes.PermissionDenied) }) t.Run("a deleted namespace is refused", func(t *testing.T) { - if err := client.CoreV1().Namespaces().Delete(t.Context(), "org-b", metav1.DeleteOptions{}); err != nil { + if err := f.client.CoreV1().Namespaces().Delete(t.Context(), "org-b", metav1.DeleteOptions{}); err != nil { t.Fatal(err) } - eventually(t, "org-b", "org-b", codes.PermissionDenied) + f.eventually(t, "org-b", "org-b", codes.PermissionDenied) }) } @@ -249,31 +251,52 @@ func TestLoadNamespaceAuthorizerNamespaceSelectors(t *testing.T) { // A secretSelector narrows a selected namespace's grant as it narrows a // listed one. -func TestFetchSecretNamespaceSelectorNarrowsBySecretLabel(t *testing.T) { - const narrow = " secretSelector:\n matchLabels:\n example.com/credential: \"true\"\n" - for _, tc := range []struct{ name, policy, atespace string }{ - {"ownNamespaceSelector", tenantPolicy + narrow, "org-a"}, - } { - t.Run(tc.name, func(t *testing.T) { - labeled := tokenSecret("org-a") - labeled.Labels = map[string]string{"example.com/credential": "true"} - other := tokenSecret("org-a") - other.Name = "ca-pool" - client := fake.NewClientset(namespace("org-a", map[string]string{tenantLabel: "a"}), labeled, other) - authz := loadPolicy(t, tc.policy) - if err := authz.WatchNamespaces(t.Context(), client); err != nil { - t.Fatalf("WatchNamespaces: %v", err) - } - srv := NewServer(client, authz) - if _, code := fetch(srv, tc.atespace, "org-a"); code != codes.OK { - t.Fatalf("labeled Secret: code %v, want OK", code) - } - if _, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ - Uri: "ate-secret://k8s.io/default/org-a/ca-pool/token", - ActorSpiffeId: "spiffe://substrate-actor.local/actor/" + tc.atespace + "/agent", - }); status.Code(err) != codes.PermissionDenied { - t.Fatalf("unlabeled Secret: %v, want PermissionDenied", err) +func TestFetchSecretOwnNamespaceSelectorNarrowsBySecretLabel(t *testing.T) { + labeled := tokenSecret("org-a") + labeled.Labels = map[string]string{"example.com/credential": "true"} + other := tokenSecret("org-a") + other.Name = "ca-pool" + client := fake.NewClientset(namespace("org-a", map[string]string{tenantLabel: "a"}), labeled, other) + authz := loadPolicy(t, tenantPolicy+" secretSelector:\n matchLabels:\n example.com/credential: \"true\"\n") + if err := authz.WatchNamespaces(t.Context(), client); err != nil { + t.Fatalf("WatchNamespaces: %v", err) + } + srv := NewServer(client, authz) + if _, code := fetch(t, srv, "org-a", "org-a"); code != codes.OK { + t.Fatalf("labeled Secret: code %v, want OK", code) + } + if _, code := fetchNamed(t, srv, "org-a", "org-a", "ca-pool"); code != codes.PermissionDenied { + t.Fatalf("unlabeled Secret: code %v, want PermissionDenied", code) + } +} + +// Static grants and a selector grant on the same atespace and namespace are +// combined per request without writing into the policy's slice, whose spare +// capacity three appends leave, so concurrent requests do not race. +func TestFetchSecretStaticAndSelectorGrantsConcurrently(t *testing.T) { + labeled := tokenSecret("org-a") + labeled.Labels = map[string]string{"example.com/credential": "true"} + client := fake.NewClientset(namespace("org-a", map[string]string{tenantLabel: "a"}), labeled) + authz := loadPolicy(t, "policies:\n"+ + "- atespace: org-a\n allowedNamespaces: [org-a]\n secretSelector: {matchLabels: {example.com/credential: \"true\"}}\n"+ + "- atespace: org-a\n allowedNamespaces: [org-a]\n secretSelector: {matchLabels: {example.com/other: \"true\"}}\n"+ + "- atespace: org-a\n allowedNamespaces: [org-a]\n secretSelector: {matchLabels: {example.com/another: \"true\"}}\n"+ + "- ownNamespaceSelector: {matchExpressions: [{key: example.com/tenant, operator: Exists}]}\n"+ + " secretSelector: {matchLabels: {example.com/third: \"true\"}}\n") + if err := authz.WatchNamespaces(t.Context(), client); err != nil { + t.Fatalf("WatchNamespaces: %v", err) + } + srv := NewServer(client, authz) + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + if _, code := fetch(t, srv, "org-a", "org-a"); code != codes.OK { + t.Errorf("code %v, want OK", code) } }) } + wg.Wait() + if got := authz.allowed["org-a"]["org-a"]; len(got) != 3 { + t.Fatalf("static grants for org-a/org-a = %d, want 3", len(got)) + } }
atespaceallowed namespaces