From 0423637355952d21694bedb46674ce073404e25b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 1 Oct 2026 20:13:33 -0500 Subject: [PATCH 1/8] feat: consume certificate service behind flag Custom hostname certificates can now come from the Milo certificate service instead of cert-manager on the downstream hub. The path is off by default; with gateway.certificateService.enabled the gateway controller requests a TLSCertificate per claimed hostname in the project control plane, answers its HTTP-01 challenges on the downstream gateway, and mirrors the issued Secret to the edge under the name the listener already references. Key changes: - Add gateway.certificateService {enabled, kubeconfigPath} with validation that hostname verification stays on, since the service issues for whatever name it is handed - Vendor certificates.miloapis.com/v1alpha1 TLSCertificate types under internal/certificates until the service module is importable - Extract HTTP-01 solver route creation so the Challenge path and the TLSCertificate path share it; serve only challenges whose dnsName is the hostname the listener claimed, and drop routes when the challenge leaves status - Mirror the edge Secret from the service-side Secret named in status.serviceSecretRef, read with the operator's own credentials, never from the project copy - Keep a valid cert-manager Certificate until its renewal time, then retire it and request a TLSCertificate, so turning the flag on issues nothing up front and never races a cert-manager renewal - Derive listener health and HTTPProxy CertificateReady from TLSCertificate conditions, with the listener kept serving on any valid downstream Secret - Watch TLSCertificates across project control planes when enabled Flag off is unchanged: the existing certificate tests pass untouched. Related to datum-cloud/enhancements#913 --- config/rbac/role.yaml | 18 + .../v1alpha1/groupversion_info.go | 29 + .../v1alpha1/tlscertificate_types.go | 130 ++++ .../v1alpha1/zz_generated.deepcopy.go | 205 ++++++ internal/cmd/manager/manager.go | 46 +- internal/config/config.go | 39 ++ internal/config/config_test.go | 51 ++ internal/config/zz_generated.deepcopy.go | 16 + .../controller/gateway_certificate_service.go | 647 ++++++++++++++++++ .../gateway_certificate_service_test.go | 504 ++++++++++++++ internal/controller/gateway_controller.go | 176 +++-- ...ownstream_certificate_solver_controller.go | 97 +-- internal/controller/http01_solver.go | 150 ++++ internal/controller/http01_solver_test.go | 70 ++ .../httpproxy_certificate_service.go | 139 ++++ .../httpproxy_certificate_service_test.go | 148 ++++ internal/controller/httpproxy_controller.go | 15 + 17 files changed, 2326 insertions(+), 154 deletions(-) create mode 100644 internal/certificates/v1alpha1/groupversion_info.go create mode 100644 internal/certificates/v1alpha1/tlscertificate_types.go create mode 100644 internal/certificates/v1alpha1/zz_generated.deepcopy.go create mode 100644 internal/controller/gateway_certificate_service.go create mode 100644 internal/controller/gateway_certificate_service_test.go create mode 100644 internal/controller/http01_solver.go create mode 100644 internal/controller/http01_solver_test.go create mode 100644 internal/controller/httpproxy_certificate_service.go create mode 100644 internal/controller/httpproxy_certificate_service_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d3aaeb88..f7e99f1c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -61,6 +61,24 @@ rules: - patch - update - watch +- apiGroups: + - certificates.miloapis.com + resources: + - tlscertificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.miloapis.com + resources: + - tlscertificates/status + verbs: + - get - apiGroups: - coordination.k8s.io resources: diff --git a/internal/certificates/v1alpha1/groupversion_info.go b/internal/certificates/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..e0bccfe9 --- /dev/null +++ b/internal/certificates/v1alpha1/groupversion_info.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package v1alpha1 carries the certificates.miloapis.com/v1alpha1 types the +// operator consumes from the Milo certificate service. It mirrors the +// service's API until its module is importable. +// +// +kubebuilder:object:generate=true +// +kubebuilder:skip +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + GroupVersion = schema.GroupVersion{Group: "certificates.miloapis.com", Version: "v1alpha1"} + + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, &TLSCertificate{}, &TLSCertificateList{}) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} diff --git a/internal/certificates/v1alpha1/tlscertificate_types.go b/internal/certificates/v1alpha1/tlscertificate_types.go new file mode 100644 index 00000000..2ab72524 --- /dev/null +++ b/internal/certificates/v1alpha1/tlscertificate_types.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IssuanceMode selects how the certificate authority validates control of the +// requested names. +type IssuanceMode string + +const ( + IssuanceModeAuto IssuanceMode = "Auto" + IssuanceModeHTTP01 IssuanceMode = "HTTP01" + IssuanceModeDNS01 IssuanceMode = "DNS01" +) + +// ChallengeType is a resolved issuance mode: the ACME challenge type used to +// validate the names. +type ChallengeType string + +const ( + ChallengeTypeHTTP01 ChallengeType = "HTTP01" + ChallengeTypeDNS01 ChallengeType = "DNS01" +) + +// DNSRecordPurpose explains why a DNS record must be published. +type DNSRecordPurpose string + +const ( + DNSRecordPurposeRouting DNSRecordPurpose = "Routing" + DNSRecordPurposeCertificate DNSRecordPurpose = "Certificate" +) + +// ChallengeState is the lifecycle state of an ACME challenge. +type ChallengeState string + +const ( + ChallengeStatePending ChallengeState = "Pending" + ChallengeStateValid ChallengeState = "Valid" + ChallengeStateInvalid ChallengeState = "Invalid" +) + +// Condition types reported on a TLSCertificate. +const ( + ConditionAccepted = "Accepted" + ConditionDNSDelegationReady = "DNSDelegationReady" + ConditionIssuing = "Issuing" + ConditionReady = "Ready" +) + +// TLSCertificateSpec defines the desired state of TLSCertificate. The service +// does not verify that the project controls the names; the caller creates a +// TLSCertificate only for names whose ownership it has already verified. +type TLSCertificateSpec struct { + DNSNames []DNSName `json:"dnsNames"` + Issuance IssuanceMode `json:"issuance,omitempty"` + SecretName string `json:"secretName,omitempty"` +} + +// DNSName is a lowercase RFC 1123 hostname, optionally prefixed with "*.". +type DNSName string + +// SecretReference names a Secret in the TLSCertificate's namespace. +type SecretReference struct { + Name string `json:"name"` +} + +// ServiceSecretReference locates the issued Secret on the cluster that runs +// the certificate service. +type ServiceSecretReference struct { + Namespace string `json:"namespace"` + Name string `json:"name"` +} + +// RequiredDNSRecord is a DNS record the name's owner must publish before +// issuance can complete. +type RequiredDNSRecord struct { + Name string `json:"name"` + Type string `json:"type"` + Content string `json:"content"` + Purpose DNSRecordPurpose `json:"purpose"` +} + +// ACMEChallenge is a live ACME challenge for one name. For HTTP01, the +// consumer serves GET /.well-known/acme-challenge/ with the body +// on dnsName. +type ACMEChallenge struct { + DNSName string `json:"dnsName"` + Type ChallengeType `json:"type"` + Token string `json:"token"` + Key string `json:"key"` + State ChallengeState `json:"state"` +} + +// TLSCertificateStatus defines the observed state of TLSCertificate. +type TLSCertificateStatus struct { + Issuance ChallengeType `json:"issuance,omitempty"` + SecretRef *SecretReference `json:"secretRef,omitempty"` + ServiceSecretRef *ServiceSecretReference `json:"serviceSecretRef,omitempty"` + DelegationTarget string `json:"delegationTarget,omitempty"` + NotBefore *metav1.Time `json:"notBefore,omitempty"` + NotAfter *metav1.Time `json:"notAfter,omitempty"` + RenewalTime *metav1.Time `json:"renewalTime,omitempty"` + RequiredDNSRecords []RequiredDNSRecord `json:"requiredDNSRecords,omitempty"` + Challenges []ACMEChallenge `json:"challenges,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +// TLSCertificate requests a publicly trusted TLS certificate for a set of +// hostnames and delivers it as a kubernetes.io/tls Secret in the same +// namespace. Its name is at most 63 characters. +// +// +kubebuilder:object:root=true +type TLSCertificate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TLSCertificateSpec `json:"spec"` + Status TLSCertificateStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type TLSCertificateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TLSCertificate `json:"items"` +} diff --git a/internal/certificates/v1alpha1/zz_generated.deepcopy.go b/internal/certificates/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..9729778b --- /dev/null +++ b/internal/certificates/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,205 @@ +//go:build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ACMEChallenge) DeepCopyInto(out *ACMEChallenge) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ACMEChallenge. +func (in *ACMEChallenge) DeepCopy() *ACMEChallenge { + if in == nil { + return nil + } + out := new(ACMEChallenge) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RequiredDNSRecord) DeepCopyInto(out *RequiredDNSRecord) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequiredDNSRecord. +func (in *RequiredDNSRecord) DeepCopy() *RequiredDNSRecord { + if in == nil { + return nil + } + out := new(RequiredDNSRecord) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretReference) DeepCopyInto(out *SecretReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference. +func (in *SecretReference) DeepCopy() *SecretReference { + if in == nil { + return nil + } + out := new(SecretReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceSecretReference) DeepCopyInto(out *ServiceSecretReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSecretReference. +func (in *ServiceSecretReference) DeepCopy() *ServiceSecretReference { + if in == nil { + return nil + } + out := new(ServiceSecretReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificate) DeepCopyInto(out *TLSCertificate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificate. +func (in *TLSCertificate) DeepCopy() *TLSCertificate { + if in == nil { + return nil + } + out := new(TLSCertificate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSCertificate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificateList) DeepCopyInto(out *TLSCertificateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TLSCertificate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificateList. +func (in *TLSCertificateList) DeepCopy() *TLSCertificateList { + if in == nil { + return nil + } + out := new(TLSCertificateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSCertificateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificateSpec) DeepCopyInto(out *TLSCertificateSpec) { + *out = *in + if in.DNSNames != nil { + in, out := &in.DNSNames, &out.DNSNames + *out = make([]DNSName, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificateSpec. +func (in *TLSCertificateSpec) DeepCopy() *TLSCertificateSpec { + if in == nil { + return nil + } + out := new(TLSCertificateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificateStatus) DeepCopyInto(out *TLSCertificateStatus) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(SecretReference) + **out = **in + } + if in.ServiceSecretRef != nil { + in, out := &in.ServiceSecretRef, &out.ServiceSecretRef + *out = new(ServiceSecretReference) + **out = **in + } + if in.NotBefore != nil { + in, out := &in.NotBefore, &out.NotBefore + *out = (*in).DeepCopy() + } + if in.NotAfter != nil { + in, out := &in.NotAfter, &out.NotAfter + *out = (*in).DeepCopy() + } + if in.RenewalTime != nil { + in, out := &in.RenewalTime, &out.RenewalTime + *out = (*in).DeepCopy() + } + if in.RequiredDNSRecords != nil { + in, out := &in.RequiredDNSRecords, &out.RequiredDNSRecords + *out = make([]RequiredDNSRecord, len(*in)) + copy(*out, *in) + } + if in.Challenges != nil { + in, out := &in.Challenges, &out.Challenges + *out = make([]ACMEChallenge, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificateStatus. +func (in *TLSCertificateStatus) DeepCopy() *TLSCertificateStatus { + if in == nil { + return nil + } + out := new(TLSCertificateStatus) + in.DeepCopyInto(out) + return out +} diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 439472de..19000086 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -44,6 +44,7 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" "go.datum.net/network-services-operator/internal/cmd/clusterdiscovery" "go.datum.net/network-services-operator/internal/config" "go.datum.net/network-services-operator/internal/controller" @@ -76,6 +77,7 @@ func init() { utilruntime.Must(cmv1.AddToScheme(scheme)) utilruntime.Must(dnsv1alpha1.AddToScheme(scheme)) utilruntime.Must(ipamv1alpha1.AddToScheme(scheme)) + utilruntime.Must(certificatesv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -384,11 +386,18 @@ func NewCommand(build BuildInfo) *cobra.Command { os.Exit(1) } + certificateServiceReader, err := newCertificateServiceReader(serverConfig.Gateway) + if err != nil { + setupLog.Error(err, "unable to build certificate service client") + os.Exit(1) + } + registeredControllers, err := setupControllers(mgr, serverConfig, controllerDeps{ - downstreamCluster: downstreamCluster, - singletonManager: singletonControllerMgr, - irohDownstream: irohDownstream, - ipamClients: ipamClients, + downstreamCluster: downstreamCluster, + singletonManager: singletonControllerMgr, + irohDownstream: irohDownstream, + ipamClients: ipamClients, + certificateServiceReader: certificateServiceReader, }) if err != nil { setupLog.Error(err, "unable to set up controllers") @@ -561,10 +570,26 @@ func setupWebhooks(mgr mcmanager.Manager, serverConfig config.NetworkServicesOpe // controllerDeps carries the clients and managers that controllers are wired // against. Fields are only populated for the sets that need them. type controllerDeps struct { - downstreamCluster cluster.Cluster - singletonManager manager.Manager - irohDownstream cluster.Cluster - ipamClients controller.IPAMClientFactory + downstreamCluster cluster.Cluster + singletonManager manager.Manager + irohDownstream cluster.Cluster + ipamClients controller.IPAMClientFactory + certificateServiceReader client.Reader +} + +// newCertificateServiceReader returns an uncached client for the cluster the +// certificate service keeps its issued Secrets on, or nil when the service is +// not consumed. Uncached so the operator never holds an informer over every +// Secret on that cluster. +func newCertificateServiceReader(gatewayConfig config.GatewayConfig) (client.Reader, error) { + if !gatewayConfig.CertificateService.Enabled { + return nil, nil + } + restConfig, err := gatewayConfig.CertificateService.RestConfig() + if err != nil { + return nil, fmt.Errorf("unable to load certificate service kubeconfig: %w", err) + } + return client.New(restConfig, client.Options{Scheme: scheme}) } // newIPAMClientFactory returns nil when no IPAM connection is configured. A @@ -680,8 +705,9 @@ func controllerRegistrations( }}, {"gateway", true, func() error { return (&controller.GatewayReconciler{ - Config: serverConfig, - DownstreamCluster: deps.downstreamCluster, + Config: serverConfig, + DownstreamCluster: deps.downstreamCluster, + CertificateServiceReader: deps.certificateServiceReader, }).SetupWithManager(mgr) }}, {"gatewayclass", true, func() error { diff --git a/internal/config/config.go b/internal/config/config.go index b72c4a07..6ecad17a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -905,6 +905,42 @@ type GatewayConfig struct { // the deletion through the entire chain (CertificateRequest, Order, // Challenge, solver resources). CertificateReissuance CertificateReissuanceConfig `json:"certificateReissuance,omitempty"` + + // CertificateService hands per-hostname certificate issuance to the Milo + // certificate service. When enabled, the gateway controller requests a + // TLSCertificate in the project control plane for each custom hostname, + // serves its HTTP-01 challenges on the downstream gateway, and mirrors the + // issued Secret downstream, instead of creating a cert-manager Certificate + // on the downstream cluster. + CertificateService CertificateServiceConfig `json:"certificateService,omitempty"` +} + +// +k8s:deepcopy-gen=true + +// CertificateServiceConfig controls consumption of the Milo certificate +// service (certificates.miloapis.com). +type CertificateServiceConfig struct { + // Enabled switches custom hostname certificate issuance from downstream + // cert-manager Certificates to upstream TLSCertificates. + // + // Defaults to false. + Enabled bool `json:"enabled,omitempty"` + + // KubeconfigPath reaches the cluster the certificate service runs on, + // where it keeps the service-side copy of each issued Secret. Empty + // means the cluster this operator runs in. + KubeconfigPath string `json:"kubeconfigPath,omitempty"` +} + +// RestConfig returns the connection to the certificate service's cluster. +func (c *CertificateServiceConfig) RestConfig() (*rest.Config, error) { + if c.KubeconfigPath != "" { + return clientcmd.BuildConfigFromFlags("", c.KubeconfigPath) + } + if cfg, err := rest.InClusterConfig(); err == nil { + return cfg, nil + } + return ctrl.GetConfig() } // +k8s:deepcopy-gen=true @@ -1490,6 +1526,9 @@ func (c *GatewayConfig) validate() error { } seen = append(seen, domain) } + if c.CertificateService.Enabled && c.DisableHostnameVerification { + errs = append(errs, errors.New("certificateService.enabled requires hostname verification: the certificate service issues for any hostname it is handed")) + } return errors.Join(errs...) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ff5bf93f..ae6b5ef4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -204,3 +204,54 @@ func TestGatewayConfig_ManagedTargetDomains(t *testing.T) { } } } + +func TestGatewayConfig_ValidateCertificateService(t *testing.T) { + tests := []struct { + name string + gateway GatewayConfig + wantSub string + }{ + { + name: "disabled by default validates", + }, + { + name: "enabled with hostname verification validates", + gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true}}, + }, + { + name: "enabled without hostname verification is rejected", + gateway: GatewayConfig{ + CertificateService: CertificateServiceConfig{Enabled: true}, + DisableHostnameVerification: true, + }, + wantSub: "certificateService.enabled requires hostname verification", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &NetworkServicesOperator{Gateway: tt.gateway} + err := cfg.Validate() + if tt.wantSub == "" { + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantSub) { + t.Fatalf("expected error containing %q, got %v", tt.wantSub, err) + } + }) + } +} + +func TestSetObjectDefaults_CertificateService(t *testing.T) { + cfg := &NetworkServicesOperator{} + SetObjectDefaults_NetworkServicesOperator(cfg) + if cfg.Gateway.CertificateService.Enabled { + t.Error("certificateService.enabled should default to false") + } + if cfg.Gateway.CertificateService.KubeconfigPath != "" { + t.Errorf("certificateService.kubeconfigPath should default to empty, got %q", cfg.Gateway.CertificateService.KubeconfigPath) + } +} diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index d6a4737c..1202dfcd 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -78,6 +78,21 @@ func (in *CertificateReissuanceConfig) DeepCopy() *CertificateReissuanceConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificateServiceConfig) DeepCopyInto(out *CertificateServiceConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificateServiceConfig. +func (in *CertificateServiceConfig) DeepCopy() *CertificateServiceConfig { + if in == nil { + return nil + } + out := new(CertificateServiceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientConnectionConfig) DeepCopyInto(out *ClientConnectionConfig) { *out = *in @@ -436,6 +451,7 @@ func (in *GatewayConfig) DeepCopyInto(out *GatewayConfig) { **out = **in } out.CertificateReissuance = in.CertificateReissuance + out.CertificateService = in.CertificateService } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayConfig. diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go new file mode 100644 index 00000000..04c4c856 --- /dev/null +++ b/internal/controller/gateway_certificate_service.go @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "strings" + "time" + + cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" + "go.datum.net/network-services-operator/internal/util/resourcename" +) + +// +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates/status,verbs=get + +const tlsCertificateSolverLabel = "networking.datumapis.com/tlscertificate-solver" + +const KindTLSCertificate = "TLSCertificate" + +const tlsCertificateRecreateDelay = 5 * time.Second + +// tlsCertificateMirrorAdmitDelay is how soon the listener is re-evaluated after +// its Secret lands downstream, since listener health was judged before the +// mirror in the same pass. +const tlsCertificateMirrorAdmitDelay = time.Second + +// tlsCertificateName is the listener's TLSCertificate name. The service caps +// names at 63 characters, so a long gateway and listener pair is truncated and +// hashed where the cert-manager Certificate name was not. +func tlsCertificateName(gatewayName string, listenerName gatewayv1.SectionName) string { + return resourcename.GetValidDNS1035Name(fmt.Sprintf("%s-%s", gatewayName, listenerName)) +} + +func tlsCertificateSolverName(certName, token string) string { + sum := sha256.Sum256([]byte(token)) + return resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-acme-%s", certName, hex.EncodeToString(sum[:])[:10])) +} + +// listenerWantsOwnCertificate reports whether a listener is one the controller +// issues a per-hostname certificate for, under either issuance path. +func (r *GatewayReconciler) listenerWantsOwnCertificate(l gatewayv1.Listener, claimedHostnames []string) (string, bool) { + if l.TLS == nil || l.TLS.Options[certificateIssuerTLSOption] == "" || l.Hostname == nil { + return "", false + } + hostname := string(*l.Hostname) + if !slices.Contains(claimedHostnames, hostname) { + return "", false + } + wildcardSuffix := "." + r.Config.Gateway.TargetDomain + if r.Config.Gateway.HasDefaultListenerTLSSecret() && + (strings.HasSuffix(hostname, wildcardSuffix) || hostname == r.Config.Gateway.TargetDomain) { + return "", false + } + return hostname, true +} + +// ensureListenerTLSCertificates is the certificate-service counterpart of +// ensureListenerCertificates: one TLSCertificate per custom hostname in the +// project control plane, its HTTP-01 challenges answered on the downstream +// gateway, and its issued Secret mirrored downstream under the name the +// listener references. +func (r *GatewayReconciler) ensureListenerTLSCertificates( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + downstreamClient client.Client, + downstreamStrategy downstreamclient.ResourceStrategy, + claimedHostnames []string, +) (result Result) { + logger := log.FromContext(ctx) + now := time.Now() + desiredCerts := sets.New[string]() + legacyKeep := sets.New[string]() + liveSolvers := sets.New[string]() + + for _, l := range upstreamGateway.Spec.Listeners { + hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) + if !wanted { + continue + } + + legacyName := listenerCertificateName(upstreamGateway.Name, l.Name) + certName := tlsCertificateName(upstreamGateway.Name, l.Name) + secretName := listenerCertificateSecretName(upstreamGateway.Name, l.Name) + + legacy, err := r.ownedLegacyCertificate(ctx, downstreamClient, upstreamGateway, downstreamGateway, legacyName) + if err != nil { + result.Err = err + return result + } + if legacy != nil && legacyCertificateHolds(legacy, now) { + legacyKeep.Insert(legacyName) + continue + } + if legacy != nil { + logger.Info("legacy certificate due for renewal, switching hostname to the certificate service", "certificate", legacyName, "hostname", hostname) + if err := r.retireLegacyCertificate(ctx, downstreamStrategy, upstreamGateway, legacy, secretName); err != nil { + result.Err = err + return result + } + } + + desiredCerts.Insert(certName) + + cert, recreating, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) + if err != nil { + result.Err = err + return result + } + if recreating { + result.RequeueAfter = tlsCertificateRecreateDelay + continue + } + + solvers, err := r.serveTLSCertificateChallenges(ctx, downstreamClient, downstreamGateway, cert, hostname) + if err != nil { + result.Err = err + return result + } + liveSolvers.Insert(solvers...) + + mirrored, err := r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName) + if err != nil { + result.Err = err + return result + } + if mirrored { + result.RequeueAfter = tlsCertificateMirrorAdmitDelay + } + } + + if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, desiredCerts); err != nil { + result.Err = err + return result + } + + if err := r.deleteStaleLegacyCertificates(ctx, downstreamClient, upstreamGateway, downstreamGateway, legacyKeep); err != nil { + result.Err = err + return result + } + + if err := r.deleteStaleTLSCertificateSolvers(ctx, downstreamClient, downstreamGateway, liveSolvers); err != nil { + result.Err = err + return result + } + + return result +} + +// ensureTLSCertificate creates the listener's TLSCertificate or confirms the +// existing one. dnsNames and secretName are immutable on the service's API, so +// one that no longer matches is deleted and requested again on the next pass. +func (r *GatewayReconciler) ensureTLSCertificate( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + certName, secretName, hostname string, +) (cert *certificatesv1alpha1.TLSCertificate, recreating bool, err error) { + logger := log.FromContext(ctx) + + desiredSpec := certificatesv1alpha1.TLSCertificateSpec{ + DNSNames: []certificatesv1alpha1.DNSName{certificatesv1alpha1.DNSName(hostname)}, + Issuance: certificatesv1alpha1.IssuanceModeAuto, + SecretName: secretName, + } + + cert = &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: upstreamGateway.Namespace, + Name: certName, + }, + } + + err = upstreamClient.Get(ctx, client.ObjectKeyFromObject(cert), cert) + switch { + case apierrors.IsNotFound(err): + if err := controllerutil.SetControllerReference(upstreamGateway, cert, upstreamClient.Scheme()); err != nil { + return nil, false, fmt.Errorf("failed to set controller reference on TLSCertificate %s: %w", certName, err) + } + cert.Spec = desiredSpec + if err := upstreamClient.Create(ctx, cert); err != nil { + return nil, false, fmt.Errorf("failed to create TLSCertificate %s: %w", certName, err) + } + logger.Info("TLSCertificate requested", "tlscertificate", certName, "hostname", hostname) + return cert, false, nil + case err != nil: + return nil, false, fmt.Errorf("failed to get TLSCertificate %s: %w", certName, err) + } + + if !slices.Equal(cert.Spec.DNSNames, desiredSpec.DNSNames) || cert.Spec.SecretName != desiredSpec.SecretName { + logger.Info("TLSCertificate no longer matches its listener, requesting it again", "tlscertificate", certName, "hostname", hostname) + if err := upstreamClient.Delete(ctx, cert); client.IgnoreNotFound(err) != nil { + return nil, false, fmt.Errorf("failed to delete TLSCertificate %s: %w", certName, err) + } + return nil, true, nil + } + + if metav1.IsControlledBy(cert, upstreamGateway) && cert.Spec.Issuance == desiredSpec.Issuance { + return cert, false, nil + } + + if err := controllerutil.SetControllerReference(upstreamGateway, cert, upstreamClient.Scheme()); err != nil { + return nil, false, fmt.Errorf("failed to set controller reference on TLSCertificate %s: %w", certName, err) + } + cert.Spec.Issuance = desiredSpec.Issuance + if err := upstreamClient.Update(ctx, cert); err != nil { + return nil, false, fmt.Errorf("failed to update TLSCertificate %s: %w", certName, err) + } + logger.Info("TLSCertificate reconciled", "tlscertificate", certName, "operation", "updated") + return cert, false, nil +} + +// serveTLSCertificateChallenges publishes every pending HTTP-01 challenge the +// service reports for the hostname this listener claimed and returns the solver +// names for every such challenge still in status, so a challenge that has moved +// on to Valid keeps its answer until the service drops it. A challenge for any +// other name is not ours to answer, whatever status says. +func (r *GatewayReconciler) serveTLSCertificateChallenges( + ctx context.Context, + downstreamClient client.Client, + downstreamGateway *gatewayv1.Gateway, + cert *certificatesv1alpha1.TLSCertificate, + hostname string, +) ([]string, error) { + logger := log.FromContext(ctx) + var live []string + + for _, challenge := range cert.Status.Challenges { + if challenge.Type != certificatesv1alpha1.ChallengeTypeHTTP01 { + continue + } + if challenge.DNSName != hostname { + logger.Info("ignoring HTTP-01 challenge for a hostname this listener did not claim", + "tlscertificate", cert.Name, "claimed", hostname, "challenge_dns_name", challenge.DNSName) + continue + } + name := tlsCertificateSolverName(cert.Name, challenge.Token) + live = append(live, name) + + if challenge.State != certificatesv1alpha1.ChallengeStatePending { + continue + } + + err := ensureHTTP01SolverRoutes(ctx, downstreamClient, downstreamClient.Scheme(), downstreamGateway, downstreamGateway, http01SolverRoute{ + name: name, + token: challenge.Token, + key: challenge.Key, + labels: map[string]string{tlsCertificateSolverLabel: cert.Name}, + }) + if err != nil { + return nil, fmt.Errorf("failed to serve HTTP-01 challenge for TLSCertificate %s: %w", cert.Name, err) + } + } + + return live, nil +} + +// mirrorTLSCertificateSecret copies the service-side issued Secret, read with +// the operator's own credentials, into the downstream gateway namespace under +// the listener's secret name, stamped with the upstream-owner labels the +// federation policy selects. The project-namespace copy is the tenant's and is +// never what reaches an edge. +func (r *GatewayReconciler) mirrorTLSCertificateSecret( + ctx context.Context, + downstreamStrategy downstreamclient.ResourceStrategy, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + cert *certificatesv1alpha1.TLSCertificate, + secretName string, +) (mirrored bool, err error) { + logger := log.FromContext(ctx) + + if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { + return false, nil + } + if cert.Status.ServiceSecretRef == nil || cert.Status.ServiceSecretRef.Name == "" { + logger.Info("TLSCertificate is Ready without a service-side Secret reference", "tlscertificate", cert.Name) + return false, nil + } + if r.CertificateServiceReader == nil { + return false, fmt.Errorf("certificate service enabled without a client for its cluster") + } + + var source corev1.Secret + sourceKey := client.ObjectKey{Namespace: cert.Status.ServiceSecretRef.Namespace, Name: cert.Status.ServiceSecretRef.Name} + if err := r.CertificateServiceReader.Get(ctx, sourceKey, &source); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("TLSCertificate is Ready but its service-side Secret is not readable yet", "tlscertificate", cert.Name, "secret", sourceKey) + return false, nil + } + return false, fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) + } + if len(source.Data["tls.crt"]) == 0 || len(source.Data["tls.key"]) == 0 { + logger.Info("service-side Secret holds no certificate material yet", "tlscertificate", cert.Name, "secret", sourceKey) + return false, nil + } + + mirror := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: downstreamGateway.Namespace, + Name: secretName, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, downstreamStrategy.GetClient(), mirror, func() error { + if mirror.CreationTimestamp.IsZero() { + mirror.Type = corev1.SecretTypeTLS + } + if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, mirror); err != nil { + return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) + } + mirror.Data = map[string][]byte{ + "tls.crt": source.Data["tls.crt"], + "tls.key": source.Data["tls.key"], + } + if ca := source.Data["ca.crt"]; len(ca) > 0 { + mirror.Data["ca.crt"] = ca + } + return nil + }) + if err != nil { + return false, fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) + } + if op != controllerutil.OperationResultNone { + logger.Info("issued Secret mirrored downstream", "secret", secretName, "operation", op) + } + + return op != controllerutil.OperationResultNone, nil +} + +// ownedLegacyCertificate returns the listener's cert-manager Certificate from +// the downstream cluster when one exists and belongs to this gateway. +func (r *GatewayReconciler) ownedLegacyCertificate( + ctx context.Context, + downstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + certName string, +) (*cmv1.Certificate, error) { + var legacy cmv1.Certificate + if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamGateway.Namespace, Name: certName}, &legacy); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to get Certificate %s: %w", certName, err) + } + if !certificateOwnedByGateway(&legacy, upstreamGateway, downstreamGateway) { + return nil, nil + } + return &legacy, nil +} + +// legacyCertificateHolds reports whether a cert-manager Certificate is serving +// and not yet due for renewal, so switching it to the service now would issue a +// certificate nobody needs yet. +func legacyCertificateHolds(cert *cmv1.Certificate, now time.Time) bool { + return certIsServing(cert, now) && cert.Status.RenewalTime != nil && cert.Status.RenewalTime.After(now) +} + +// retireLegacyCertificate hands the listener over to the service: the Secret +// gains the gateway's own owner so cert-manager's owner reference no longer +// takes it along, then the Certificate goes so cert-manager does not renew a +// hostname the service is about to issue for. +func (r *GatewayReconciler) retireLegacyCertificate( + ctx context.Context, + downstreamStrategy downstreamclient.ResourceStrategy, + upstreamGateway *gatewayv1.Gateway, + legacy *cmv1.Certificate, + secretName string, +) error { + downstreamClient := downstreamStrategy.GetClient() + + var secret corev1.Secret + err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: legacy.Namespace, Name: secretName}, &secret) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return fmt.Errorf("failed to get Secret %s: %w", secretName, err) + default: + if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, &secret); err != nil { + return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) + } + if err := downstreamClient.Update(ctx, &secret); err != nil { + return fmt.Errorf("failed to keep Secret %s past its Certificate: %w", secretName, err) + } + } + + if err := downstreamClient.Delete(ctx, legacy); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete legacy Certificate %s: %w", legacy.Name, err) + } + return nil +} + +// deleteStaleLegacyCertificates removes the gateway's cert-manager Certificates +// for listeners that no longer carry one, mirroring what the cert-manager path +// does for removed hostnames. +func (r *GatewayReconciler) deleteStaleLegacyCertificates( + ctx context.Context, + downstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + legacyKeep sets.Set[string], +) error { + logger := log.FromContext(ctx) + + var certList cmv1.CertificateList + if err := downstreamClient.List(ctx, &certList, client.InNamespace(downstreamGateway.Namespace)); err != nil { + return fmt.Errorf("failed to list Certificates: %w", err) + } + + for i := range certList.Items { + cert := &certList.Items[i] + if legacyKeep.Has(cert.Name) || !certificateOwnedByGateway(cert, upstreamGateway, downstreamGateway) { + continue + } + logger.Info("deleting stale Certificate", "certificate", cert.Name) + if err := downstreamClient.Delete(ctx, cert); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete stale Certificate %s: %w", cert.Name, err) + } + } + + return nil +} + +func certificateOwnedByGateway(cert *cmv1.Certificate, upstreamGateway, downstreamGateway *gatewayv1.Gateway) bool { + ownedByStrategy := cert.Labels[downstreamclient.UpstreamOwnerKindLabel] == KindGateway && + cert.Labels[downstreamclient.UpstreamOwnerNameLabel] == upstreamGateway.Name && + cert.Labels[downstreamclient.UpstreamOwnerNamespaceLabel] == upstreamGateway.Namespace + return ownedByStrategy || metav1.IsControlledBy(cert, downstreamGateway) +} + +func (r *GatewayReconciler) deleteStaleTLSCertificates( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + desiredCerts sets.Set[string], +) error { + logger := log.FromContext(ctx) + + var certs certificatesv1alpha1.TLSCertificateList + if err := upstreamClient.List(ctx, &certs, client.InNamespace(upstreamGateway.Namespace)); err != nil { + return fmt.Errorf("failed to list TLSCertificates: %w", err) + } + + for i := range certs.Items { + cert := &certs.Items[i] + if desiredCerts.Has(cert.Name) || !metav1.IsControlledBy(cert, upstreamGateway) { + continue + } + logger.Info("deleting stale TLSCertificate", "tlscertificate", cert.Name) + if err := upstreamClient.Delete(ctx, cert); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete stale TLSCertificate %s: %w", cert.Name, err) + } + } + + return nil +} + +func (r *GatewayReconciler) deleteStaleTLSCertificateSolvers( + ctx context.Context, + downstreamClient client.Client, + downstreamGateway *gatewayv1.Gateway, + liveSolvers sets.Set[string], +) error { + logger := log.FromContext(ctx) + + var routes gatewayv1.HTTPRouteList + if err := downstreamClient.List(ctx, &routes, + client.InNamespace(downstreamGateway.Namespace), + client.HasLabels{tlsCertificateSolverLabel}, + ); err != nil { + return fmt.Errorf("failed to list TLSCertificate solver routes: %w", err) + } + + for i := range routes.Items { + route := &routes.Items[i] + if liveSolvers.Has(route.Name) || !metav1.IsControlledBy(route, downstreamGateway) { + continue + } + logger.Info("removing finished HTTP-01 solver", "solver", route.Name) + if err := deleteHTTP01SolverRoutes(ctx, downstreamClient, route.Namespace, route.Name); err != nil { + return err + } + } + + return nil +} + +// evaluateListenerTLSCertificateHealth is evaluateListenerCertHealth for the +// certificate-service path: conditions come from the upstream TLSCertificate, +// the served material from the mirrored downstream Secret. +func (r *GatewayReconciler) evaluateListenerTLSCertificateHealth( + ctx context.Context, + upstreamClient client.Client, + downstreamClient client.Client, + downstreamNamespace string, + upstreamGateway *gatewayv1.Gateway, + claimedHostnames []string, +) map[gatewayv1.SectionName]listenerCertStatus { + logger := log.FromContext(ctx) + health := make(map[gatewayv1.SectionName]listenerCertStatus) + now := time.Now() + + clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) + + for _, l := range upstreamGateway.Spec.Listeners { + hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) + if !wanted { + continue + } + + status := r.listenerTLSCertificateHealth(ctx, upstreamClient, downstreamClient, downstreamNamespace, upstreamGateway, l.Name, hostname, now) + health[l.Name] = status + recordListenerCertHealth(logger, upstreamGateway, l.Name, hostname, status) + } + + return health +} + +// listenerTLSCertificateHealth keeps a listener serving whenever its downstream +// Secret holds a usable certificate, whatever the TLSCertificate is doing: a +// renewal in flight or a first issuance replacing a legacy certificate must +// not take HTTPS down. Only when nothing usable is downstream do the +// TLSCertificate's conditions decide what the customer is told. +func (r *GatewayReconciler) listenerTLSCertificateHealth( + ctx context.Context, + upstreamClient client.Client, + downstreamClient client.Client, + downstreamNamespace string, + upstreamGateway *gatewayv1.Gateway, + listenerName gatewayv1.SectionName, + hostname string, + now time.Time, +) listenerCertStatus { + logger := log.FromContext(ctx) + + certName := tlsCertificateName(upstreamGateway.Name, listenerName) + secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) + + secretStatus := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + + var cert certificatesv1alpha1.TLSCertificate + if err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamGateway.Namespace, Name: certName}, &cert); err != nil { + if !apierrors.IsNotFound(err) { + logger.Error(err, "failed to get listener TLSCertificate", "tlscertificate", certName) + } + if secretStatus.healthy { + return secretStatus + } + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certIssuanceFailingMessage(hostname), + pending: true, + secretName: secretName, + } + } + + if secretStatus.healthy { + if apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) && cert.Status.NotAfter != nil { + secretStatus.notAfter = cert.Status.NotAfter + } + return secretStatus + } + + if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: tlsCertificateRejectedMessage(hostname, accepted.Message), + secretName: secretName, + } + } + + if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certIssuanceFailingMessage(hostname), + pending: true, + secretName: secretName, + } + } + + if cert.Status.NotBefore != nil && cert.Status.NotBefore.After(now) { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certNotYetValidMessage(hostname), + secretName: secretName, + } + } + if cert.Status.NotAfter != nil && !cert.Status.NotAfter.After(now.Add(listenerCertExpiryMargin)) { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certExpiredMessage(hostname), + notAfter: cert.Status.NotAfter, + secretName: secretName, + } + } + + secretStatus.pending = true + return secretStatus +} + +func tlsCertificateRejectedMessage(hostname, detail string) string { + if detail == "" { + return certIssuanceFailingMessage(hostname) + } + return fmt.Sprintf("We couldn't issue a TLS certificate for %s, so HTTPS for this hostname is unavailable: %s", hostname, detail) +} + +// listGatewaysForTLSCertificateFunc enqueues the Gateway that controls a +// TLSCertificate, in the cluster the TLSCertificate lives in. +func (r *GatewayReconciler) listGatewaysForTLSCertificateFunc(clusterName multicluster.ClusterName, _ cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []mcreconcile.Request { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.Kind != KindGateway { + return nil + } + return []mcreconcile.Request{{ + ClusterName: clusterName, + Request: reconcile.Request{ + NamespacedName: client.ObjectKey{Namespace: obj.GetNamespace(), Name: owner.Name}, + }, + }} + }) +} diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go new file mode 100644 index 00000000..bd19c058 --- /dev/null +++ b/internal/controller/gateway_certificate_service_test.go @@ -0,0 +1,504 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + "go.datum.net/network-services-operator/internal/config" + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" + gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" +) + +func newCertificateServiceTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + testScheme := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(testScheme)) + require.NoError(t, gatewayv1.Install(testScheme)) + require.NoError(t, discoveryv1.AddToScheme(testScheme)) + require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + require.NoError(t, cmv1.AddToScheme(testScheme)) + require.NoError(t, envoygatewayv1alpha1.AddToScheme(testScheme)) + require.NoError(t, certificatesv1alpha1.AddToScheme(testScheme)) + return testScheme +} + +func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + + upstreamNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}, + } + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + + const ( + gatewayName = "test-gw" + listenerName = gatewayv1.SectionName("https-hostname-0") + hostname = "custom.example.com" + serviceNS = "certificates-system" + serviceSecret = "svc-secret" + upstreamCluster = "test" + ) + certName := tlsCertificateName(gatewayName, listenerName) + legacyName := listenerCertificateName(gatewayName, listenerName) + secretName := listenerCertificateSecretName(gatewayName, listenerName) + + baseConfig := config.GatewayConfig{ + DownstreamGatewayClassName: "test-suite", + DownstreamHostnameAccountingNamespace: "default", + TargetDomain: "test-suite.com", + DefaultListenerTLSSecretName: "wildcard-test-suite-tls", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, + }, + ClusterIssuerMap: map[string]string{autoIssuerSentinel: "datum-gateway-http"}, + CertificateService: config.CertificateServiceConfig{Enabled: true}, + } + testCfg := config.NetworkServicesOperator{Gateway: baseConfig} + + customListener := gatewayv1.Listener{ + Name: listenerName, + Protocol: gatewayv1.HTTPSProtocolType, + Port: DefaultHTTPSPort, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + AllowedRoutes: &gatewayv1.AllowedRoutes{ + Namespaces: &gatewayv1.RouteNamespaces{From: ptr.To(gatewayv1.NamespacesFromSame)}, + }, + TLS: &gatewayv1.ListenerTLSConfig{ + Mode: ptr.To(gatewayv1.TLSModeTerminate), + Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, + }, + }, + } + + verifiedDomain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { + d.Spec.DomainName = hostname + apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.DomainConditionVerified, + Status: metav1.ConditionTrue, + }) + }) + + now := time.Now() + serviceCertPEM, serviceKeyPEM := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + projectCertPEM, projectKeyPEM := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + legacyCertPEM, legacyKeyPEM := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + + tlsSecret := func(namespace, name string, certPEM, keyPEM []byte) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": certPEM, "tls.key": keyPEM}, + } + } + + newTLSCertificate := func(owner *gatewayv1.Gateway, name string, mutate func(*certificatesv1alpha1.TLSCertificate)) *certificatesv1alpha1.TLSCertificate { + cert := &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: name, UID: uuid.NewUUID()}, + Spec: certificatesv1alpha1.TLSCertificateSpec{ + DNSNames: []certificatesv1alpha1.DNSName{hostname}, + Issuance: certificatesv1alpha1.IssuanceModeAuto, + SecretName: secretName, + }, + } + if owner != nil { + require.NoError(t, controllerutil.SetControllerReference(owner, cert, testScheme)) + } + if mutate != nil { + mutate(cert) + } + return cert + } + + readyStatus := func(cert *certificatesv1alpha1.TLSCertificate) { + cert.Status.NotBefore = &metav1.Time{Time: now.Add(-time.Hour)} + cert.Status.NotAfter = &metav1.Time{Time: now.Add(60 * 24 * time.Hour)} + cert.Status.SecretRef = &certificatesv1alpha1.SecretReference{Name: secretName} + cert.Status.ServiceSecretRef = &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: serviceSecret} + apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue, Reason: "Accepted"}) + apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}) + } + + legacyLabels := map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + upstreamCluster, + downstreamclient.UpstreamOwnerGroupLabel: gatewayv1.GroupName, + downstreamclient.UpstreamOwnerKindLabel: KindGateway, + downstreamclient.UpstreamOwnerNameLabel: gatewayName, + downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace.Name, + } + newLegacyCertificate := func(renewalTime time.Time) *cmv1.Certificate { + return &cmv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: legacyName, UID: uuid.NewUUID(), Labels: legacyLabels}, + Spec: cmv1.CertificateSpec{SecretName: secretName, DNSNames: []string{hostname}}, + Status: cmv1.CertificateStatus{ + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + RenewalTime: &metav1.Time{Time: renewalTime}, + Conditions: []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionTrue}}, + }, + } + } + + downstreamGatewaySeed := func() *gatewayv1.Gateway { + return &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: downstreamNamespaceName, + Name: gatewayName, + UID: uuid.NewUUID(), + Labels: legacyLabels, + }, + Spec: gatewayv1.GatewaySpec{GatewayClassName: "test-suite"}, + } + } + + solverObjects := func(owner *gatewayv1.Gateway, token string) []client.Object { + name := tlsCertificateSolverName(certName, token) + labels := map[string]string{http01SolverLabel: labelValueTrue, tlsCertificateSolverLabel: certName} + route := &gatewayv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: name, Labels: labels}} + filter := &envoygatewayv1alpha1.HTTPRouteFilter{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: name, Labels: labels}} + require.NoError(t, controllerutil.SetControllerReference(owner, route, testScheme)) + require.NoError(t, controllerutil.SetControllerReference(owner, filter, testScheme)) + return []client.Object{route, filter} + } + + type env struct { + upstream client.Client + downstream client.Client + service client.Client + result Result + reconciler *GatewayReconciler + } + + tests := []struct { + name string + upstreamObjects func(gw *gatewayv1.Gateway) []client.Object + downstreamObjects func() []client.Object + serviceObjects []client.Object + assert func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) + }{ + { + name: "requests a TLSCertificate upstream and withholds the listener until it issues", + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) + assert.Equal(t, []certificatesv1alpha1.DNSName{hostname}, cert.Spec.DNSNames) + assert.Equal(t, certificatesv1alpha1.IssuanceModeAuto, cert.Spec.Issuance) + assert.Equal(t, secretName, cert.Spec.SecretName) + assert.True(t, metav1.IsControlledBy(&cert, upstreamGateway), "TLSCertificate should be controlled by the upstream Gateway") + + var legacy cmv1.Certificate + err := e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &legacy) + assert.True(t, apierrors.IsNotFound(err), "no cert-manager Certificate should be created downstream") + + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener must be withheld until a certificate is issued") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, gatewayutil.DefaultHTTPSListenerName), "shared wildcard listener is untouched") + }, + }, + { + name: "serves pending HTTP-01 challenges only for the hostname the listener claimed", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.Challenges = []certificatesv1alpha1.ACMEChallenge{ + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-a", Key: "key-a", State: certificatesv1alpha1.ChallengeStatePending}, + {DNSName: "evil.example.com", Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-b", Key: "key-b", State: certificatesv1alpha1.ChallengeStatePending}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeDNS01, Token: "tok-c", Key: "key-c", State: certificatesv1alpha1.ChallengeStatePending}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-d", Key: "key-d", State: certificatesv1alpha1.ChallengeStateInvalid}, + } + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + served := tlsCertificateSolverName(certName, "tok-a") + + var route gatewayv1.HTTPRoute + require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: served}, &route)) + assert.Equal(t, labelValueTrue, route.Labels[http01SolverLabel]) + assert.Equal(t, certName, route.Labels[tlsCertificateSolverLabel]) + require.Len(t, route.Spec.Rules, 1) + require.Len(t, route.Spec.Rules[0].Matches, 1) + assert.Equal(t, "/.well-known/acme-challenge/tok-a", ptr.Deref(route.Spec.Rules[0].Matches[0].Path.Value, "")) + assert.Equal(t, gatewayv1.ObjectName(gatewayName), route.Spec.ParentRefs[0].Name) + owner := metav1.GetControllerOf(&route) + require.NotNil(t, owner) + assert.Equal(t, KindGateway, owner.Kind) + + var filter envoygatewayv1alpha1.HTTPRouteFilter + require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: served}, &filter)) + assert.Equal(t, "key-a", ptr.Deref(filter.Spec.DirectResponse.Body.Inline, "")) + + for _, token := range []string{"tok-b", "tok-c", "tok-d"} { + err := e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, token)}, &gatewayv1.HTTPRoute{}) + assert.True(t, apierrors.IsNotFound(err), "no solver route for challenge %s", token) + } + }, + }, + { + name: "removes solver routes once their challenge leaves status", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.Challenges = []certificatesv1alpha1.ACMEChallenge{ + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-keep", Key: "key-keep", State: certificatesv1alpha1.ChallengeStateValid}, + } + })} + }, + downstreamObjects: func() []client.Object { + gw := downstreamGatewaySeed() + objs := make([]client.Object, 0, 5) + objs = append(objs, gw) + objs = append(objs, solverObjects(gw, "tok-keep")...) + objs = append(objs, solverObjects(gw, "tok-old")...) + return objs + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, "tok-keep")}, &gatewayv1.HTTPRoute{}), "a Valid challenge keeps its answer until the service drops it") + + old := tlsCertificateSolverName(certName, "tok-old") + assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: old}, &gatewayv1.HTTPRoute{}))) + assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: old}, &envoygatewayv1alpha1.HTTPRouteFilter{}))) + }, + }, + { + name: "mirrors the service-side Secret when Ready and admits the listener", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{ + newTLSCertificate(gw, certName, readyStatus), + tlsSecret(upstreamNamespace.Name, secretName, projectCertPEM, projectKeyPEM), + } + }, + serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, serviceKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + var mirror corev1.Secret + require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &mirror)) + assert.Equal(t, corev1.SecretTypeTLS, mirror.Type) + assert.Equal(t, serviceCertPEM, mirror.Data["tls.crt"], "edge material comes from the service-side Secret") + assert.Equal(t, serviceKeyPEM, mirror.Data["tls.key"]) + assert.NotEqual(t, projectCertPEM, mirror.Data["tls.crt"], "the project copy must never reach the edge") + assert.Equal(t, "cluster-"+upstreamCluster, mirror.Labels[downstreamclient.UpstreamOwnerClusterNameLabel]) + assert.Equal(t, upstreamNamespace.Name, mirror.Labels[downstreamclient.UpstreamOwnerNamespaceLabel]) + assert.Equal(t, KindGateway, mirror.Labels[downstreamclient.UpstreamOwnerKindLabel]) + + assert.Equal(t, tlsCertificateMirrorAdmitDelay, e.result.RequeueAfter, "a fresh mirror asks for a prompt re-evaluation") + + health := e.reconciler.evaluateListenerTLSCertificateHealth(ctx, e.upstream, e.downstream, downstreamNamespaceName, upstreamGateway, []string{hostname}) + status, gated := health[listenerName] + require.True(t, gated) + assert.True(t, status.healthy, "listener is admitted on the next pass: %s", status.message) + assert.Equal(t, secretName, status.secretName) + require.NotNil(t, status.notAfter) + assert.WithinDuration(t, now.Add(60*24*time.Hour), status.notAfter.Time, time.Minute) + }, + }, + { + name: "a rejected request withholds the listener and tells the customer why", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, + Reason: "DeniedDomain", Message: "names under datum.net are denied", + }) + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + + var found bool + for _, ls := range upstreamGateway.Status.Listeners { + if ls.Name != listenerName { + continue + } + found = true + resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) + require.NotNil(t, resolved) + assert.Equal(t, metav1.ConditionFalse, resolved.Status) + assert.Contains(t, resolved.Message, "names under datum.net are denied") + } + assert.True(t, found, "listener status should be reported") + }, + }, + { + name: "keeps a valid cert-manager certificate that is not due for renewal", + downstreamObjects: func() []client.Object { + return []client.Object{ + newLegacyCertificate(now.Add(30 * 24 * time.Hour)), + tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM), + } + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + var certs certificatesv1alpha1.TLSCertificateList + require.NoError(t, e.upstream.List(ctx, &certs, client.InNamespace(upstreamNamespace.Name))) + assert.Empty(t, certs.Items, "no TLSCertificate is requested while the legacy certificate holds") + + assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{}), "legacy Certificate stays") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener keeps serving on the legacy certificate") + }, + }, + { + name: "switches to the service when the cert-manager certificate is due for renewal", + downstreamObjects: func() []client.Object { + legacy := newLegacyCertificate(now.Add(-time.Hour)) + secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) + require.NoError(t, controllerutil.SetControllerReference(legacy, secret, testScheme)) + return []client.Object{legacy, secret} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{})), "legacy Certificate is retired so cert-manager does not renew it") + + var secret corev1.Secret + require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"], "the serving Secret is kept as-is until the service issues") + assert.Equal(t, upstreamNamespace.Name, secret.Labels[downstreamclient.UpstreamOwnerNamespaceLabel]) + var anchored bool + for _, ref := range secret.OwnerReferences { + if ref.Kind == "ConfigMap" { + anchored = true + } + } + assert.True(t, anchored, "Secret gains the gateway anchor so the Certificate's owner reference no longer takes it along") + + assert.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{}), "TLSCertificate is requested") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener keeps serving on the retained Secret") + }, + }, + { + name: "deletes the TLSCertificate of a listener that is gone and leaves others alone", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{ + newTLSCertificate(gw, tlsCertificateName(gatewayName, "https-hostname-9"), nil), + newTLSCertificate(nil, "someone-elses-cert", nil), + } + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + assert.True(t, apierrors.IsNotFound(e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, "https-hostname-9")}, &certificatesv1alpha1.TLSCertificate{}))) + assert.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: "someone-elses-cert"}, &certificatesv1alpha1.TLSCertificate{})) + assert.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{})) + }, + }, + { + name: "requests the TLSCertificate again when its immutable spec no longer matches", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"previous.example.com"} + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + err := e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{}) + assert.True(t, apierrors.IsNotFound(err), "mismatched TLSCertificate is deleted so the next pass requests it afresh") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.Spec.Listeners = append(g.Spec.Listeners, customListener) + }) + + upstreamObjects := []client.Object{ + verifiedDomain.DeepCopy(), + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: gatewayv1.GatewayClassSpec{ControllerName: gatewayv1.GatewayController("test")}, + }, + } + if tt.upstreamObjects != nil { + upstreamObjects = append(upstreamObjects, tt.upstreamObjects(upstreamGateway)...) + } + var downstreamObjects []client.Object + if tt.downstreamObjects != nil { + downstreamObjects = tt.downstreamObjects() + } + for _, obj := range append(append([]client.Object{}, upstreamObjects...), downstreamObjects...) { + if obj.GetUID() == "" { + obj.SetUID(uuid.NewUUID()) + } + obj.SetCreationTimestamp(metav1.Now()) + } + + fakeUpstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace). + WithObjects(upstreamObjects...). + WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). + WithStatusSubresource(upstreamObjects...). + Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamObjects...). + WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}). + Build() + + fakeServiceClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(tt.serviceObjects...). + Build() + + ctx := log.IntoContext(context.Background(), logger) + + reconciler := &GatewayReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + Config: testCfg, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + CertificateServiceReader: fakeServiceClient, + } + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy(upstreamCluster, fakeUpstreamClient, fakeDownstreamClient) + + reconciler.prepareUpstreamGateway(upstreamGateway) + result, downstreamGateway := reconciler.ensureDownstreamGateway(ctx, upstreamCluster, fakeUpstreamClient, upstreamGateway, downstreamStrategy) + require.NoError(t, result.Err) + _, err := result.Complete(ctx) + require.NoError(t, err) + + updatedUpstream := &gatewayv1.Gateway{} + require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), updatedUpstream)) + + tt.assert(t, env{ + upstream: fakeUpstreamClient, + downstream: fakeDownstreamClient, + service: fakeServiceClient, + result: result, + reconciler: reconciler, + }, updatedUpstream, downstreamGateway) + }) + } +} + +func TestTLSCertificateName(t *testing.T) { + long := tlsCertificateName("a-gateway-with-a-deliberately-very-long-name-for-this-test", "https-hostname-with-a-long-name-0") + assert.LessOrEqual(t, len(long), 63) + assert.Equal(t, "gw-https-0", tlsCertificateName("gw", "https-0")) +} diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 176b67e5..976360c1 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -15,6 +15,7 @@ import ( cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -45,6 +46,7 @@ import ( mcsource "sigs.k8s.io/multicluster-runtime/pkg/source" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" "go.datum.net/network-services-operator/internal/config" downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" @@ -92,6 +94,11 @@ type GatewayReconciler struct { Config config.NetworkServicesOperator DownstreamCluster cluster.Cluster + + // CertificateServiceReader reads the certificate service's cluster, where + // the service-side copy of each issued Secret lives. Required when + // Config.Gateway.CertificateService.Enabled. + CertificateServiceReader client.Reader } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -304,13 +311,25 @@ func (r *GatewayReconciler) ensureDownstreamGateway( // reissueFailedCertificate): this path is read-only and gates on any // unusable cert, re-issuance mutates and fires only on cert-manager // hard-fail (LastFailureTime). See #260. - listenerCertHealth := r.evaluateListenerCertHealth( - ctx, - downstreamClient, - downstreamGateway.Namespace, - upstreamGateway, - claimedHostnames, - ) + var listenerCertHealth map[gatewayv1.SectionName]listenerCertStatus + if r.Config.Gateway.CertificateService.Enabled { + listenerCertHealth = r.evaluateListenerTLSCertificateHealth( + ctx, + upstreamClient, + downstreamClient, + downstreamGateway.Namespace, + upstreamGateway, + claimedHostnames, + ) + } else { + listenerCertHealth = r.evaluateListenerCertHealth( + ctx, + downstreamClient, + downstreamGateway.Namespace, + upstreamGateway, + claimedHostnames, + ) + } desiredDownstreamGateway := r.getDesiredDownstreamGateway( ctx, @@ -361,14 +380,27 @@ func (r *GatewayReconciler) ensureDownstreamGateway( } } - certResult := r.ensureListenerCertificates( - ctx, - upstreamGateway, - downstreamGateway, - downstreamClient, - downstreamStrategy, - claimedHostnames, - ) + var certResult Result + if r.Config.Gateway.CertificateService.Enabled { + certResult = r.ensureListenerTLSCertificates( + ctx, + upstreamClient, + upstreamGateway, + downstreamGateway, + downstreamClient, + downstreamStrategy, + claimedHostnames, + ) + } else { + certResult = r.ensureListenerCertificates( + ctx, + upstreamGateway, + downstreamGateway, + downstreamClient, + downstreamStrategy, + claimedHostnames, + ) + } if certResult.ShouldReturn() { return certResult.Merge(result), nil } @@ -534,45 +566,55 @@ func (r *GatewayReconciler) evaluateListenerCertHealth( status := r.listenerCertHealth(ctx, downstreamClient, downstreamNamespace, upstreamGateway.Name, l.Name, hostname, now) health[l.Name] = status - // Mark this listener as managed regardless of its health, so the - // SLI ratio (withheld / managed) can be computed fleet-wide. - gatewayListenerCertManaged.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, string(l.Name), hostname, - ).Set(1) - - if !status.healthy { - logArgs := []any{ - "listener", l.Name, "hostname", hostname, - "reason", status.reason, "message", status.message, - } - // Include the expiry timestamp when available so log queries can - // identify exactly when the cert stopped being valid. - if status.notAfter != nil { - logArgs = append(logArgs, "cert_not_after", status.notAfter.UTC().Format(time.RFC3339)) - } - logger.Info("listener certificate unhealthy", logArgs...) - - gatewayListenerCertWithheld.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, - string(l.Name), hostname, string(status.reason), - ).Set(1) - gatewayListenerCertGatingTotal.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, - string(l.Name), hostname, string(status.reason), - ).Inc() - } else if status.notAfter != nil { - // Record the expiry timestamp for healthy certs so operators can - // alert before the next expiry rather than after. - gatewayListenerCertExpiryTime.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, - string(l.Name), hostname, status.secretName, - ).Set(float64(status.notAfter.Unix())) - } + recordListenerCertHealth(logger, upstreamGateway, l.Name, hostname, status) } return health } +// recordListenerCertHealth publishes one listener's certificate health as +// metrics and logs, the same way for every issuance path. +func recordListenerCertHealth( + logger logr.Logger, + upstreamGateway *gatewayv1.Gateway, + listenerName gatewayv1.SectionName, + hostname string, + status listenerCertStatus, +) { + gatewayListenerCertManaged.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, string(listenerName), hostname, + ).Set(1) + + if !status.healthy { + logArgs := []any{ + "listener", listenerName, "hostname", hostname, + "reason", status.reason, "message", status.message, + } + // Include the expiry timestamp when available so log queries can + // identify exactly when the cert stopped being valid. + if status.notAfter != nil { + logArgs = append(logArgs, "cert_not_after", status.notAfter.UTC().Format(time.RFC3339)) + } + logger.Info("listener certificate unhealthy", logArgs...) + + gatewayListenerCertWithheld.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, + string(listenerName), hostname, string(status.reason), + ).Set(1) + gatewayListenerCertGatingTotal.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, + string(listenerName), hostname, string(status.reason), + ).Inc() + } else if status.notAfter != nil { + // Record the expiry timestamp for healthy certs so operators can + // alert before the next expiry rather than after. + gatewayListenerCertExpiryTime.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, + string(listenerName), hostname, status.secretName, + ).Set(float64(status.notAfter.Unix())) + } +} + // listenerCertHealth reports whether a single listener's certificate can be // used to serve HTTPS right now, covering every way it can be unusable: still // being issued, not yet valid, expired, or missing. @@ -637,9 +679,26 @@ func (r *GatewayReconciler) listenerCertHealth( } } - // Finally, load the stored certificate and key and confirm they match and - // are still valid. This catches a broken or mismatched certificate that - // would otherwise be served and break HTTPS for the listener. + if secretStatus := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now); !secretStatus.healthy { + return secretStatus + } + + return listenerCertStatus{healthy: true, notAfter: cert.Status.NotAfter, secretName: secretName} +} + +// listenerSecretHealth loads the stored certificate and key and confirms they +// match and are still valid, catching a broken or mismatched certificate that +// would otherwise be served and break HTTPS for the listener. +func listenerSecretHealth( + ctx context.Context, + downstreamClient client.Client, + downstreamNamespace string, + secretName string, + hostname string, + now time.Time, +) listenerCertStatus { + logger := log.FromContext(ctx) + var secret corev1.Secret if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { if apierrors.IsNotFound(err) { @@ -675,7 +734,11 @@ func (r *GatewayReconciler) listenerCertHealth( } } - return listenerCertStatus{healthy: true, notAfter: cert.Status.NotAfter, secretName: secretName} + status := listenerCertStatus{healthy: true, secretName: secretName} + if leaf := keyPair.Leaf; leaf != nil { + status.notAfter = &metav1.Time{Time: leaf.NotAfter} + } + return status } // certIsReady reports whether a cert-manager Certificate has Ready=True. @@ -2999,6 +3062,13 @@ func (r *GatewayReconciler) SetupWithManager(mgr mcmanager.Manager) error { ) } + if r.Config.Gateway.CertificateService.Enabled { + builder = builder.Watches( + &certificatesv1alpha1.TLSCertificate{}, + r.listGatewaysForTLSCertificateFunc, + ) + } + return builder. WithOptions(controller.TypedOptions[mcreconcile.Request]{ MaxConcurrentReconciles: r.Config.Gateway.MaxConcurrentReconciles, diff --git a/internal/controller/gateway_downstream_certificate_solver_controller.go b/internal/controller/gateway_downstream_certificate_solver_controller.go index 717c8f3a..edd2a260 100644 --- a/internal/controller/gateway_downstream_certificate_solver_controller.go +++ b/internal/controller/gateway_downstream_certificate_solver_controller.go @@ -3,18 +3,14 @@ package controller import ( "context" "fmt" - "net/http" - envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -160,94 +156,13 @@ func (r *GatewayDownstreamCertificateSolverReconciler) Reconcile(ctx context.Con logger.Info("Successfully retrieved key and token from cert-manager Challenge for downstream certificate") - // Create HTTPRouteFilter with inline direct response - httpRouteFilter := &envoygatewayv1alpha1.HTTPRouteFilter{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: challenge.GetName(), - }, - } - - if err := controllerutil.SetControllerReference(challenge, httpRouteFilter, r.DownstreamCluster.GetScheme()); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to set controller reference on HTTPRouteFilter: %w", err) - } - - result, err := controllerutil.CreateOrUpdate(ctx, cl, httpRouteFilter, func() error { - httpRouteFilter.Labels = map[string]string{ - "meta.datumapis.com/http01-solver": labelValueTrue, - } - httpRouteFilter.Spec = envoygatewayv1alpha1.HTTPRouteFilterSpec{ - DirectResponse: &envoygatewayv1alpha1.HTTPDirectResponseFilter{ - ContentType: ptr.To("text/plain"), - StatusCode: ptr.To(http.StatusOK), - Body: &envoygatewayv1alpha1.CustomResponseBody{ - Type: ptr.To(envoygatewayv1alpha1.ResponseValueTypeInline), - Inline: ptr.To(key), - }, - }, - } - - return nil - }) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to create or update HTTPRouteFilter: %w", err) - } - logger.Info("HTTPRouteFilter reconciled", "result", result) - - // Attach HTTPRoute at expected path with direct response filter - httpRoute := &gatewayv1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: challenge.GetName(), - }, - } - - if err := controllerutil.SetControllerReference(challenge, httpRoute, r.DownstreamCluster.GetScheme()); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to set controller reference on HTTPRoute: %w", err) - } - - result, err = controllerutil.CreateOrUpdate(ctx, cl, httpRoute, func() error { - httpRoute.Labels = map[string]string{ - "meta.datumapis.com/http01-solver": labelValueTrue, - } - httpRoute.Spec = gatewayv1.HTTPRouteSpec{ - CommonRouteSpec: gatewayv1.CommonRouteSpec{ - ParentRefs: []gatewayv1.ParentReference{ - { - Name: gatewayv1.ObjectName(gateway.GetName()), - }, - }, - }, - Rules: []gatewayv1.HTTPRouteRule{ - { - Matches: []gatewayv1.HTTPRouteMatch{ - { - Path: &gatewayv1.HTTPPathMatch{ - Type: ptr.To(gatewayv1.PathMatchExact), - Value: ptr.To(fmt.Sprintf("/.well-known/acme-challenge/%s", token)), - }, - }, - }, - Filters: []gatewayv1.HTTPRouteFilter{ - { - Type: gatewayv1.HTTPRouteFilterExtensionRef, - ExtensionRef: &gatewayv1.LocalObjectReference{ - Group: envoygatewayv1alpha1.GroupName, - Kind: envoygatewayv1alpha1.KindHTTPRouteFilter, - Name: gatewayv1.ObjectName(httpRouteFilter.GetName()), - }, - }, - }, - }, - }, - } - - return nil - }) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to create or update HTTPRoute: %w", err) + if err := ensureHTTP01SolverRoutes(ctx, cl, r.DownstreamCluster.GetScheme(), challenge, &gateway, http01SolverRoute{ + name: challenge.GetName(), + token: token, + key: key, + }); err != nil { + return ctrl.Result{}, err } - logger.Info("HTTPRoute reconciled", "result", result) return ctrl.Result{}, nil } diff --git a/internal/controller/http01_solver.go b/internal/controller/http01_solver.go new file mode 100644 index 00000000..bfd14862 --- /dev/null +++ b/internal/controller/http01_solver.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "maps" + "net/http" + + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +const http01SolverLabel = "meta.datumapis.com/http01-solver" + +// http01SolverRoute names one ACME HTTP-01 answer to publish on a downstream +// gateway: GET /.well-known/acme-challenge/ returns . +type http01SolverRoute struct { + name string + token string + key string + labels map[string]string +} + +func (s http01SolverRoute) objectLabels() map[string]string { + labels := map[string]string{http01SolverLabel: labelValueTrue} + maps.Copy(labels, s.labels) + return labels +} + +func (s http01SolverRoute) path() string { + return fmt.Sprintf("/.well-known/acme-challenge/%s", s.token) +} + +// ensureHTTP01SolverRoutes creates or updates the HTTPRouteFilter and HTTPRoute +// that answer a challenge on the gateway, both controlled by owner so they +// leave with it. +func ensureHTTP01SolverRoutes( + ctx context.Context, + cl client.Client, + scheme *runtime.Scheme, + owner client.Object, + gateway *gatewayv1.Gateway, + solver http01SolverRoute, +) error { + logger := log.FromContext(ctx) + + httpRouteFilter := &envoygatewayv1alpha1.HTTPRouteFilter{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gateway.Namespace, + Name: solver.name, + }, + } + + result, err := controllerutil.CreateOrUpdate(ctx, cl, httpRouteFilter, func() error { + if err := controllerutil.SetControllerReference(owner, httpRouteFilter, scheme); err != nil { + return fmt.Errorf("failed to set controller reference on HTTPRouteFilter: %w", err) + } + httpRouteFilter.Labels = solver.objectLabels() + httpRouteFilter.Spec = envoygatewayv1alpha1.HTTPRouteFilterSpec{ + DirectResponse: &envoygatewayv1alpha1.HTTPDirectResponseFilter{ + ContentType: ptr.To("text/plain"), + StatusCode: ptr.To(http.StatusOK), + Body: &envoygatewayv1alpha1.CustomResponseBody{ + Type: ptr.To(envoygatewayv1alpha1.ResponseValueTypeInline), + Inline: ptr.To(solver.key), + }, + }, + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create or update HTTPRouteFilter: %w", err) + } + logger.Info("HTTPRouteFilter reconciled", "name", solver.name, "result", result) + + httpRoute := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gateway.Namespace, + Name: solver.name, + }, + } + + result, err = controllerutil.CreateOrUpdate(ctx, cl, httpRoute, func() error { + if err := controllerutil.SetControllerReference(owner, httpRoute, scheme); err != nil { + return fmt.Errorf("failed to set controller reference on HTTPRoute: %w", err) + } + httpRoute.Labels = solver.objectLabels() + httpRoute.Spec = gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{ + { + Name: gatewayv1.ObjectName(gateway.GetName()), + }, + }, + }, + Rules: []gatewayv1.HTTPRouteRule{ + { + Matches: []gatewayv1.HTTPRouteMatch{ + { + Path: &gatewayv1.HTTPPathMatch{ + Type: ptr.To(gatewayv1.PathMatchExact), + Value: ptr.To(solver.path()), + }, + }, + }, + Filters: []gatewayv1.HTTPRouteFilter{ + { + Type: gatewayv1.HTTPRouteFilterExtensionRef, + ExtensionRef: &gatewayv1.LocalObjectReference{ + Group: envoygatewayv1alpha1.GroupName, + Kind: envoygatewayv1alpha1.KindHTTPRouteFilter, + Name: gatewayv1.ObjectName(httpRouteFilter.GetName()), + }, + }, + }, + }, + }, + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create or update HTTPRoute: %w", err) + } + logger.Info("HTTPRoute reconciled", "name", solver.name, "result", result) + + return nil +} + +// deleteHTTP01SolverRoutes removes the solver objects published under name in +// the namespace, tolerating ones that are already gone. +func deleteHTTP01SolverRoutes(ctx context.Context, cl client.Client, namespace, name string) error { + meta := metav1.ObjectMeta{Namespace: namespace, Name: name} + for _, obj := range []client.Object{ + &gatewayv1.HTTPRoute{ObjectMeta: meta}, + &envoygatewayv1alpha1.HTTPRouteFilter{ObjectMeta: meta}, + } { + if err := cl.Delete(ctx, obj); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete solver %T %s/%s: %w", obj, namespace, name, err) + } + } + return nil +} diff --git a/internal/controller/http01_solver_test.go b/internal/controller/http01_solver_test.go new file mode 100644 index 00000000..54797054 --- /dev/null +++ b/internal/controller/http01_solver_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestEnsureAndDeleteHTTP01SolverRoutes(t *testing.T) { + testScheme := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(testScheme)) + require.NoError(t, gatewayv1.Install(testScheme)) + require.NoError(t, envoygatewayv1alpha1.AddToScheme(testScheme)) + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns-1", Name: "gw", UID: uuid.NewUUID()}, + } + cl := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(gateway).Build() + ctx := context.Background() + + solver := http01SolverRoute{ + name: "gw-acme-abc", + token: "the-token", + key: "the-key", + labels: map[string]string{tlsCertificateSolverLabel: "gw-https-0"}, + } + + require.NoError(t, ensureHTTP01SolverRoutes(ctx, cl, testScheme, gateway, gateway, solver)) + require.NoError(t, ensureHTTP01SolverRoutes(ctx, cl, testScheme, gateway, gateway, solver), "re-running is idempotent") + + key := client.ObjectKey{Namespace: "ns-1", Name: "gw-acme-abc"} + + var route gatewayv1.HTTPRoute + require.NoError(t, cl.Get(ctx, key, &route)) + assert.Equal(t, labelValueTrue, route.Labels[http01SolverLabel]) + assert.Equal(t, "gw-https-0", route.Labels[tlsCertificateSolverLabel]) + require.Len(t, route.Spec.ParentRefs, 1) + assert.Equal(t, gatewayv1.ObjectName("gw"), route.Spec.ParentRefs[0].Name) + require.Len(t, route.Spec.Rules, 1) + assert.Equal(t, "/.well-known/acme-challenge/the-token", ptr.Deref(route.Spec.Rules[0].Matches[0].Path.Value, "")) + assert.Equal(t, gatewayv1.ObjectName("gw-acme-abc"), route.Spec.Rules[0].Filters[0].ExtensionRef.Name) + owner := metav1.GetControllerOf(&route) + require.NotNil(t, owner) + assert.Equal(t, gateway.UID, owner.UID) + + var filter envoygatewayv1alpha1.HTTPRouteFilter + require.NoError(t, cl.Get(ctx, key, &filter)) + assert.Equal(t, "the-key", ptr.Deref(filter.Spec.DirectResponse.Body.Inline, "")) + assert.Equal(t, labelValueTrue, filter.Labels[http01SolverLabel]) + assert.Equal(t, gateway.UID, metav1.GetControllerOf(&filter).UID) + + require.NoError(t, deleteHTTP01SolverRoutes(ctx, cl, "ns-1", "gw-acme-abc")) + assert.True(t, apierrors.IsNotFound(cl.Get(ctx, key, &gatewayv1.HTTPRoute{}))) + assert.True(t, apierrors.IsNotFound(cl.Get(ctx, key, &envoygatewayv1alpha1.HTTPRouteFilter{}))) + assert.NoError(t, deleteHTTP01SolverRoutes(ctx, cl, "ns-1", "gw-acme-abc"), "deleting again is a no-op") +} diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go new file mode 100644 index 00000000..01468643 --- /dev/null +++ b/internal/controller/httpproxy_certificate_service.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" +) + +const certificateProvisioningMessage = "We're provisioning and applying a certificate to this hostname - it may take a few minutes" + +// tlsCertificateReadyCondition derives a hostname's CertificateReady condition +// from its TLSCertificate in the project control plane. A listener still served +// by a cert-manager Certificate that is not yet due for renewal has no +// TLSCertificate, so that Certificate answers for it until the switch. +func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( + ctx context.Context, + upstreamClient client.Client, + downstreamClient client.Client, + downstreamNamespace string, + upstreamNamespace string, + certName string, + legacyName string, + generation int64, +) metav1.Condition { + condition := metav1.Condition{ + Type: networkingv1alpha.HostnameConditionCertificateReady, + ObservedGeneration: generation, + } + + var cert certificatesv1alpha1.TLSCertificate + err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace, Name: certName}, &cert) + switch { + case err == nil: + condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) + return condition + case !apierrors.IsNotFound(err): + condition.Status = metav1.ConditionUnknown + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = fmt.Sprintf("Failed to get certificate: %v", err) + return condition + } + + legacy := newUnstructuredForGVK(certificateGVK) + if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: legacyName}, legacy); err == nil { + ready, readyErr := isCertificateReady(legacy) + switch { + case readyErr != nil: + condition.Status = metav1.ConditionUnknown + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = fmt.Sprintf("Failed to check certificate status: %v", readyErr) + case ready: + condition.Status = metav1.ConditionTrue + condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued + condition.Message = "Certificate is ready" + default: + condition.Status = metav1.ConditionFalse + condition.Reason, condition.Message = getCertificateReadyConditionReason(legacy) + } + return condition + } + + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = certificateProvisioningMessage + return condition +} + +// tlsCertificateReadyState maps TLSCertificate conditions onto the +// CertificateReady vocabulary: Ready wins, a rejected spec is a provisioning +// failure carrying the service's explanation, an in-flight order is a +// challenge in progress, and anything else is pending. +func tlsCertificateReadyState(cert *certificatesv1alpha1.TLSCertificate) (metav1.ConditionStatus, string, string) { + if apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { + return metav1.ConditionTrue, networkingv1alpha.CertificateReadyReasonCertificateIssued, "Certificate is ready" + } + + if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + message := accepted.Message + if message == "" { + message = "Certificate request was rejected" + } + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonProvisioningFailed, message + } + + if issuing := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionIssuing); issuing != nil && issuing.Status == metav1.ConditionTrue { + message := certificateProvisioningMessage + if issuing.Message != "" { + message = fmt.Sprintf("%s (%s)", certificateProvisioningMessage, issuing.Message) + } + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonChallengeInProgress, message + } + + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonPending, certificateProvisioningMessage +} + +// enqueueHTTPProxyForTLSCertificate follows TLSCertificate -> Gateway -> +// HTTPProxy through controller references in the same project control plane. +func (r *HTTPProxyReconciler) enqueueHTTPProxyForTLSCertificate(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + gatewayRef := metav1.GetControllerOf(obj) + if gatewayRef == nil || gatewayRef.Kind != KindGateway { + return nil + } + + var gateway gatewayv1.Gateway + if err := cl.GetClient().Get(ctx, client.ObjectKey{Namespace: obj.GetNamespace(), Name: gatewayRef.Name}, &gateway); err != nil { + if !apierrors.IsNotFound(err) { + log.FromContext(ctx).Error(err, "failed to get Gateway owning TLSCertificate", "tlscertificate", obj.GetName()) + } + return nil + } + + proxyRef := metav1.GetControllerOf(&gateway) + if proxyRef == nil || proxyRef.Kind != KindHTTPProxy { + return nil + } + + return []mcreconcile.Request{{ + ClusterName: clusterName, + Request: ctrl.Request{NamespacedName: client.ObjectKey{Namespace: gateway.Namespace, Name: proxyRef.Name}}, + }} + }) +} diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go new file mode 100644 index 00000000..5b803292 --- /dev/null +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + "go.datum.net/network-services-operator/internal/config" +) + +func TestBuildCertificateStatusesCertificateService(t *testing.T) { + t.Parallel() + + testScheme := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(testScheme)) + require.NoError(t, gatewayv1.Install(testScheme)) + require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + require.NoError(t, certificatesv1alpha1.AddToScheme(testScheme)) + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-ns", UID: types.UID("ns-uid")}} + downstreamNamespaceName := "ns-" + string(ns.UID) + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name}, + Spec: gatewayv1.GatewaySpec{Listeners: []gatewayv1.Listener{{ + Name: "https-hostname-0", + Protocol: gatewayv1.HTTPSProtocolType, + Hostname: ptr.To(gatewayv1.Hostname("app.example.com")), + }}}, + } + httpProxy := &networkingv1alpha.HTTPProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name, Generation: 3}, + Spec: networkingv1alpha.HTTPProxySpec{Hostnames: []gatewayv1.Hostname{"app.example.com"}}, + } + certName := tlsCertificateName("my-proxy", "https-hostname-0") + + tlsCert := func(conditions ...metav1.Condition) *certificatesv1alpha1.TLSCertificate { + return &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: certName}, + Status: certificatesv1alpha1.TLSCertificateStatus{Conditions: conditions}, + } + } + legacyReady := func() *unstructured.Unstructured { + cert := newUnstructuredForGVK(certificateGVK) + cert.SetNamespace(downstreamNamespaceName) + cert.SetName(listenerCertificateName("my-proxy", "https-hostname-0")) + _ = unstructured.SetNestedSlice(cert.Object, []any{map[string]any{"type": "Ready", "status": "True"}}, "status", "conditions") + return cert + } + + tests := []struct { + name string + upstream []client.Object + downstream []client.Object + wantStatus metav1.ConditionStatus + wantReason string + wantMessage string + }{ + { + name: "Ready maps to CertificateIssued", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, + }, + { + name: "Accepted=False maps to ProvisioningFailed with the service's message", + upstream: []client.Object{tlsCert(metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain", Message: "names under datum.net are denied", + })}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, + wantMessage: "names under datum.net are denied", + }, + { + name: "Issuing maps to ChallengeInProgress", + upstream: []client.Object{tlsCert( + metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue}, + metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue, Reason: "OrderInFlight"}, + )}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonChallengeInProgress, + }, + { + name: "accepted but not yet issuing is Pending", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue})}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonPending, + }, + { + name: "no TLSCertificate yet but a ready legacy Certificate answers", + downstream: []client.Object{legacyReady()}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, + }, + { + name: "nothing requested yet is Pending", + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonPending, + wantMessage: certificateProvisioningMessage, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + upstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(ns).WithObjects(tt.upstream...).Build() + downstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(tt.downstream...).Build() + + r := &HTTPProxyReconciler{ + Config: config.NetworkServicesOperator{Gateway: config.GatewayConfig{ + TargetDomain: "example.net", + CertificateService: config.CertificateServiceConfig{Enabled: true}, + }}, + DownstreamCluster: &clusterWithClient{c: downstreamClient, scheme: testScheme}, + } + + got := r.buildCertificateStatuses(context.Background(), upstreamClient, "local", gateway, httpProxy) + require.Len(t, got, 1) + assert.Equal(t, "app.example.com", got[0].Hostname) + cond := apimeta.FindStatusCondition(got[0].Conditions, networkingv1alpha.HostnameConditionCertificateReady) + require.NotNil(t, cond) + assert.Equal(t, tt.wantStatus, cond.Status) + assert.Equal(t, tt.wantReason, cond.Reason) + assert.Equal(t, int64(3), cond.ObservedGeneration) + if tt.wantMessage != "" { + assert.Equal(t, tt.wantMessage, cond.Message) + } + }) + } +} diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 2000fe6e..bb69b037 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -42,6 +42,7 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" "go.datum.net/network-services-operator/internal/config" downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" conditionutil "go.datum.net/network-services-operator/internal/util/condition" @@ -763,6 +764,13 @@ func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { builder = builder.WatchesRawSource(downstreamCertificateClusterSource) } + if r.Config.Gateway.CertificateService.Enabled { + builder = builder.Watches( + &certificatesv1alpha1.TLSCertificate{}, + r.enqueueHTTPProxyForTLSCertificate, + ) + } + return builder. WithOptions(controller.TypedOptions[mcreconcile.Request]{ MaxConcurrentReconciles: r.Config.HTTPProxy.MaxConcurrentReconciles, @@ -1507,6 +1515,13 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( } certName := resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-%s", gateway.Name, l.Name)) + + if r.Config.Gateway.CertificateService.Enabled { + apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), certName, httpProxy.Generation)) + statuses = append(statuses, hs) + continue + } + certificate := newUnstructuredForGVK(certificateGVK) certKey := client.ObjectKey{Namespace: downstreamNamespaceName, Name: certName} From 3c739f9d2d46f62bb18cc750b7f64c7b1ec387f5 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 1 Oct 2026 21:30:23 -0500 Subject: [PATCH 2/8] fix: isolate the certificate service from routing Review of the certificate service consumer found that a failure in the new issuance step could stop a gateway's routes, DNS and status from reconciling, that issued material was mirrored without being checked, and that a flag turned back off would reissue every migrated hostname. Key changes: - Record a failed certificate step in the listener's status and retry it with backoff instead of ending the gateway reconcile; routes, DNS and status continue whatever the service does - Refuse a service-side Secret reference outside the configured service namespace, and verify key pair, hostname coverage and validity before any material may replace what is serving - Never delete a TLSCertificate this gateway does not control, and name TLSCertificates from a hash of gateway and listener so two listeners can never share one - Scope HTTP-01 solver routes to the claimed hostname and skip tokens that are not ACME tokens without failing the step - Hand a hostname to the service ahead of cert-manager's renewal time, never while cert-manager has a renewal in flight - With the flag off, keep serving a Secret the service handed over or issued until it is due for renewal, then return the hostname to cert-manager; remove the gateway's own TLSCertificates and solver routes so the service stops renewing it - Report a hostname as ready while its previous certificate still serves, and name the DNS records the service is waiting on Related to datum-cloud/enhancements#913 --- internal/config/config.go | 15 + internal/config/config_test.go | 10 +- internal/config/zz_generated.defaults.go | 4 + .../controller/gateway_certificate_service.go | 500 ++++++++++++++---- .../gateway_certificate_service_test.go | 365 ++++++++++++- internal/controller/gateway_controller.go | 36 +- internal/controller/http01_solver.go | 10 +- .../httpproxy_certificate_service.go | 28 +- .../httpproxy_certificate_service_test.go | 32 ++ internal/controller/httpproxy_controller.go | 2 +- 10 files changed, 868 insertions(+), 134 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 6ecad17a..8c154c32 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -930,6 +930,18 @@ type CertificateServiceConfig struct { // where it keeps the service-side copy of each issued Secret. Empty // means the cluster this operator runs in. KubeconfigPath string `json:"kubeconfigPath,omitempty"` + + // SecretNamespace is the only namespace on that cluster a TLSCertificate's + // serviceSecretRef may point into; a reference anywhere else is refused. + // + // +default="certificates-system" + SecretNamespace string `json:"secretNamespace,omitempty"` +} + +func SetDefaults_CertificateServiceConfig(obj *CertificateServiceConfig) { + if obj.SecretNamespace == "" { + obj.SecretNamespace = "certificates-system" + } } // RestConfig returns the connection to the certificate service's cluster. @@ -1529,6 +1541,9 @@ func (c *GatewayConfig) validate() error { if c.CertificateService.Enabled && c.DisableHostnameVerification { errs = append(errs, errors.New("certificateService.enabled requires hostname verification: the certificate service issues for any hostname it is handed")) } + if c.CertificateService.Enabled && strings.TrimSpace(c.CertificateService.SecretNamespace) == "" { + errs = append(errs, errors.New("certificateService.secretNamespace is required when certificateService.enabled")) + } return errors.Join(errs...) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ae6b5ef4..caf38a02 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -216,12 +216,17 @@ func TestGatewayConfig_ValidateCertificateService(t *testing.T) { }, { name: "enabled with hostname verification validates", + gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system"}}, + }, + { + name: "enabled without a secret namespace is rejected", gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true}}, + wantSub: "certificateService.secretNamespace is required", }, { name: "enabled without hostname verification is rejected", gateway: GatewayConfig{ - CertificateService: CertificateServiceConfig{Enabled: true}, + CertificateService: CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system"}, DisableHostnameVerification: true, }, wantSub: "certificateService.enabled requires hostname verification", @@ -254,4 +259,7 @@ func TestSetObjectDefaults_CertificateService(t *testing.T) { if cfg.Gateway.CertificateService.KubeconfigPath != "" { t.Errorf("certificateService.kubeconfigPath should default to empty, got %q", cfg.Gateway.CertificateService.KubeconfigPath) } + if got, want := cfg.Gateway.CertificateService.SecretNamespace, "certificates-system"; got != want { + t.Errorf("certificateService.secretNamespace = %q, want %q", got, want) + } } diff --git a/internal/config/zz_generated.defaults.go b/internal/config/zz_generated.defaults.go index 84aa88f5..bf274d86 100644 --- a/internal/config/zz_generated.defaults.go +++ b/internal/config/zz_generated.defaults.go @@ -292,6 +292,10 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { if in.Gateway.CertificateReissuance.MaxRetries == 0 { in.Gateway.CertificateReissuance.MaxRetries = 3 } + SetDefaults_CertificateServiceConfig(&in.Gateway.CertificateService) + if in.Gateway.CertificateService.SecretNamespace == "" { + in.Gateway.CertificateService.SecretNamespace = "certificates-system" + } if in.HTTPProxy.GatewayClassName == "" { in.HTTPProxy.GatewayClassName = "datum-external-global-proxy" } diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go index 04c4c856..67d5b237 100644 --- a/internal/controller/gateway_certificate_service.go +++ b/internal/controller/gateway_certificate_service.go @@ -5,17 +5,22 @@ package controller import ( "context" "crypto/sha256" + "crypto/tls" + "crypto/x509" "encoding/hex" "fmt" + "regexp" "slices" "strings" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -29,7 +34,6 @@ import ( certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" - "go.datum.net/network-services-operator/internal/util/resourcename" ) // +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates,verbs=get;list;watch;create;update;patch;delete @@ -37,25 +41,54 @@ import ( const tlsCertificateSolverLabel = "networking.datumapis.com/tlscertificate-solver" -const KindTLSCertificate = "TLSCertificate" +// tlsCertificateManagedLabel marks a downstream Secret the certificate service +// path owns: either handed over from cert-manager at the switch or written by +// the mirror. The cert-manager path reads it to avoid reissuing on rollback. +const tlsCertificateManagedLabel = "networking.datumapis.com/certificate-service" -const tlsCertificateRecreateDelay = 5 * time.Second +const KindTLSCertificate = "TLSCertificate" // tlsCertificateMirrorAdmitDelay is how soon the listener is re-evaluated after // its Secret lands downstream, since listener health was judged before the // mirror in the same pass. const tlsCertificateMirrorAdmitDelay = time.Second -// tlsCertificateName is the listener's TLSCertificate name. The service caps -// names at 63 characters, so a long gateway and listener pair is truncated and -// hashed where the cert-manager Certificate name was not. +// tlsCertificateSwitchLead is how far ahead of cert-manager's own renewal time +// a hostname moves to the service, so the two never order for the same name. +const tlsCertificateSwitchLead = 48 * time.Hour + +const ( + certificateServiceBackoffBase = 5 * time.Second + certificateServiceBackoffMax = 5 * time.Minute + legacyRecheckMax = 6 * time.Hour +) + +var acmeTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`) + +var nonDNSLabelChars = regexp.MustCompile(`[^a-z0-9-]+`) + +// tlsCertificateName names the listener's TLSCertificate: a readable prefix +// from the gateway and listener plus a hash of both, so "a-b"/"c" and "a"/"b-c" +// never meet, inside the service's 63 character cap and valid as a DNS label +// whatever the gateway was called. func tlsCertificateName(gatewayName string, listenerName gatewayv1.SectionName) string { - return resourcename.GetValidDNS1035Name(fmt.Sprintf("%s-%s", gatewayName, listenerName)) + sum := sha256.Sum256([]byte(gatewayName + "/" + string(listenerName))) + suffix := hex.EncodeToString(sum[:])[:10] + + prefix := nonDNSLabelChars.ReplaceAllString(strings.ToLower(gatewayName+"-"+string(listenerName)), "-") + if max := 63 - len(suffix) - 1; len(prefix) > max { + prefix = prefix[:max] + } + prefix = strings.Trim(prefix, "-") + if prefix == "" { + return suffix + } + return prefix + "-" + suffix } func tlsCertificateSolverName(certName, token string) string { sum := sha256.Sum256([]byte(token)) - return resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-acme-%s", certName, hex.EncodeToString(sum[:])[:10])) + return fmt.Sprintf("%s-acme-%s", certName, hex.EncodeToString(sum[:])[:10]) } // listenerWantsOwnCertificate reports whether a listener is one the controller @@ -76,11 +109,52 @@ func (r *GatewayReconciler) listenerWantsOwnCertificate(l gatewayv1.Listener, cl return hostname, true } +// listenerIssuerResolvable applies the same rule the cert-manager path does to +// the certificate-issuer option: an `auto` that maps to nothing and inherits +// from no other listener leaves the listener un-programmed. +func (r *GatewayReconciler) listenerIssuerResolvable(upstreamGateway *gatewayv1.Gateway, l gatewayv1.Listener) bool { + issuer := string(l.TLS.Options[certificateIssuerTLSOption]) + if mapped := r.Config.Gateway.ClusterIssuerMap[issuer]; mapped != "" { + issuer = mapped + } + return issuer != autoIssuerSentinel || r.resolveAutoIssuer(upstreamGateway) != "" +} + +// certificateServiceRequeue hands back the delay before the next attempt for a +// gateway whose certificate-service step failed, doubling per consecutive +// failure, and forgets the gateway once a pass succeeds. +func (r *GatewayReconciler) certificateServiceRequeue(gateway types.UID, failed bool) time.Duration { + if !failed { + r.certificateServiceFailures.Delete(gateway) + return 0 + } + attempts := 1 + if previous, ok := r.certificateServiceFailures.Load(gateway); ok { + attempts = previous.(int) + 1 + } + r.certificateServiceFailures.Store(gateway, attempts) + delay := certificateServiceBackoffBase << (attempts - 1) + if attempts > 10 || delay > certificateServiceBackoffMax { + delay = certificateServiceBackoffMax + } + return delay +} + +func certificateServiceUnavailableMessage(hostname string) string { + return fmt.Sprintf("We couldn't request a TLS certificate for %s just now and will keep trying. HTTPS for this hostname stays as it is in the meantime.", hostname) +} + +func certificateBeingReplacedMessage(hostname string) string { + return fmt.Sprintf("The TLS certificate request for %s is being replaced. HTTPS for this hostname stays as it is in the meantime.", hostname) +} + // ensureListenerTLSCertificates is the certificate-service counterpart of // ensureListenerCertificates: one TLSCertificate per custom hostname in the // project control plane, its HTTP-01 challenges answered on the downstream // gateway, and its issued Secret mirrored downstream under the name the -// listener references. +// listener references. Nothing here fails the gateway reconcile: a listener +// whose step failed keeps whatever Secret it has, the failure is returned as a +// message for its status, and the whole step retries with backoff. func (r *GatewayReconciler) ensureListenerTLSCertificates( ctx context.Context, upstreamClient client.Client, @@ -89,16 +163,29 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( downstreamClient client.Client, downstreamStrategy downstreamclient.ResourceStrategy, claimedHostnames []string, -) (result Result) { +) (result Result, issues map[gatewayv1.SectionName]string) { logger := log.FromContext(ctx) now := time.Now() + issues = make(map[gatewayv1.SectionName]string) desiredCerts := sets.New[string]() legacyKeep := sets.New[string]() liveSolvers := sets.New[string]() + failed := false + + requeueSooner := func(d time.Duration) { + if d > 0 && (result.RequeueAfter == 0 || d < result.RequeueAfter) { + result.RequeueAfter = d + } + } + fail := func(l gatewayv1.Listener, hostname string, err error) { + logger.Error(err, "certificate service step failed", "listener", l.Name, "hostname", hostname) + issues[l.Name] = certificateServiceUnavailableMessage(hostname) + failed = true + } for _, l := range upstreamGateway.Spec.Listeners { hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) - if !wanted { + if !wanted || !r.listenerIssuerResolvable(upstreamGateway, l) { continue } @@ -108,77 +195,93 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( legacy, err := r.ownedLegacyCertificate(ctx, downstreamClient, upstreamGateway, downstreamGateway, legacyName) if err != nil { - result.Err = err - return result - } - if legacy != nil && legacyCertificateHolds(legacy, now) { - legacyKeep.Insert(legacyName) + fail(l, hostname, err) continue } if legacy != nil { - logger.Info("legacy certificate due for renewal, switching hostname to the certificate service", "certificate", legacyName, "hostname", hostname) + if holds, recheck := legacyCertificateHolds(legacy, now); holds { + legacyKeep.Insert(legacyName) + requeueSooner(min(recheck, legacyRecheckMax)) + continue + } + logger.Info("legacy certificate nearing renewal, switching hostname to the certificate service", "certificate", legacyName, "hostname", hostname) if err := r.retireLegacyCertificate(ctx, downstreamStrategy, upstreamGateway, legacy, secretName); err != nil { - result.Err = err - return result + fail(l, hostname, err) + continue } } desiredCerts.Insert(certName) - cert, recreating, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) + cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) if err != nil { - result.Err = err - return result + fail(l, hostname, err) + continue } - if recreating { - result.RequeueAfter = tlsCertificateRecreateDelay + if state != tlsCertificateSettled { + issues[l.Name] = certificateBeingReplacedMessage(hostname) + failed = true continue } - solvers, err := r.serveTLSCertificateChallenges(ctx, downstreamClient, downstreamGateway, cert, hostname) + solvers, rejected, err := r.serveTLSCertificateChallenges(ctx, downstreamClient, downstreamGateway, cert, hostname) if err != nil { - result.Err = err - return result + fail(l, hostname, err) + continue } liveSolvers.Insert(solvers...) + if rejected != "" { + issues[l.Name] = rejected + } - mirrored, err := r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName) + mirrored, rejected, err := r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName, hostname, now) if err != nil { - result.Err = err - return result + fail(l, hostname, err) + continue + } + if rejected != "" { + issues[l.Name] = rejected } if mirrored { - result.RequeueAfter = tlsCertificateMirrorAdmitDelay + requeueSooner(tlsCertificateMirrorAdmitDelay) } } if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, desiredCerts); err != nil { - result.Err = err - return result + logger.Error(err, "failed to clean up TLSCertificates") + failed = true } - if err := r.deleteStaleLegacyCertificates(ctx, downstreamClient, upstreamGateway, downstreamGateway, legacyKeep); err != nil { - result.Err = err - return result + logger.Error(err, "failed to clean up legacy Certificates") + failed = true } - if err := r.deleteStaleTLSCertificateSolvers(ctx, downstreamClient, downstreamGateway, liveSolvers); err != nil { - result.Err = err - return result + logger.Error(err, "failed to clean up HTTP-01 solvers") + failed = true } - return result + requeueSooner(r.certificateServiceRequeue(upstreamGateway.UID, failed)) + + return result, issues } +type tlsCertificateState int + +const ( + tlsCertificateSettled tlsCertificateState = iota + tlsCertificateReplacing +) + // ensureTLSCertificate creates the listener's TLSCertificate or confirms the // existing one. dnsNames and secretName are immutable on the service's API, so -// one that no longer matches is deleted and requested again on the next pass. +// one of ours that no longer matches is deleted and requested again once it is +// gone. One this gateway does not control is never touched. func (r *GatewayReconciler) ensureTLSCertificate( ctx context.Context, upstreamClient client.Client, upstreamGateway *gatewayv1.Gateway, certName, secretName, hostname string, -) (cert *certificatesv1alpha1.TLSCertificate, recreating bool, err error) { +) (*certificatesv1alpha1.TLSCertificate, tlsCertificateState, error) { logger := log.FromContext(ctx) desiredSpec := certificatesv1alpha1.TLSCertificateSpec{ @@ -187,66 +290,71 @@ func (r *GatewayReconciler) ensureTLSCertificate( SecretName: secretName, } - cert = &certificatesv1alpha1.TLSCertificate{ + cert := &certificatesv1alpha1.TLSCertificate{ ObjectMeta: metav1.ObjectMeta{ Namespace: upstreamGateway.Namespace, Name: certName, }, } - err = upstreamClient.Get(ctx, client.ObjectKeyFromObject(cert), cert) + err := upstreamClient.Get(ctx, client.ObjectKeyFromObject(cert), cert) switch { case apierrors.IsNotFound(err): if err := controllerutil.SetControllerReference(upstreamGateway, cert, upstreamClient.Scheme()); err != nil { - return nil, false, fmt.Errorf("failed to set controller reference on TLSCertificate %s: %w", certName, err) + return nil, tlsCertificateSettled, fmt.Errorf("failed to set controller reference on TLSCertificate %s: %w", certName, err) } cert.Spec = desiredSpec if err := upstreamClient.Create(ctx, cert); err != nil { - return nil, false, fmt.Errorf("failed to create TLSCertificate %s: %w", certName, err) + return nil, tlsCertificateSettled, fmt.Errorf("failed to create TLSCertificate %s: %w", certName, err) } logger.Info("TLSCertificate requested", "tlscertificate", certName, "hostname", hostname) - return cert, false, nil + return cert, tlsCertificateSettled, nil case err != nil: - return nil, false, fmt.Errorf("failed to get TLSCertificate %s: %w", certName, err) + return nil, tlsCertificateSettled, fmt.Errorf("failed to get TLSCertificate %s: %w", certName, err) + } + + if !metav1.IsControlledBy(cert, upstreamGateway) { + return nil, tlsCertificateSettled, fmt.Errorf("TLSCertificate %s exists but is not controlled by Gateway %s", certName, upstreamGateway.Name) + } + + if !cert.DeletionTimestamp.IsZero() { + return cert, tlsCertificateReplacing, nil } if !slices.Equal(cert.Spec.DNSNames, desiredSpec.DNSNames) || cert.Spec.SecretName != desiredSpec.SecretName { logger.Info("TLSCertificate no longer matches its listener, requesting it again", "tlscertificate", certName, "hostname", hostname) - if err := upstreamClient.Delete(ctx, cert); client.IgnoreNotFound(err) != nil { - return nil, false, fmt.Errorf("failed to delete TLSCertificate %s: %w", certName, err) + if err := upstreamClient.Delete(ctx, cert, client.Preconditions{UID: &cert.UID}); client.IgnoreNotFound(err) != nil { + return nil, tlsCertificateSettled, fmt.Errorf("failed to delete TLSCertificate %s: %w", certName, err) } - return nil, true, nil + return cert, tlsCertificateReplacing, nil } - if metav1.IsControlledBy(cert, upstreamGateway) && cert.Spec.Issuance == desiredSpec.Issuance { - return cert, false, nil + if cert.Spec.Issuance == desiredSpec.Issuance { + return cert, tlsCertificateSettled, nil } - if err := controllerutil.SetControllerReference(upstreamGateway, cert, upstreamClient.Scheme()); err != nil { - return nil, false, fmt.Errorf("failed to set controller reference on TLSCertificate %s: %w", certName, err) - } cert.Spec.Issuance = desiredSpec.Issuance if err := upstreamClient.Update(ctx, cert); err != nil { - return nil, false, fmt.Errorf("failed to update TLSCertificate %s: %w", certName, err) + return nil, tlsCertificateSettled, fmt.Errorf("failed to update TLSCertificate %s: %w", certName, err) } logger.Info("TLSCertificate reconciled", "tlscertificate", certName, "operation", "updated") - return cert, false, nil + return cert, tlsCertificateSettled, nil } // serveTLSCertificateChallenges publishes every pending HTTP-01 challenge the // service reports for the hostname this listener claimed and returns the solver // names for every such challenge still in status, so a challenge that has moved // on to Valid keeps its answer until the service drops it. A challenge for any -// other name is not ours to answer, whatever status says. +// other name, or with a token that is not an ACME token, is not ours to answer, +// whatever status says. func (r *GatewayReconciler) serveTLSCertificateChallenges( ctx context.Context, downstreamClient client.Client, downstreamGateway *gatewayv1.Gateway, cert *certificatesv1alpha1.TLSCertificate, hostname string, -) ([]string, error) { +) (live []string, rejected string, err error) { logger := log.FromContext(ctx) - var live []string for _, challenge := range cert.Status.Challenges { if challenge.Type != certificatesv1alpha1.ChallengeTypeHTTP01 { @@ -257,6 +365,11 @@ func (r *GatewayReconciler) serveTLSCertificateChallenges( "tlscertificate", cert.Name, "claimed", hostname, "challenge_dns_name", challenge.DNSName) continue } + if !acmeTokenPattern.MatchString(challenge.Token) { + logger.Info("ignoring HTTP-01 challenge whose token is not an ACME token", "tlscertificate", cert.Name, "hostname", hostname) + rejected = certificateServiceUnavailableMessage(hostname) + continue + } name := tlsCertificateSolverName(cert.Name, challenge.Token) live = append(live, name) @@ -265,24 +378,26 @@ func (r *GatewayReconciler) serveTLSCertificateChallenges( } err := ensureHTTP01SolverRoutes(ctx, downstreamClient, downstreamClient.Scheme(), downstreamGateway, downstreamGateway, http01SolverRoute{ - name: name, - token: challenge.Token, - key: challenge.Key, - labels: map[string]string{tlsCertificateSolverLabel: cert.Name}, + name: name, + token: challenge.Token, + key: challenge.Key, + hostnames: []gatewayv1.Hostname{gatewayv1.Hostname(hostname)}, + labels: map[string]string{tlsCertificateSolverLabel: cert.Name}, }) if err != nil { - return nil, fmt.Errorf("failed to serve HTTP-01 challenge for TLSCertificate %s: %w", cert.Name, err) + return nil, "", fmt.Errorf("failed to serve HTTP-01 challenge for TLSCertificate %s: %w", cert.Name, err) } } - return live, nil + return live, rejected, nil } // mirrorTLSCertificateSecret copies the service-side issued Secret, read with -// the operator's own credentials, into the downstream gateway namespace under -// the listener's secret name, stamped with the upstream-owner labels the -// federation policy selects. The project-namespace copy is the tenant's and is -// never what reaches an edge. +// the operator's own credentials from the configured service namespace, into +// the downstream gateway namespace under the listener's secret name, stamped +// with the upstream-owner labels the federation policy selects. The material is +// parsed, matched, checked against the hostname and its expiry before it may +// replace what is serving; the project-namespace copy is never read. func (r *GatewayReconciler) mirrorTLSCertificateSecret( ctx context.Context, downstreamStrategy downstreamclient.ResourceStrategy, @@ -290,48 +405,66 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( downstreamGateway *gatewayv1.Gateway, cert *certificatesv1alpha1.TLSCertificate, secretName string, -) (mirrored bool, err error) { + hostname string, + now time.Time, +) (mirrored bool, rejected string, err error) { logger := log.FromContext(ctx) if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { - return false, nil + return false, "", nil } - if cert.Status.ServiceSecretRef == nil || cert.Status.ServiceSecretRef.Name == "" { + ref := cert.Status.ServiceSecretRef + if ref == nil || ref.Name == "" { logger.Info("TLSCertificate is Ready without a service-side Secret reference", "tlscertificate", cert.Name) - return false, nil + return false, "", nil + } + if ref.Namespace != r.Config.Gateway.CertificateService.SecretNamespace { + logger.Info("refusing service-side Secret outside the certificate service namespace", + "tlscertificate", cert.Name, "namespace", ref.Namespace, "expected", r.Config.Gateway.CertificateService.SecretNamespace) + return false, certificateServiceUnavailableMessage(hostname), nil } if r.CertificateServiceReader == nil { - return false, fmt.Errorf("certificate service enabled without a client for its cluster") + return false, "", fmt.Errorf("certificate service enabled without a client for its cluster") + } + + downstreamClient := downstreamStrategy.GetClient() + mirror := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: downstreamGateway.Namespace, + Name: secretName, + }, + } + if err := downstreamClient.Get(ctx, client.ObjectKeyFromObject(mirror), mirror); client.IgnoreNotFound(err) != nil { + return false, "", fmt.Errorf("failed to get Secret %s: %w", secretName, err) + } + if mirrorHoldsIssuance(mirror, cert) { + return false, "", nil } var source corev1.Secret - sourceKey := client.ObjectKey{Namespace: cert.Status.ServiceSecretRef.Namespace, Name: cert.Status.ServiceSecretRef.Name} + sourceKey := client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name} if err := r.CertificateServiceReader.Get(ctx, sourceKey, &source); err != nil { if apierrors.IsNotFound(err) { logger.Info("TLSCertificate is Ready but its service-side Secret is not readable yet", "tlscertificate", cert.Name, "secret", sourceKey) - return false, nil + return false, "", nil } - return false, fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) - } - if len(source.Data["tls.crt"]) == 0 || len(source.Data["tls.key"]) == 0 { - logger.Info("service-side Secret holds no certificate material yet", "tlscertificate", cert.Name, "secret", sourceKey) - return false, nil + return false, "", fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) } - mirror := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: downstreamGateway.Namespace, - Name: secretName, - }, + if err := validateIssuedMaterial(source.Data["tls.crt"], source.Data["tls.key"], hostname, now); err != nil { + logger.Info("refusing service-side Secret that does not hold a usable certificate for the hostname", + "tlscertificate", cert.Name, "secret", sourceKey, "reason", err.Error()) + return false, certificateServiceUnavailableMessage(hostname), nil } - op, err := controllerutil.CreateOrUpdate(ctx, downstreamStrategy.GetClient(), mirror, func() error { + op, err := controllerutil.CreateOrUpdate(ctx, downstreamClient, mirror, func() error { if mirror.CreationTimestamp.IsZero() { mirror.Type = corev1.SecretTypeTLS } if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, mirror); err != nil { return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) } + mirror.Labels[tlsCertificateManagedLabel] = labelValueTrue mirror.Data = map[string][]byte{ "tls.crt": source.Data["tls.crt"], "tls.key": source.Data["tls.key"], @@ -342,13 +475,60 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( return nil }) if err != nil { - return false, fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) + return false, "", fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) } if op != controllerutil.OperationResultNone { logger.Info("issued Secret mirrored downstream", "secret", secretName, "operation", op) } - return op != controllerutil.OperationResultNone, nil + return op != controllerutil.OperationResultNone, "", nil +} + +// mirrorHoldsIssuance reports whether the downstream Secret already carries the +// issuance the TLSCertificate describes, so the service cluster is not read +// again for it. +func mirrorHoldsIssuance(mirror *corev1.Secret, cert *certificatesv1alpha1.TLSCertificate) bool { + if mirror.CreationTimestamp.IsZero() || mirror.Labels[tlsCertificateManagedLabel] != labelValueTrue || cert.Status.NotAfter == nil { + return false + } + leaf, err := parseLeafCertificate(mirror.Data["tls.crt"], mirror.Data["tls.key"]) + if err != nil { + return false + } + return leaf.NotAfter.Truncate(time.Second).Equal(cert.Status.NotAfter.Truncate(time.Second)) +} + +func parseLeafCertificate(certPEM, keyPEM []byte) (*x509.Certificate, error) { + keyPair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, err + } + if keyPair.Leaf != nil { + return keyPair.Leaf, nil + } + return x509.ParseCertificate(keyPair.Certificate[0]) +} + +// validateIssuedMaterial accepts only a matching key pair whose leaf covers the +// hostname and is valid now. +func validateIssuedMaterial(certPEM, keyPEM []byte, hostname string, now time.Time) error { + if len(certPEM) == 0 || len(keyPEM) == 0 { + return fmt.Errorf("secret holds no certificate material") + } + leaf, err := parseLeafCertificate(certPEM, keyPEM) + if err != nil { + return fmt.Errorf("certificate and key do not form a key pair: %w", err) + } + if err := leaf.VerifyHostname(hostname); err != nil { + return fmt.Errorf("certificate does not cover %s: %w", hostname, err) + } + if now.Before(leaf.NotBefore) { + return fmt.Errorf("certificate is not valid until %s", leaf.NotBefore.UTC().Format(time.RFC3339)) + } + if !leaf.NotAfter.After(now.Add(listenerCertExpiryMargin)) { + return fmt.Errorf("certificate expired at %s", leaf.NotAfter.UTC().Format(time.RFC3339)) + } + return nil } // ownedLegacyCertificate returns the listener's cert-manager Certificate from @@ -373,16 +553,33 @@ func (r *GatewayReconciler) ownedLegacyCertificate( return &legacy, nil } -// legacyCertificateHolds reports whether a cert-manager Certificate is serving -// and not yet due for renewal, so switching it to the service now would issue a -// certificate nobody needs yet. -func legacyCertificateHolds(cert *cmv1.Certificate, now time.Time) bool { - return certIsServing(cert, now) && cert.Status.RenewalTime != nil && cert.Status.RenewalTime.After(now) +// legacyCertificateHolds reports whether a cert-manager Certificate should keep +// its hostname for now: it is serving and not yet within the switch lead of its +// renewal time, or cert-manager is mid-renewal and must not be interrupted. The +// returned duration is when to look again. +func legacyCertificateHolds(cert *cmv1.Certificate, now time.Time) (bool, time.Duration) { + if !certIsServing(cert, now) { + return false, 0 + } + for _, c := range cert.Status.Conditions { + if c.Type == cmv1.CertificateConditionIssuing && c.Status == cmmeta.ConditionTrue { + return true, 10 * time.Minute + } + } + if cert.Status.RenewalTime == nil { + return false, 0 + } + switchAt := cert.Status.RenewalTime.Add(-tlsCertificateSwitchLead) + if now.Before(switchAt) { + return true, switchAt.Sub(now) + } + return false, 0 } // retireLegacyCertificate hands the listener over to the service: the Secret // gains the gateway's own owner so cert-manager's owner reference no longer -// takes it along, then the Certificate goes so cert-manager does not renew a +// takes it along and the hand-over label the cert-manager path honours on +// rollback, then the Certificate goes so cert-manager does not renew a // hostname the service is about to issue for. func (r *GatewayReconciler) retireLegacyCertificate( ctx context.Context, @@ -403,12 +600,13 @@ func (r *GatewayReconciler) retireLegacyCertificate( if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, &secret); err != nil { return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) } + secret.Labels[tlsCertificateManagedLabel] = labelValueTrue if err := downstreamClient.Update(ctx, &secret); err != nil { return fmt.Errorf("failed to keep Secret %s past its Certificate: %w", secretName, err) } } - if err := downstreamClient.Delete(ctx, legacy); client.IgnoreNotFound(err) != nil { + if err := downstreamClient.Delete(ctx, legacy, client.Preconditions{UID: &legacy.UID}); client.IgnoreNotFound(err) != nil { return fmt.Errorf("failed to delete legacy Certificate %s: %w", legacy.Name, err) } return nil @@ -452,6 +650,9 @@ func certificateOwnedByGateway(cert *cmv1.Certificate, upstreamGateway, downstre return ownedByStrategy || metav1.IsControlledBy(cert, downstreamGateway) } +// deleteStaleTLSCertificates removes TLSCertificates this gateway controls that +// no listener wants any more. Ownership is the controller reference's UID, so a +// namesake another gateway owns is left alone. func (r *GatewayReconciler) deleteStaleTLSCertificates( ctx context.Context, upstreamClient client.Client, @@ -467,11 +668,11 @@ func (r *GatewayReconciler) deleteStaleTLSCertificates( for i := range certs.Items { cert := &certs.Items[i] - if desiredCerts.Has(cert.Name) || !metav1.IsControlledBy(cert, upstreamGateway) { + if desiredCerts.Has(cert.Name) || !metav1.IsControlledBy(cert, upstreamGateway) || !cert.DeletionTimestamp.IsZero() { continue } logger.Info("deleting stale TLSCertificate", "tlscertificate", cert.Name) - if err := upstreamClient.Delete(ctx, cert); client.IgnoreNotFound(err) != nil { + if err := upstreamClient.Delete(ctx, cert, client.Preconditions{UID: &cert.UID}); client.IgnoreNotFound(err) != nil { return fmt.Errorf("failed to delete stale TLSCertificate %s: %w", cert.Name, err) } } @@ -509,6 +710,75 @@ func (r *GatewayReconciler) deleteStaleTLSCertificateSolvers( return nil } +// cleanupCertificateServiceLeftovers runs on the cert-manager path for a +// gateway that was once on the service: it drops the solver routes and the +// TLSCertificates this gateway controls, so the service stops renewing names +// cert-manager has taken back. Nothing here fails the reconcile, and the +// project control plane is only asked about TLSCertificates when a handed-over +// Secret shows the gateway was ever on the service. +func (r *GatewayReconciler) cleanupCertificateServiceLeftovers( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + downstreamClient client.Client, +) { + logger := log.FromContext(ctx) + + if err := r.deleteStaleTLSCertificateSolvers(ctx, downstreamClient, downstreamGateway, sets.New[string]()); err != nil { + logger.Error(err, "failed to remove certificate service solvers after rollback") + } + + var handedOver corev1.SecretList + if err := downstreamClient.List(ctx, &handedOver, + client.InNamespace(downstreamGateway.Namespace), + client.MatchingLabels{ + tlsCertificateManagedLabel: labelValueTrue, + downstreamclient.UpstreamOwnerKindLabel: KindGateway, + downstreamclient.UpstreamOwnerNameLabel: upstreamGateway.Name, + downstreamclient.UpstreamOwnerNamespaceLabel: upstreamGateway.Namespace, + }, + ); err != nil { + logger.Error(err, "failed to list handed-over Secrets after rollback") + return + } + if len(handedOver.Items) == 0 { + return + } + + if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, sets.New[string]()); err != nil { + logger.Error(err, "failed to remove TLSCertificates after rollback") + } +} + +// handedOverSecretHealth reports whether the listener's downstream Secret was +// handed to or written by the certificate service and still holds a usable +// certificate for the hostname. renewalDue says the certificate is in the last +// third of its life, where cert-manager would renew it anyway. +func handedOverSecretHealth( + ctx context.Context, + downstreamClient client.Client, + downstreamNamespace string, + secretName string, + hostname string, + now time.Time, +) (status listenerCertStatus, handedOver bool, renewalDue bool) { + var secret corev1.Secret + if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { + return listenerCertStatus{}, false, false + } + if secret.Labels[tlsCertificateManagedLabel] != labelValueTrue { + return listenerCertStatus{}, false, false + } + leaf, err := parseLeafCertificate(secret.Data["tls.crt"], secret.Data["tls.key"]) + if err != nil || leaf.VerifyHostname(hostname) != nil || now.Before(leaf.NotBefore) || !leaf.NotAfter.After(now.Add(listenerCertExpiryMargin)) { + return listenerCertStatus{}, true, true + } + lifetime := leaf.NotAfter.Sub(leaf.NotBefore) + renewalDue = !now.Before(leaf.NotAfter.Add(-lifetime / 3)) + return listenerCertStatus{healthy: true, secretName: secretName, notAfter: &metav1.Time{Time: leaf.NotAfter}}, true, renewalDue +} + // evaluateListenerTLSCertificateHealth is evaluateListenerCertHealth for the // certificate-service path: conditions come from the upstream TLSCertificate, // the served material from the mirrored downstream Secret. @@ -540,6 +810,31 @@ func (r *GatewayReconciler) evaluateListenerTLSCertificateHealth( return health } +// servingSecretHealth is listenerSecretHealth plus a check that the leaf +// actually covers the hostname, so a listener whose hostname changed cannot +// keep serving the previous name's certificate. +func servingSecretHealth( + ctx context.Context, + downstreamClient client.Client, + downstreamNamespace string, + secretName string, + hostname string, + now time.Time, +) listenerCertStatus { + status := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + if !status.healthy { + return status + } + var secret corev1.Secret + if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { + return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), pending: true, secretName: secretName} + } + if err := validateIssuedMaterial(secret.Data["tls.crt"], secret.Data["tls.key"], hostname, now); err != nil { + return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), secretName: secretName} + } + return status +} + // listenerTLSCertificateHealth keeps a listener serving whenever its downstream // Secret holds a usable certificate, whatever the TLSCertificate is doing: a // renewal in flight or a first issuance replacing a legacy certificate must @@ -560,7 +855,7 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( certName := tlsCertificateName(upstreamGateway.Name, listenerName) secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) - secretStatus := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + secretStatus := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) var cert certificatesv1alpha1.TLSCertificate if err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamGateway.Namespace, Name: certName}, &cert); err != nil { @@ -579,9 +874,6 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( } if secretStatus.healthy { - if apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) && cert.Status.NotAfter != nil { - secretStatus.notAfter = cert.Status.NotAfter - } return secretStatus } diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go index bd19c058..ac1e0e0d 100644 --- a/internal/controller/gateway_certificate_service_test.go +++ b/internal/controller/gateway_certificate_service_test.go @@ -4,6 +4,7 @@ package controller import ( "context" + "strings" "testing" "time" @@ -18,6 +19,7 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -79,7 +81,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, }, ClusterIssuerMap: map[string]string{autoIssuerSentinel: "datum-gateway-http"}, - CertificateService: config.CertificateServiceConfig{Enabled: true}, + CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: serviceNS}, } testCfg := config.NetworkServicesOperator{Gateway: baseConfig} @@ -228,16 +230,17 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { c.Status.Challenges = []certificatesv1alpha1.ACMEChallenge{ - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-a", Key: "key-a", State: certificatesv1alpha1.ChallengeStatePending}, - {DNSName: "evil.example.com", Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-b", Key: "key-b", State: certificatesv1alpha1.ChallengeStatePending}, - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeDNS01, Token: "tok-c", Key: "key-c", State: certificatesv1alpha1.ChallengeStatePending}, - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-d", Key: "key-d", State: certificatesv1alpha1.ChallengeStateInvalid}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-aaaaaaaaaaaaaa", Key: "key-a", State: certificatesv1alpha1.ChallengeStatePending}, + {DNSName: "evil.example.com", Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-bbbbbbbbbbbbbb", Key: "key-b", State: certificatesv1alpha1.ChallengeStatePending}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeDNS01, Token: "token-cccccccccccccc", Key: "key-c", State: certificatesv1alpha1.ChallengeStatePending}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-dddddddddddddd", Key: "key-d", State: certificatesv1alpha1.ChallengeStateInvalid}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "bad token/../x", Key: "key-e", State: certificatesv1alpha1.ChallengeStatePending}, } })} }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { ctx := context.Background() - served := tlsCertificateSolverName(certName, "tok-a") + served := tlsCertificateSolverName(certName, "token-aaaaaaaaaaaaaa") var route gatewayv1.HTTPRoute require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: served}, &route)) @@ -245,8 +248,9 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { assert.Equal(t, certName, route.Labels[tlsCertificateSolverLabel]) require.Len(t, route.Spec.Rules, 1) require.Len(t, route.Spec.Rules[0].Matches, 1) - assert.Equal(t, "/.well-known/acme-challenge/tok-a", ptr.Deref(route.Spec.Rules[0].Matches[0].Path.Value, "")) + assert.Equal(t, "/.well-known/acme-challenge/token-aaaaaaaaaaaaaa", ptr.Deref(route.Spec.Rules[0].Matches[0].Path.Value, "")) assert.Equal(t, gatewayv1.ObjectName(gatewayName), route.Spec.ParentRefs[0].Name) + assert.Equal(t, []gatewayv1.Hostname{hostname}, route.Spec.Hostnames, "solver route is scoped to the claimed hostname") owner := metav1.GetControllerOf(&route) require.NotNil(t, owner) assert.Equal(t, KindGateway, owner.Kind) @@ -255,10 +259,11 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: served}, &filter)) assert.Equal(t, "key-a", ptr.Deref(filter.Spec.DirectResponse.Body.Inline, "")) - for _, token := range []string{"tok-b", "tok-c", "tok-d"} { + for _, token := range []string{"token-bbbbbbbbbbbbbb", "token-cccccccccccccc", "token-dddddddddddddd", "bad token/../x"} { err := e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, token)}, &gatewayv1.HTTPRoute{}) assert.True(t, apierrors.IsNotFound(err), "no solver route for challenge %s", token) } + assert.NotEqual(t, certificateServiceBackoffBase, e.result.RequeueAfter, "a rejected token is not a failure of the step") }, }, { @@ -266,7 +271,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { c.Status.Challenges = []certificatesv1alpha1.ACMEChallenge{ - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "tok-keep", Key: "key-keep", State: certificatesv1alpha1.ChallengeStateValid}, + {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-keepkeepkeepkeep", Key: "key-keep", State: certificatesv1alpha1.ChallengeStateValid}, } })} }, @@ -274,15 +279,15 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { gw := downstreamGatewaySeed() objs := make([]client.Object, 0, 5) objs = append(objs, gw) - objs = append(objs, solverObjects(gw, "tok-keep")...) - objs = append(objs, solverObjects(gw, "tok-old")...) + objs = append(objs, solverObjects(gw, "token-keepkeepkeepkeep")...) + objs = append(objs, solverObjects(gw, "token-oldoldoldoldold")...) return objs }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { ctx := context.Background() - assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, "tok-keep")}, &gatewayv1.HTTPRoute{}), "a Valid challenge keeps its answer until the service drops it") + assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, "token-keepkeepkeepkeep")}, &gatewayv1.HTTPRoute{}), "a Valid challenge keeps its answer until the service drops it") - old := tlsCertificateSolverName(certName, "tok-old") + old := tlsCertificateSolverName(certName, "token-oldoldoldoldold") assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: old}, &gatewayv1.HTTPRoute{}))) assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: old}, &envoygatewayv1alpha1.HTTPRouteFilter{}))) }, @@ -365,9 +370,9 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, }, { - name: "switches to the service when the cert-manager certificate is due for renewal", + name: "switches to the service ahead of cert-manager's renewal time", downstreamObjects: func() []client.Object { - legacy := newLegacyCertificate(now.Add(-time.Hour)) + legacy := newLegacyCertificate(now.Add(24 * time.Hour)) secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) require.NoError(t, controllerutil.SetControllerReference(legacy, secret, testScheme)) return []client.Object{legacy, secret} @@ -380,6 +385,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"], "the serving Secret is kept as-is until the service issues") assert.Equal(t, upstreamNamespace.Name, secret.Labels[downstreamclient.UpstreamOwnerNamespaceLabel]) + assert.Equal(t, labelValueTrue, secret.Labels[tlsCertificateManagedLabel], "the hand-over is recorded for rollback") var anchored bool for _, ref := range secret.OwnerReferences { if ref.Kind == "ConfigMap" { @@ -392,6 +398,100 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener keeps serving on the retained Secret") }, }, + { + name: "does not retire a cert-manager certificate while cert-manager is renewing it", + downstreamObjects: func() []client.Object { + legacy := newLegacyCertificate(now.Add(-time.Hour)) + legacy.Status.Conditions = append(legacy.Status.Conditions, cmv1.CertificateCondition{Type: cmv1.CertificateConditionIssuing, Status: cmmeta.ConditionTrue}) + return []client.Object{legacy, tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM)} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{}), "legacy Certificate stays while its renewal is in flight") + var certs certificatesv1alpha1.TLSCertificateList + require.NoError(t, e.upstream.List(ctx, &certs, client.InNamespace(upstreamNamespace.Name))) + assert.Empty(t, certs.Items) + + strategy := downstreamclient.NewMappedNamespaceResourceStrategy(upstreamCluster, e.upstream, e.downstream) + step, issues := e.reconciler.ensureListenerTLSCertificates(ctx, e.upstream, upstreamGateway, downstreamGateway, e.downstream, strategy, []string{hostname}) + assert.Empty(t, issues) + assert.Equal(t, 10*time.Minute, step.RequeueAfter, "the hold is re-checked while cert-manager renews") + }, + }, + { + name: "never deletes a mismatched TLSCertificate another owner controls", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + other := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: "other-gw", UID: uuid.NewUUID()}} + return []client.Object{newTLSCertificate(other, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"previous.example.com"} + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) + assert.Equal(t, []certificatesv1alpha1.DNSName{"previous.example.com"}, cert.Spec.DNSNames, "someone else's TLSCertificate is left alone") + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter, "the clash is retried with backoff, not an error") + assertListenerMessageContains(t, upstreamGateway, listenerName, "keep trying") + }, + }, + { + name: "refuses a service-side Secret outside the service namespace", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.ServiceSecretRef.Namespace = "victim-namespace" + })} + }, + serviceObjects: []client.Object{tlsSecret("victim-namespace", serviceSecret, serviceCertPEM, serviceKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) + assertListenerMessageContains(t, upstreamGateway, listenerName, "keep trying") + }, + }, + { + name: "refuses service-side material whose key does not match", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, projectKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) + }, + }, + { + name: "refuses service-side material for another hostname", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: func() []client.Object { + crt, key := generateTLSKeyPair(t, "other.example.com", now.Add(-time.Hour), now.Add(60*24*time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) + }, + }, + { + name: "refuses expired service-side material and keeps the serving Secret", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + downstreamObjects: func() []client.Object { + secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) + secret.Labels = map[string]string{tlsCertificateManagedLabel: labelValueTrue} + return []client.Object{secret} + }, + serviceObjects: func() []client.Object { + crt, key := generateTLSKeyPair(t, hostname, now.Add(-48*time.Hour), now.Add(-time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var secret corev1.Secret + require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"], "a valid serving Secret is never replaced by material that fails validation") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + }, + }, { name: "deletes the TLSCertificate of a listener that is gone and leaves others alone", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { @@ -497,8 +597,241 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { } } +func assertListenerMessageContains(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, want string) { + t.Helper() + for _, ls := range gateway.Status.Listeners { + if ls.Name != listener { + continue + } + resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) + require.NotNil(t, resolved) + assert.Equal(t, metav1.ConditionFalse, resolved.Status) + assert.Contains(t, resolved.Message, want) + return + } + t.Fatalf("listener %s not found in status", listener) +} + func TestTLSCertificateName(t *testing.T) { long := tlsCertificateName("a-gateway-with-a-deliberately-very-long-name-for-this-test", "https-hostname-with-a-long-name-0") assert.LessOrEqual(t, len(long), 63) - assert.Equal(t, "gw-https-0", tlsCertificateName("gw", "https-0")) + assert.NotEqual(t, tlsCertificateName("a-b", "c"), tlsCertificateName("a", "b-c"), "gateway and listener never blur into each other") + dotted := tlsCertificateName("my.dotted.gateway.name.that.is.long.enough.to.be.cut", "https-0") + assert.Regexp(t, `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, dotted) + assert.LessOrEqual(t, len(dotted), 63) + assert.Equal(t, tlsCertificateName("gw", "https-0"), tlsCertificateName("gw", "https-0")) + assert.True(t, strings.HasPrefix(tlsCertificateName("gw", "https-0"), "gw-https-0-")) +} + +func TestCertificateServiceRequeueBackoff(t *testing.T) { + r := &GatewayReconciler{} + uid := types.UID("gw") + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true)) + assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(uid, true)) + assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(uid, true)) + for range 10 { + r.certificateServiceRequeue(uid, true) + } + assert.Equal(t, certificateServiceBackoffMax, r.certificateServiceRequeue(uid, true)) + assert.Zero(t, r.certificateServiceRequeue(uid, false)) + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true), "a success resets the backoff") +} + +func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + withoutCertificates := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(withoutCertificates)) + require.NoError(t, gatewayv1.Install(withoutCertificates)) + require.NoError(t, discoveryv1.AddToScheme(withoutCertificates)) + require.NoError(t, networkingv1alpha.AddToScheme(withoutCertificates)) + downstreamScheme := newCertificateServiceTestScheme(t) + + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}} + const hostname = "custom.example.com" + testCfg := config.NetworkServicesOperator{Gateway: config.GatewayConfig{ + DownstreamGatewayClassName: "test-suite", + DownstreamHostnameAccountingNamespace: "default", + TargetDomain: "test-suite.com", + DefaultListenerTLSSecretName: "wildcard-test-suite-tls", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, + ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer"}, + CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system"}, + }} + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, "test-gw", func(g *gatewayv1.Gateway) { + g.Spec.Listeners = append(g.Spec.Listeners, gatewayv1.Listener{ + Name: "https-hostname-0", Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + TLS: &gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModeTerminate), Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer", + }}, + }) + }) + domain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { + d.Spec.DomainName = hostname + apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue}) + }) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + route := newHTTPRoute(upstreamNamespace.Name, "app", func(r *gatewayv1.HTTPRoute) { + r.Spec.ParentRefs = []gatewayv1.ParentReference{{Name: gatewayv1.ObjectName(upstreamGateway.Name)}} + r.Spec.Hostnames = []gatewayv1.Hostname{hostname} + }) + for _, obj := range []client.Object{domain, gatewayClass, route} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + fakeUpstreamClient := fake.NewClientBuilder().WithScheme(withoutCertificates). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, route). + WithStatusSubresource(upstreamGateway, route).Build() + fakeDownstreamClient := fake.NewClientBuilder().WithScheme(downstreamScheme).WithStatusSubresource(&gatewayv1.Gateway{}).Build() + + reconciler := &GatewayReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + Config: testCfg, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + CertificateServiceReader: fake.NewClientBuilder().WithScheme(downstreamScheme).Build(), + } + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", fakeUpstreamClient, fakeDownstreamClient) + + reconciler.prepareUpstreamGateway(upstreamGateway) + result, downstreamGateway := reconciler.ensureDownstreamGateway(ctx, "test", fakeUpstreamClient, upstreamGateway, downstreamStrategy) + require.NoError(t, result.Err, "a missing TLSCertificate API must not fail the gateway") + _, err := result.Complete(ctx) + require.NoError(t, err) + assert.Equal(t, certificateServiceBackoffBase, result.RequeueAfter) + + require.NotNil(t, downstreamGateway) + assert.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKeyFromObject(downstreamGateway), &gatewayv1.Gateway{}), "the downstream gateway is still programmed") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, gatewayutil.DefaultHTTPListenerName)) + + var updated gatewayv1.Gateway + require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), &updated)) + assert.NotEmpty(t, updated.Status.Listeners, "gateway status is still written") + assertListenerMessageContains(t, &updated, "https-hostname-0", "keep trying") + + var downstreamRoutes gatewayv1.HTTPRouteList + require.NoError(t, fakeDownstreamClient.List(ctx, &downstreamRoutes, client.InNamespace(downstreamGateway.Namespace))) + assert.NotEmpty(t, downstreamRoutes.Items, "downstream routes are still reconciled") +} + +func TestCertificateServiceRollbackKeepsHandedOverSecret(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + const hostname = "custom.example.com" + const gatewayName = "test-gw" + const listenerName = gatewayv1.SectionName("https-hostname-0") + secretName := listenerCertificateSecretName(gatewayName, listenerName) + legacyName := listenerCertificateName(gatewayName, listenerName) + certName := tlsCertificateName(gatewayName, listenerName) + + testCfg := config.NetworkServicesOperator{Gateway: config.GatewayConfig{ + DownstreamGatewayClassName: "test-suite", + DownstreamHostnameAccountingNamespace: "default", + TargetDomain: "test-suite.com", + DefaultListenerTLSSecretName: "wildcard-test-suite-tls", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, + ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer"}, + }} + + now := time.Now() + tests := []struct { + name string + notBefore time.Time + notAfter time.Time + wantCertificate bool + wantListenerAdmit bool + }{ + {name: "fresh handed-over Secret keeps serving and is not reissued", notBefore: now.Add(-24 * time.Hour), notAfter: now.Add(80 * 24 * time.Hour), wantCertificate: false, wantListenerAdmit: true}, + {name: "handed-over Secret in its renewal window gets a cert-manager Certificate again", notBefore: now.Add(-70 * 24 * time.Hour), notAfter: now.Add(20 * 24 * time.Hour), wantCertificate: true, wantListenerAdmit: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.Spec.Listeners = append(g.Spec.Listeners, gatewayv1.Listener{ + Name: listenerName, Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + TLS: &gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModeTerminate), Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer", + }}, + }) + }) + domain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { + d.Spec.DomainName = hostname + apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue}) + }) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + leftover := &certificatesv1alpha1.TLSCertificate{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: certName}} + require.NoError(t, controllerutil.SetControllerReference(upstreamGateway, leftover, testScheme)) + foreign := &certificatesv1alpha1.TLSCertificate{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: "foreign"}} + + downstreamGateway := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: gatewayName, UID: uuid.NewUUID(), Labels: map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-test", + downstreamclient.UpstreamOwnerKindLabel: KindGateway, + downstreamclient.UpstreamOwnerNameLabel: gatewayName, + downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace.Name, + }}, Spec: gatewayv1.GatewaySpec{GatewayClassName: "test-suite"}} + crt, key := generateTLSKeyPair(t, hostname, tt.notBefore, tt.notAfter) + handedOver := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: secretName, Labels: map[string]string{ + tlsCertificateManagedLabel: labelValueTrue, + downstreamclient.UpstreamOwnerKindLabel: KindGateway, + downstreamclient.UpstreamOwnerNameLabel: gatewayName, + downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace.Name, + }}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": crt, "tls.key": key}, + } + solverName := tlsCertificateSolverName(certName, "token-leftoverleftover") + solverRoute := &gatewayv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: solverName, Labels: map[string]string{tlsCertificateSolverLabel: certName}}} + solverFilter := &envoygatewayv1alpha1.HTTPRouteFilter{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: solverName, Labels: map[string]string{tlsCertificateSolverLabel: certName}}} + require.NoError(t, controllerutil.SetControllerReference(downstreamGateway, solverRoute, testScheme)) + require.NoError(t, controllerutil.SetControllerReference(downstreamGateway, solverFilter, testScheme)) + + for _, obj := range []client.Object{domain, gatewayClass, leftover, foreign, handedOver, solverRoute, solverFilter} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + downstreamGateway.SetCreationTimestamp(metav1.Now()) + + fakeUpstreamClient := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, leftover, foreign). + WithStatusSubresource(upstreamGateway).Build() + fakeDownstreamClient := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(downstreamGateway, handedOver, solverRoute, solverFilter). + WithStatusSubresource(&gatewayv1.Gateway{}).Build() + + reconciler := &GatewayReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + Config: testCfg, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + } + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", fakeUpstreamClient, fakeDownstreamClient) + + reconciler.prepareUpstreamGateway(upstreamGateway) + result, downstream := reconciler.ensureDownstreamGateway(ctx, "test", fakeUpstreamClient, upstreamGateway, downstreamStrategy) + require.NoError(t, result.Err) + _, err := result.Complete(ctx) + require.NoError(t, err) + + certErr := fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{}) + if tt.wantCertificate { + assert.NoError(t, certErr, "cert-manager takes the hostname back once the Secret is due for renewal") + } else { + assert.True(t, apierrors.IsNotFound(certErr), "a fresh handed-over Secret is not reissued on rollback") + } + assert.Equal(t, tt.wantListenerAdmit, gatewayutil.GetListenerByName(downstream.Spec.Listeners, listenerName) != nil) + + assert.True(t, apierrors.IsNotFound(fakeUpstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{})), "the gateway's TLSCertificate is removed on rollback") + assert.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: "foreign"}, &certificatesv1alpha1.TLSCertificate{}), "a TLSCertificate the gateway does not control stays") + assert.True(t, apierrors.IsNotFound(fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: solverName}, &gatewayv1.HTTPRoute{})), "leftover solver routes are removed") + assert.True(t, apierrors.IsNotFound(fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: solverName}, &envoygatewayv1alpha1.HTTPRouteFilter{}))) + }) + } } diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 976360c1..e99d053c 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -10,6 +10,7 @@ import ( "slices" "strconv" "strings" + "sync" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -99,6 +100,8 @@ type GatewayReconciler struct { // the service-side copy of each issued Secret lives. Required when // Config.Gateway.CertificateService.Enabled. CertificateServiceReader client.Reader + + certificateServiceFailures sync.Map } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -380,9 +383,12 @@ func (r *GatewayReconciler) ensureDownstreamGateway( } } - var certResult Result + var certificateServiceRequeue time.Duration if r.Config.Gateway.CertificateService.Enabled { - certResult = r.ensureListenerTLSCertificates( + // The service is a dependency of issuance, not of routing: whatever it + // does, DNS, status and routes below still reconcile, and a listener + // that could not be served tells the customer why. + certResult, issues := r.ensureListenerTLSCertificates( ctx, upstreamClient, upstreamGateway, @@ -391,8 +397,15 @@ func (r *GatewayReconciler) ensureDownstreamGateway( downstreamStrategy, claimedHostnames, ) + for name, message := range issues { + if status, gated := listenerCertHealth[name]; gated && !status.healthy { + status.message = message + listenerCertHealth[name] = status + } + } + certificateServiceRequeue = certResult.RequeueAfter } else { - certResult = r.ensureListenerCertificates( + certResult := r.ensureListenerCertificates( ctx, upstreamGateway, downstreamGateway, @@ -400,9 +413,10 @@ func (r *GatewayReconciler) ensureDownstreamGateway( downstreamStrategy, claimedHostnames, ) - } - if certResult.ShouldReturn() { - return certResult.Merge(result), nil + if certResult.ShouldReturn() { + return certResult.Merge(result), nil + } + r.cleanupCertificateServiceLeftovers(ctx, upstreamClient, upstreamGateway, downstreamGateway, downstreamClient) } dnsResult := r.ensureDownstreamGatewayDNSEndpoints( @@ -468,6 +482,9 @@ func (r *GatewayReconciler) ensureDownstreamGateway( break } } + if certificateServiceRequeue > 0 && (result.RequeueAfter == 0 || certificateServiceRequeue < result.RequeueAfter) { + result.RequeueAfter = certificateServiceRequeue + } addresses := make([]gatewayv1.GatewayStatusAddress, 0, len(targetDomainHostnames)) @@ -636,6 +653,9 @@ func (r *GatewayReconciler) listenerCertHealth( var cert cmv1.Certificate if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: certName}, &cert); err != nil { if apierrors.IsNotFound(err) { + if status, handedOver, _ := handedOverSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now); handedOver && status.healthy { + return status + } return listenerCertStatus{ reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certIssuanceFailingMessage(hostname), @@ -1045,6 +1065,10 @@ func (r *GatewayReconciler) ensureListenerCertificates( isNew := cert.CreationTimestamp.IsZero() if isNew { + if status, handedOver, renewalDue := handedOverSecretHealth(ctx, downstreamClient, downstreamGateway.Namespace, secretName, hostname, time.Now()); handedOver && status.healthy && !renewalDue { + logger.Info("keeping certificate-service issued Secret until it is due for renewal", "secret", secretName, "hostname", hostname) + continue + } if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, cert); err != nil { result.Err = fmt.Errorf("failed to set strategy reference on Certificate %s: %w", certName, err) return result diff --git a/internal/controller/http01_solver.go b/internal/controller/http01_solver.go index bfd14862..8f40b04d 100644 --- a/internal/controller/http01_solver.go +++ b/internal/controller/http01_solver.go @@ -23,10 +23,11 @@ const http01SolverLabel = "meta.datumapis.com/http01-solver" // http01SolverRoute names one ACME HTTP-01 answer to publish on a downstream // gateway: GET /.well-known/acme-challenge/ returns . type http01SolverRoute struct { - name string - token string - key string - labels map[string]string + name string + token string + key string + hostnames []gatewayv1.Hostname + labels map[string]string } func (s http01SolverRoute) objectLabels() map[string]string { @@ -94,6 +95,7 @@ func ensureHTTP01SolverRoutes( } httpRoute.Labels = solver.objectLabels() httpRoute.Spec = gatewayv1.HTTPRouteSpec{ + Hostnames: solver.hostnames, CommonRouteSpec: gatewayv1.CommonRouteSpec{ ParentRefs: []gatewayv1.ParentReference{ { diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go index 01468643..ebb32ed2 100644 --- a/internal/controller/httpproxy_certificate_service.go +++ b/internal/controller/httpproxy_certificate_service.go @@ -5,6 +5,8 @@ package controller import ( "context" "fmt" + "strings" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -36,6 +38,8 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( upstreamNamespace string, certName string, legacyName string, + secretName string, + hostname string, generation int64, ) metav1.Condition { condition := metav1.Condition{ @@ -48,6 +52,13 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( switch { case err == nil: condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) + if condition.Status != metav1.ConditionTrue { + if serving := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()); serving.healthy { + condition.Status = metav1.ConditionTrue + condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued + condition.Message = "Certificate is ready; a renewal is in progress" + } + } return condition case !apierrors.IsNotFound(err): condition.Status = metav1.ConditionUnknown @@ -103,10 +114,23 @@ func tlsCertificateReadyState(cert *certificatesv1alpha1.TLSCertificate) (metav1 if issuing.Message != "" { message = fmt.Sprintf("%s (%s)", certificateProvisioningMessage, issuing.Message) } - return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonChallengeInProgress, message + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonChallengeInProgress, message + requiredDNSRecordsHint(cert) } - return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonPending, certificateProvisioningMessage + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonPending, certificateProvisioningMessage + requiredDNSRecordsHint(cert) +} + +// requiredDNSRecordsHint tells the customer which records issuance is waiting +// on, when the service has named any. +func requiredDNSRecordsHint(cert *certificatesv1alpha1.TLSCertificate) string { + records := make([]string, 0, len(cert.Status.RequiredDNSRecords)) + for _, record := range cert.Status.RequiredDNSRecords { + records = append(records, fmt.Sprintf("%s %s %s", record.Name, record.Type, record.Content)) + } + if len(records) == 0 { + return "" + } + return ". Publish these DNS records to continue: " + strings.Join(records, "; ") } // enqueueHTTPProxyForTLSCertificate follows TLSCertificate -> Gateway -> diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go index 5b803292..df07c2df 100644 --- a/internal/controller/httpproxy_certificate_service_test.go +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -5,6 +5,7 @@ package controller import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,9 +66,17 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { return cert } + crt, key := generateTLSKeyPair(t, "app.example.com", time.Now().Add(-time.Hour), time.Now().Add(30*24*time.Hour)) + servingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName("my-proxy", "https-hostname-0")}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": crt, "tls.key": key}, + } + tests := []struct { name string upstream []client.Object + mutate func(*certificatesv1alpha1.TLSCertificate) downstream []client.Object wantStatus metav1.ConditionStatus wantReason string @@ -103,6 +112,26 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { wantStatus: metav1.ConditionFalse, wantReason: networkingv1alpha.CertificateReadyReasonPending, }, + { + name: "Issuing with required DNS records names them for the customer", + upstream: []client.Object{tlsCert( + metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue, Reason: "OrderInFlight"}, + ), nil}[:1], + mutate: func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.RequiredDNSRecords = []certificatesv1alpha1.RequiredDNSRecord{{Name: "_acme-challenge.app.example.com", Type: "CNAME", Content: "abc.acme-dns.example.net", Purpose: certificatesv1alpha1.DNSRecordPurposeCertificate}} + }, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonChallengeInProgress, + wantMessage: certificateProvisioningMessage + ". Publish these DNS records to continue: _acme-challenge.app.example.com CNAME abc.acme-dns.example.net", + }, + { + name: "not Ready yet but a serving downstream Secret keeps the hostname ready", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue})}, + downstream: []client.Object{servingSecret}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, + wantMessage: "Certificate is ready; a renewal is in progress", + }, { name: "no TLSCertificate yet but a ready legacy Certificate answers", downstream: []client.Object{legacyReady()}, @@ -121,6 +150,9 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() + if tt.mutate != nil { + tt.mutate(tt.upstream[0].(*certificatesv1alpha1.TLSCertificate)) + } upstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(ns).WithObjects(tt.upstream...).Build() downstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(tt.downstream...).Build() diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index bb69b037..117be93d 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1517,7 +1517,7 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( certName := resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-%s", gateway.Name, l.Name)) if r.Config.Gateway.CertificateService.Enabled { - apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), certName, httpProxy.Generation)) + apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), certName, listenerCertificateSecretName(gateway.Name, l.Name), hostname, httpProxy.Generation)) statuses = append(statuses, hs) continue } From 7ed86a4e0a0424c4a5e43c9687bc1513a5b78050 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 1 Oct 2026 21:53:42 -0500 Subject: [PATCH 3/8] fix: surface certificate renewals the service cannot do A hostname can serve for weeks on its previous certificate while every attempt to replace it fails; until now that only reached the log. The customer and operators now hear about it before the expiry does, and the hand-over to the service no longer waits forever on a stuck cert-manager renewal. Key changes: - Report a blocked renewal on the listener as CertificateRenewalBlocked and on the HTTPProxy hostname as CertificateReady=True with reason RenewalFailing, whether the failure is a rejected request, a refused service-side Secret, or a failed call; count each in nso_certificate_service_failures_total by reason - Hand over anyway when cert-manager's renewal has failed, has been in flight for over a day, or under a week of lifetime remains - Repeat the last message instead of the failing calls while a gateway is in backoff, and forget the gateway when it is deleted - Bound each read of the service cluster to ten seconds - Drop the hand-over marker once cert-manager has the Secret back - Pin the TLSCertificate naming scheme with golden values Related to datum-cloud/enhancements#913 --- api/v1alpha/httpproxy_types.go | 4 + internal/agent/catalog.go | 12 + internal/config/config.go | 13 ++ .../controller/gateway_certificate_service.go | 166 +++++++++++--- .../gateway_certificate_service_test.go | 211 +++++++++++++++++- internal/controller/gateway_controller.go | 28 ++- .../httpproxy_certificate_service.go | 13 ++ .../httpproxy_certificate_service_test.go | 41 +++- internal/controller/httpproxy_controller.go | 13 +- internal/controller/metrics.go | 13 ++ 10 files changed, 464 insertions(+), 50 deletions(-) diff --git a/api/v1alpha/httpproxy_types.go b/api/v1alpha/httpproxy_types.go index 489feee5..88ab9434 100644 --- a/api/v1alpha/httpproxy_types.go +++ b/api/v1alpha/httpproxy_types.go @@ -551,6 +551,10 @@ const ( // CertificateReadyReasonChallengeInProgress indicates an ACME challenge is in progress. CertificateReadyReasonChallengeInProgress = "ChallengeInProgress" + + // CertificateReadyReasonRenewalFailing indicates the hostname still serves a + // valid certificate but its replacement cannot be issued; the message says why. + CertificateReadyReasonRenewalFailing = "RenewalFailing" ) // Reasons for HostnameConditionAvailable. diff --git a/internal/agent/catalog.go b/internal/agent/catalog.go index 4541496b..ce5cd838 100644 --- a/internal/agent/catalog.go +++ b/internal/agent/catalog.go @@ -540,6 +540,18 @@ var dnsAndCertCatalog = []ReasonInfo{ "status message.", Skill: SkillCertificateNotIssued, }, + { + Reason: networkingv1alpha.CertificateReadyReasonRenewalFailing, + ConditionType: networkingv1alpha.HostnameConditionCertificateReady, + Actionability: ActionabilityUser, + Scope: ScopeOneHostname, + Explanation: "HTTPS still works on this hostname's current certificate, but the replacement " + + "cannot be issued. When the current certificate expires HTTPS will fail.", + Remediation: "The status message says what blocked the renewal. If it names the hostname " + + "itself, check that it still resolves publicly to this load balancer; otherwise " + + "it is a platform issue and support can see the same message.", + Skill: SkillCertificateNotIssued, + }, } // domainCatalog covers the Domain behind a custom hostname. This is where diff --git a/internal/config/config.go b/internal/config/config.go index 8c154c32..d108d1d4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -946,6 +946,19 @@ func SetDefaults_CertificateServiceConfig(obj *CertificateServiceConfig) { // RestConfig returns the connection to the certificate service's cluster. func (c *CertificateServiceConfig) RestConfig() (*rest.Config, error) { + cfg, err := c.restConfig() + if err != nil { + return nil, err + } + cfg.Timeout = certificateServiceRequestTimeout + return cfg, nil +} + +// certificateServiceRequestTimeout bounds each read of the service cluster, so +// a hung connection there cannot stall a gateway reconcile. +const certificateServiceRequestTimeout = 10 * time.Second + +func (c *CertificateServiceConfig) restConfig() (*rest.Config, error) { if c.KubeconfigPath != "" { return clientcmd.BuildConfigFromFlags("", c.KubeconfigPath) } diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go index 67d5b237..327abbea 100644 --- a/internal/controller/gateway_certificate_service.go +++ b/internal/controller/gateway_certificate_service.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/hex" + "errors" "fmt" "regexp" "slices" @@ -120,26 +121,69 @@ func (r *GatewayReconciler) listenerIssuerResolvable(upstreamGateway *gatewayv1. return issuer != autoIssuerSentinel || r.resolveAutoIssuer(upstreamGateway) != "" } -// certificateServiceRequeue hands back the delay before the next attempt for a -// gateway whose certificate-service step failed, doubling per consecutive -// failure, and forgets the gateway once a pass succeeds. -func (r *GatewayReconciler) certificateServiceRequeue(gateway types.UID, failed bool) time.Duration { +// certificateServiceBackoff is what the step remembers about a gateway whose +// last pass failed: how many in a row, when it may try again, and what it told +// the listeners, so an event-driven reconcile inside the window repeats the +// message rather than the calls. +type certificateServiceBackoff struct { + attempts int + nextTry time.Time + message string + listeners []gatewayv1.SectionName +} + +// certificateServiceRequeue records the outcome of a pass and hands back the +// delay before the next attempt, doubling per consecutive failure up to a cap, +// and forgets the gateway once a pass succeeds. +func (r *GatewayReconciler) certificateServiceRequeue(gateway types.UID, failed bool, now time.Time, issues map[gatewayv1.SectionName]string) time.Duration { if !failed { r.certificateServiceFailures.Delete(gateway) return 0 } attempts := 1 if previous, ok := r.certificateServiceFailures.Load(gateway); ok { - attempts = previous.(int) + 1 + attempts = previous.(certificateServiceBackoff).attempts + 1 } - r.certificateServiceFailures.Store(gateway, attempts) delay := certificateServiceBackoffBase << (attempts - 1) if attempts > 10 || delay > certificateServiceBackoffMax { delay = certificateServiceBackoffMax } + backoff := certificateServiceBackoff{attempts: attempts, nextTry: now.Add(delay)} + for listener, message := range issues { + backoff.listeners = append(backoff.listeners, listener) + backoff.message = message + } + r.certificateServiceFailures.Store(gateway, backoff) return delay } +// certificateServiceInBackoff reports whether the gateway's last failure is +// still cooling off, and if so the listeners it concerned and how long is left. +func (r *GatewayReconciler) certificateServiceInBackoff(gateway types.UID, now time.Time) (certificateServiceBackoff, time.Duration, bool) { + previous, ok := r.certificateServiceFailures.Load(gateway) + if !ok { + return certificateServiceBackoff{}, 0, false + } + backoff := previous.(certificateServiceBackoff) + if !now.Before(backoff.nextTry) { + return backoff, 0, false + } + return backoff, backoff.nextTry.Sub(now), true +} + +func recordCertificateServiceFailure(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, reason string) { + certificateServiceFailuresTotal.WithLabelValues(gateway.Namespace, gateway.Name, string(listener), reason).Inc() +} + +const ( + certificateServiceReasonStepFailed = "StepFailed" + certificateServiceReasonNotOwned = "NotOwned" + certificateServiceReasonRejected = "Rejected" + certificateServiceReasonNamespaceRefused = "NamespaceRefused" + certificateServiceReasonMaterialRefused = "MaterialRefused" + certificateServiceReasonTokenRefused = "TokenRefused" +) + func certificateServiceUnavailableMessage(hostname string) string { return fmt.Sprintf("We couldn't request a TLS certificate for %s just now and will keep trying. HTTPS for this hostname stays as it is in the meantime.", hostname) } @@ -180,9 +224,18 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( fail := func(l gatewayv1.Listener, hostname string, err error) { logger.Error(err, "certificate service step failed", "listener", l.Name, "hostname", hostname) issues[l.Name] = certificateServiceUnavailableMessage(hostname) + recordCertificateServiceFailure(upstreamGateway, l.Name, certificateServiceReasonStepFailed) failed = true } + if backoff, remaining, cooling := r.certificateServiceInBackoff(upstreamGateway.UID, now); cooling { + for _, listener := range backoff.listeners { + issues[listener] = backoff.message + } + requeueSooner(remaining) + return result, issues + } + for _, l := range upstreamGateway.Spec.Listeners { hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) if !wanted || !r.listenerIssuerResolvable(upstreamGateway, l) { @@ -215,6 +268,9 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) if err != nil { + if errors.Is(err, errTLSCertificateNotOwned) { + recordCertificateServiceFailure(upstreamGateway, l.Name, certificateServiceReasonNotOwned) + } fail(l, hostname, err) continue } @@ -232,15 +288,17 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( liveSolvers.Insert(solvers...) if rejected != "" { issues[l.Name] = rejected + recordCertificateServiceFailure(upstreamGateway, l.Name, certificateServiceReasonTokenRefused) } - mirrored, rejected, err := r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName, hostname, now) + mirrored, refusal, err := r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName, hostname, now) if err != nil { fail(l, hostname, err) continue } - if rejected != "" { - issues[l.Name] = rejected + if refusal.reason != "" { + issues[l.Name] = refusal.message + recordCertificateServiceFailure(upstreamGateway, l.Name, refusal.reason) } if mirrored { requeueSooner(tlsCertificateMirrorAdmitDelay) @@ -260,11 +318,22 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( failed = true } - requeueSooner(r.certificateServiceRequeue(upstreamGateway.UID, failed)) + requeueSooner(r.certificateServiceRequeue(upstreamGateway.UID, failed, now, issues)) return result, issues } +var errTLSCertificateNotOwned = errors.New("TLSCertificate exists but is not controlled by this Gateway") + +// mirrorRefusal says why issued material was not taken, for the customer and +// for the failure counter. +type mirrorRefusal struct { + reason string + message string +} + +func (m mirrorRefusal) String() string { return m.reason } + type tlsCertificateState int const ( @@ -314,7 +383,7 @@ func (r *GatewayReconciler) ensureTLSCertificate( } if !metav1.IsControlledBy(cert, upstreamGateway) { - return nil, tlsCertificateSettled, fmt.Errorf("TLSCertificate %s exists but is not controlled by Gateway %s", certName, upstreamGateway.Name) + return nil, tlsCertificateSettled, fmt.Errorf("%w: %s", errTLSCertificateNotOwned, certName) } if !cert.DeletionTimestamp.IsZero() { @@ -407,24 +476,24 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( secretName string, hostname string, now time.Time, -) (mirrored bool, rejected string, err error) { +) (mirrored bool, refusal mirrorRefusal, err error) { logger := log.FromContext(ctx) if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { - return false, "", nil + return false, mirrorRefusal{}, nil } ref := cert.Status.ServiceSecretRef if ref == nil || ref.Name == "" { logger.Info("TLSCertificate is Ready without a service-side Secret reference", "tlscertificate", cert.Name) - return false, "", nil + return false, mirrorRefusal{}, nil } if ref.Namespace != r.Config.Gateway.CertificateService.SecretNamespace { logger.Info("refusing service-side Secret outside the certificate service namespace", "tlscertificate", cert.Name, "namespace", ref.Namespace, "expected", r.Config.Gateway.CertificateService.SecretNamespace) - return false, certificateServiceUnavailableMessage(hostname), nil + return false, mirrorRefusal{reason: certificateServiceReasonNamespaceRefused, message: certificateServiceUnavailableMessage(hostname)}, nil } if r.CertificateServiceReader == nil { - return false, "", fmt.Errorf("certificate service enabled without a client for its cluster") + return false, mirrorRefusal{}, fmt.Errorf("certificate service enabled without a client for its cluster") } downstreamClient := downstreamStrategy.GetClient() @@ -435,10 +504,10 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( }, } if err := downstreamClient.Get(ctx, client.ObjectKeyFromObject(mirror), mirror); client.IgnoreNotFound(err) != nil { - return false, "", fmt.Errorf("failed to get Secret %s: %w", secretName, err) + return false, mirrorRefusal{}, fmt.Errorf("failed to get Secret %s: %w", secretName, err) } if mirrorHoldsIssuance(mirror, cert) { - return false, "", nil + return false, mirrorRefusal{}, nil } var source corev1.Secret @@ -446,15 +515,15 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( if err := r.CertificateServiceReader.Get(ctx, sourceKey, &source); err != nil { if apierrors.IsNotFound(err) { logger.Info("TLSCertificate is Ready but its service-side Secret is not readable yet", "tlscertificate", cert.Name, "secret", sourceKey) - return false, "", nil + return false, mirrorRefusal{}, nil } - return false, "", fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) + return false, mirrorRefusal{}, fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) } if err := validateIssuedMaterial(source.Data["tls.crt"], source.Data["tls.key"], hostname, now); err != nil { logger.Info("refusing service-side Secret that does not hold a usable certificate for the hostname", "tlscertificate", cert.Name, "secret", sourceKey, "reason", err.Error()) - return false, certificateServiceUnavailableMessage(hostname), nil + return false, mirrorRefusal{reason: certificateServiceReasonMaterialRefused, message: certificateServiceUnavailableMessage(hostname)}, nil } op, err := controllerutil.CreateOrUpdate(ctx, downstreamClient, mirror, func() error { @@ -475,13 +544,13 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( return nil }) if err != nil { - return false, "", fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) + return false, mirrorRefusal{}, fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) } if op != controllerutil.OperationResultNone { logger.Info("issued Secret mirrored downstream", "secret", secretName, "operation", op) } - return op != controllerutil.OperationResultNone, "", nil + return op != controllerutil.OperationResultNone, mirrorRefusal{}, nil } // mirrorHoldsIssuance reports whether the downstream Secret already carries the @@ -553,18 +622,38 @@ func (r *GatewayReconciler) ownedLegacyCertificate( return &legacy, nil } +// legacyHoldWhileIssuing bounds how long a cert-manager renewal in flight can +// defer the hand-over; past it the renewal is taken as stuck. +const legacyHoldWhileIssuing = 24 * time.Hour + +// legacyHandOverFloor is the remaining lifetime below which the hand-over goes +// ahead whatever cert-manager is doing, so the service has time to issue. +const legacyHandOverFloor = 7 * 24 * time.Hour + // legacyCertificateHolds reports whether a cert-manager Certificate should keep // its hostname for now: it is serving and not yet within the switch lead of its -// renewal time, or cert-manager is mid-renewal and must not be interrupted. The -// returned duration is when to look again. +// renewal time, or cert-manager is mid-renewal and must not be interrupted. A +// renewal that has failed, has been in flight for too long, or has too little +// lifetime left to wait for does not hold. The returned duration is when to +// look again. func legacyCertificateHolds(cert *cmv1.Certificate, now time.Time) (bool, time.Duration) { if !certIsServing(cert, now) { return false, 0 } + if cert.Status.NotAfter != nil && cert.Status.NotAfter.Sub(now) < legacyHandOverFloor { + return false, 0 + } for _, c := range cert.Status.Conditions { - if c.Type == cmv1.CertificateConditionIssuing && c.Status == cmmeta.ConditionTrue { - return true, 10 * time.Minute + if c.Type != cmv1.CertificateConditionIssuing || c.Status != cmmeta.ConditionTrue { + continue + } + if cert.Status.LastFailureTime != nil { + return false, 0 } + if !c.LastTransitionTime.IsZero() && now.Sub(c.LastTransitionTime.Time) > legacyHoldWhileIssuing { + return false, 0 + } + return true, 10 * time.Minute } if cert.Status.RenewalTime == nil { return false, 0 @@ -749,6 +838,18 @@ func (r *GatewayReconciler) cleanupCertificateServiceLeftovers( if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, sets.New[string]()); err != nil { logger.Error(err, "failed to remove TLSCertificates after rollback") } + + for i := range handedOver.Items { + secret := &handedOver.Items[i] + var legacy cmv1.Certificate + if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: secret.Namespace, Name: secret.Name}, &legacy); err != nil || !certIsReady(&legacy) { + continue + } + delete(secret.Labels, tlsCertificateManagedLabel) + if err := downstreamClient.Update(ctx, secret); err != nil { + logger.Error(err, "failed to clear hand-over marker after cert-manager retook the Secret", "secret", secret.Name) + } + } } // handedOverSecretHealth reports whether the listener's downstream Secret was @@ -873,11 +974,20 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( } } + rejected := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted) + if rejected != nil && rejected.Status != metav1.ConditionFalse { + rejected = nil + } + if secretStatus.healthy { + if rejected != nil { + secretStatus.renewalBlocked = tlsCertificateRejectedMessage(hostname, rejected.Message) + recordCertificateServiceFailure(upstreamGateway, listenerName, certificateServiceReasonRejected) + } return secretStatus } - if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + if accepted := rejected; accepted != nil { return listenerCertStatus{ reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: tlsCertificateRejectedMessage(hostname, accepted.Message), diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go index ac1e0e0d..5e093fee 100644 --- a/internal/controller/gateway_certificate_service_test.go +++ b/internal/controller/gateway_certificate_service_test.go @@ -11,6 +11,8 @@ import ( cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -25,6 +27,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -204,6 +207,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { upstreamObjects func(gw *gatewayv1.Gateway) []client.Object downstreamObjects func() []client.Object serviceObjects []client.Object + serviceForbidden bool assert func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) }{ { @@ -471,6 +475,62 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) }, }, + { + name: "a rejection while the previous certificate serves is reported as a blocked renewal", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain", Message: "names under datum.net are denied", + }) + })} + }, + downstreamObjects: func() []client.Object { + secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) + secret.Labels = map[string]string{tlsCertificateManagedLabel: labelValueTrue} + return []client.Object{secret} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") + assertListenerRenewalBlocked(t, upstreamGateway, listenerName, "names under datum.net are denied") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRejected), 1.0) + }, + }, + { + name: "a forbidden service-side read while the previous certificate serves is reported as a blocked renewal", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + downstreamObjects: func() []client.Object { + return []client.Object{tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM)} + }, + serviceForbidden: true, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") + assertListenerRenewalBlocked(t, upstreamGateway, listenerName, "keep trying") + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), 1.0) + }, + }, + { + name: "refused material while the previous certificate serves is reported as a blocked renewal", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + downstreamObjects: func() []client.Object { + return []client.Object{tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM)} + }, + serviceObjects: func() []client.Object { + crt, key := generateTLSKeyPair(t, "other.example.com", now.Add(-time.Hour), now.Add(60*24*time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var secret corev1.Secret + require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"]) + assertListenerRenewalBlocked(t, upstreamGateway, listenerName, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonMaterialRefused), 1.0) + }, + }, { name: "refuses expired service-side material and keeps the serving Secret", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { @@ -562,10 +622,17 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}). Build() - fakeServiceClient := fake.NewClientBuilder(). + serviceBuilder := fake.NewClientBuilder(). WithScheme(testScheme). - WithObjects(tt.serviceObjects...). - Build() + WithObjects(tt.serviceObjects...) + if tt.serviceForbidden { + serviceBuilder = serviceBuilder.WithInterceptorFuncs(interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return apierrors.NewForbidden(corev1.Resource("secrets"), serviceSecret, nil) + }, + }) + } + fakeServiceClient := serviceBuilder.Build() ctx := log.IntoContext(context.Background(), logger) @@ -597,6 +664,32 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { } } +func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, want string) { + t.Helper() + for _, ls := range gateway.Status.Listeners { + if ls.Name != listener { + continue + } + resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) + require.NotNil(t, resolved) + assert.Equal(t, metav1.ConditionTrue, resolved.Status, "the listener itself stays resolved") + blocked := apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked) + require.NotNil(t, blocked, "a blocked renewal is reported while the listener serves") + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Equal(t, listenerReasonRenewalFailing, blocked.Reason) + assert.Contains(t, blocked.Message, want) + return + } + t.Fatalf("listener %s not found in status", listener) +} + +func counterValue(t *testing.T, vec *prometheus.CounterVec, labels ...string) float64 { + t.Helper() + var m dto.Metric + require.NoError(t, vec.WithLabelValues(labels...).Write(&m)) + return m.GetCounter().GetValue() +} + func assertListenerMessageContains(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, want string) { t.Helper() for _, ls := range gateway.Status.Listeners { @@ -626,15 +719,100 @@ func TestTLSCertificateName(t *testing.T) { func TestCertificateServiceRequeueBackoff(t *testing.T) { r := &GatewayReconciler{} uid := types.UID("gw") - assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true)) - assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(uid, true)) - assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(uid, true)) + now := time.Now() + issues := map[gatewayv1.SectionName]string{"https-0": "trying"} + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) + assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) + assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) for range 10 { - r.certificateServiceRequeue(uid, true) + r.certificateServiceRequeue(uid, true, now, issues) } - assert.Equal(t, certificateServiceBackoffMax, r.certificateServiceRequeue(uid, true)) - assert.Zero(t, r.certificateServiceRequeue(uid, false)) - assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true), "a success resets the backoff") + assert.Equal(t, certificateServiceBackoffMax, r.certificateServiceRequeue(uid, true, now, issues)) + + backoff, remaining, cooling := r.certificateServiceInBackoff(uid, now.Add(time.Minute)) + assert.True(t, cooling, "an event inside the window repeats the message instead of the calls") + assert.Equal(t, []gatewayv1.SectionName{"https-0"}, backoff.listeners) + assert.Equal(t, "trying", backoff.message) + assert.Equal(t, certificateServiceBackoffMax-time.Minute, remaining) + _, _, cooling = r.certificateServiceInBackoff(uid, now.Add(certificateServiceBackoffMax)) + assert.False(t, cooling) + + assert.Zero(t, r.certificateServiceRequeue(uid, false, now, nil)) + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true, now, issues), "a success resets the backoff") + _, _, cooling = r.certificateServiceInBackoff(types.UID("other"), now) + assert.False(t, cooling) +} + +func TestTLSCertificateNameGolden(t *testing.T) { + for _, tt := range []struct{ gateway, listener, want string }{ + {"gw", "https-0", "gw-https-0-bb35a504d1"}, + {"a-b", "c", "a-b-c-4e84717d75"}, + {"a", "b-c", "a-b-c-b88f83c840"}, + {"my.dotted.gateway.name.that.is.long.enough.to.be.cut", "https-0", "my-dotted-gateway-name-that-is-long-enough-to-be-cut-c30f01e877"}, + {"a-gateway-with-a-deliberately-very-long-name-for-this-test", "https-hostname-with-a-long-name-0", "a-gateway-with-a-deliberately-very-long-name-for-thi-8c34270044"}, + } { + assert.Equal(t, tt.want, tlsCertificateName(tt.gateway, gatewayv1.SectionName(tt.listener))) + } +} + +func TestValidateIssuedMaterialWildcard(t *testing.T) { + now := time.Now() + wildcardCrt, wildcardKey := generateTLSKeyPair(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + apexCrt, apexKey := generateTLSKeyPair(t, "example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + + assert.NoError(t, validateIssuedMaterial(wildcardCrt, wildcardKey, "*.example.com", now)) + assert.NoError(t, validateIssuedMaterial(wildcardCrt, wildcardKey, "app.example.com", now)) + assert.Error(t, validateIssuedMaterial(wildcardCrt, wildcardKey, "example.com", now), "a wildcard does not cover the apex") + assert.Error(t, validateIssuedMaterial(apexCrt, apexKey, "*.example.com", now), "an apex certificate does not cover a wildcard listener") +} + +func TestLegacyCertificateHolds(t *testing.T) { + now := time.Now() + base := func() *cmv1.Certificate { + return &cmv1.Certificate{Status: cmv1.CertificateStatus{ + NotBefore: &metav1.Time{Time: now.Add(-30 * 24 * time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + RenewalTime: &metav1.Time{Time: now.Add(-time.Hour)}, + Conditions: []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionTrue}}, + }} + } + issuing := func(since time.Time) cmv1.CertificateCondition { + return cmv1.CertificateCondition{Type: cmv1.CertificateConditionIssuing, Status: cmmeta.ConditionTrue, LastTransitionTime: &metav1.Time{Time: since}} + } + + cert := base() + cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-time.Hour))) + holds, recheck := legacyCertificateHolds(cert, now) + assert.True(t, holds, "a fresh renewal in flight is left to finish") + assert.Equal(t, 10*time.Minute, recheck) + + cert = base() + cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-25*time.Hour))) + holds, _ = legacyCertificateHolds(cert, now) + assert.False(t, holds, "a renewal in flight for over a day is stuck") + + cert = base() + cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-time.Hour))) + cert.Status.LastFailureTime = &metav1.Time{Time: now.Add(-10 * time.Minute)} + holds, _ = legacyCertificateHolds(cert, now) + assert.False(t, holds, "a renewal that has failed does not hold") + + cert = base() + cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-time.Hour))) + cert.Status.NotAfter = &metav1.Time{Time: now.Add(5 * 24 * time.Hour)} + holds, _ = legacyCertificateHolds(cert, now) + assert.False(t, holds, "under a week of lifetime hands over regardless") + + cert = base() + cert.Status.RenewalTime = &metav1.Time{Time: now.Add(10 * 24 * time.Hour)} + holds, recheck = legacyCertificateHolds(cert, now) + assert.True(t, holds) + assert.InDelta(t, (10*24*time.Hour - tlsCertificateSwitchLead).Seconds(), recheck.Seconds(), 1) + + cert = base() + cert.Status.Conditions = []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionFalse, Reason: "Failed"}} + holds, _ = legacyCertificateHolds(cert, now) + assert.False(t, holds, "a certificate that is not serving never holds") } func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { @@ -805,7 +983,7 @@ func TestCertificateServiceRollbackKeepsHandedOverSecret(t *testing.T) { WithStatusSubresource(upstreamGateway).Build() fakeDownstreamClient := fake.NewClientBuilder().WithScheme(testScheme). WithObjects(downstreamGateway, handedOver, solverRoute, solverFilter). - WithStatusSubresource(&gatewayv1.Gateway{}).Build() + WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}).Build() reconciler := &GatewayReconciler{ mgr: &fakeMockManager{cl: fakeUpstreamClient}, @@ -832,6 +1010,17 @@ func TestCertificateServiceRollbackKeepsHandedOverSecret(t *testing.T) { assert.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: "foreign"}, &certificatesv1alpha1.TLSCertificate{}), "a TLSCertificate the gateway does not control stays") assert.True(t, apierrors.IsNotFound(fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: solverName}, &gatewayv1.HTTPRoute{})), "leftover solver routes are removed") assert.True(t, apierrors.IsNotFound(fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: solverName}, &envoygatewayv1alpha1.HTTPRouteFilter{}))) + + if tt.wantCertificate { + var legacy cmv1.Certificate + require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &legacy)) + legacy.Status.Conditions = []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionTrue}} + require.NoError(t, fakeDownstreamClient.Status().Update(ctx, &legacy)) + reconciler.cleanupCertificateServiceLeftovers(ctx, fakeUpstreamClient, upstreamGateway, downstreamGateway, fakeDownstreamClient) + var secret corev1.Secret + require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.NotContains(t, secret.Labels, tlsCertificateManagedLabel, "the hand-over marker goes once cert-manager has the Secret back") + } }) } } diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index e99d053c..06de897b 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -398,10 +398,16 @@ func (r *GatewayReconciler) ensureDownstreamGateway( claimedHostnames, ) for name, message := range issues { - if status, gated := listenerCertHealth[name]; gated && !status.healthy { + status, gated := listenerCertHealth[name] + if !gated { + continue + } + if status.healthy { + status.renewalBlocked = message + } else { status.message = message - listenerCertHealth[name] = status } + listenerCertHealth[name] = status } certificateServiceRequeue = certResult.RequeueAfter } else { @@ -528,8 +534,15 @@ type listenerCertStatus struct { // Carried here so the expiry gauge can be labelled with the secret name // without recomputing it outside listenerCertHealth. secretName string + // renewalBlocked, when set on a healthy listener, says the certificate it + // serves cannot be replaced and why, so the customer hears about it before + // the expiry turns it into an outage. + renewalBlocked string } +const listenerConditionCertificateRenewalBlocked = "CertificateRenewalBlocked" +const listenerReasonRenewalFailing = "RenewalFailing" + // clearListenerCertMetrics removes every certificate-health gauge series for a // gateway. Used both before re-recording each reconcile and on gateway deletion // so a removed listener never leaves a series stuck at its last value. The gating @@ -1825,6 +1838,7 @@ func (r *GatewayReconciler) finalizeGateway( gatewayProgrammedTotal.DeleteLabelValues(upstreamGateway.Namespace, upstreamGateway.Name) // Clear this gateway's cert-health series now that it is gone. clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) + r.certificateServiceFailures.Delete(upstreamGateway.UID) // Clean up DNS records created by this gateway if r.Config.Gateway.EnableDNSIntegration { @@ -2207,6 +2221,16 @@ func (r *GatewayReconciler) ensureDownstreamGatewayHTTPRoutes( apimeta.SetStatusCondition(&status.Conditions, programmedCondition) apimeta.SetStatusCondition(&status.Conditions, resolvedRefsCondition) + if certStatus, gated := listenerCertHealth[listener.Name]; gated && certStatus.healthy && certStatus.renewalBlocked != "" { + apimeta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: listenerConditionCertificateRenewalBlocked, + Status: metav1.ConditionTrue, + Reason: listenerReasonRenewalFailing, + Message: certStatus.renewalBlocked, + ObservedGeneration: upstreamGateway.Generation, + }) + } + listenerStatus = append(listenerStatus, status) } diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go index ebb32ed2..b6741a89 100644 --- a/internal/controller/httpproxy_certificate_service.go +++ b/internal/controller/httpproxy_certificate_service.go @@ -40,6 +40,7 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( legacyName string, secretName string, hostname string, + listenerConditions []metav1.Condition, generation int64, ) metav1.Condition { condition := metav1.Condition{ @@ -54,11 +55,23 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) if condition.Status != metav1.ConditionTrue { if serving := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()); serving.healthy { + failing := condition.Reason == networkingv1alpha.CertificateReadyReasonProvisioningFailed condition.Status = metav1.ConditionTrue condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued condition.Message = "Certificate is ready; a renewal is in progress" + if failing { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed" + if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Message != "" { + condition.Message += ": " + accepted.Message + } + } } } + if blocked := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateRenewalBlocked); blocked != nil && blocked.Status == metav1.ConditionTrue && condition.Status == metav1.ConditionTrue { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed: " + blocked.Message + } return condition case !apierrors.IsNotFound(err): condition.Status = metav1.ConditionUnknown diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go index df07c2df..ba5aa272 100644 --- a/internal/controller/httpproxy_certificate_service_test.go +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -74,13 +74,14 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { } tests := []struct { - name string - upstream []client.Object - mutate func(*certificatesv1alpha1.TLSCertificate) - downstream []client.Object - wantStatus metav1.ConditionStatus - wantReason string - wantMessage string + name string + upstream []client.Object + mutate func(*certificatesv1alpha1.TLSCertificate) + listenerConditions []metav1.Condition + downstream []client.Object + wantStatus metav1.ConditionStatus + wantReason string + wantMessage string }{ { name: "Ready maps to CertificateIssued", @@ -132,6 +133,26 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, wantMessage: "Certificate is ready; a renewal is in progress", }, + { + name: "Accepted=False with a serving downstream Secret reports RenewalFailing but stays ready", + upstream: []client.Object{tlsCert(metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain", Message: "names under datum.net are denied", + })}, + downstream: []client.Object{servingSecret}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonRenewalFailing, + wantMessage: "Certificate is ready but cannot be renewed: names under datum.net are denied", + }, + { + name: "a blocked renewal the gateway reports surfaces as RenewalFailing", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + listenerConditions: []metav1.Condition{{ + Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", + }}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonRenewalFailing, + wantMessage: "Certificate is ready but cannot be renewed: We couldn't request a TLS certificate for app.example.com just now", + }, { name: "no TLSCertificate yet but a ready legacy Certificate answers", downstream: []client.Object{legacyReady()}, @@ -164,7 +185,11 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { DownstreamCluster: &clusterWithClient{c: downstreamClient, scheme: testScheme}, } - got := r.buildCertificateStatuses(context.Background(), upstreamClient, "local", gateway, httpProxy) + gw := gateway.DeepCopy() + if tt.listenerConditions != nil { + gw.Status.Listeners = []gatewayv1.ListenerStatus{{Name: "https-hostname-0", Conditions: tt.listenerConditions}} + } + got := r.buildCertificateStatuses(context.Background(), upstreamClient, "local", gw, httpProxy) require.Len(t, got, 1) assert.Equal(t, "app.example.com", got[0].Hostname) cond := apimeta.FindStatusCondition(got[0].Conditions, networkingv1alpha.HostnameConditionCertificateReady) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 117be93d..4891210f 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1517,7 +1517,7 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( certName := resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-%s", gateway.Name, l.Name)) if r.Config.Gateway.CertificateService.Enabled { - apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), certName, listenerCertificateSecretName(gateway.Name, l.Name), hostname, httpProxy.Generation)) + apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), certName, listenerCertificateSecretName(gateway.Name, l.Name), hostname, listenerStatusConditions(gateway, l.Name), httpProxy.Generation)) statuses = append(statuses, hs) continue } @@ -1584,6 +1584,17 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( return statuses } +// listenerStatusConditions returns the conditions the gateway reports for one +// of its listeners, or nil. +func listenerStatusConditions(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName) []metav1.Condition { + for _, ls := range gateway.Status.Listeners { + if ls.Name == listener { + return ls.Conditions + } + } + return nil +} + // getCertificateReadyConditionReason returns the reason and message for the // CertificateReady condition based on the cert-manager Certificate's Ready condition. func getCertificateReadyConditionReason(certificate *unstructured.Unstructured) (string, string) { diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go index e15e07d6..234892aa 100644 --- a/internal/controller/metrics.go +++ b/internal/controller/metrics.go @@ -83,6 +83,19 @@ var ( // Use sum(nso_gateway_listener_cert_withheld) to count how many listeners are // currently dark across the fleet, or filter by namespace/name/listener/hostname // to find the specific affected object during an incident. + // certificateServiceFailuresTotal counts every certificate-service step that + // could not do its work for a listener, by reason, whether or not the + // listener still serves a certificate. A hostname can serve for weeks on its + // previous certificate while every attempt to replace it fails; this is the + // signal that says so before the expiry does. + certificateServiceFailuresTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nso_certificate_service_failures_total", + Help: "Total certificate-service steps that failed for a Gateway listener, by reason.", + }, + []string{jsonKeyNamespace, jsonKeyName, metricLabelListener, metricLabelReason}, + ) + gatewayListenerCertWithheld = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "nso_gateway_listener_cert_withheld", From f82c180f664b1800213f34ce59cedf2765b96872 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 2 Oct 2026 06:31:00 -0500 Subject: [PATCH 4/8] fix: clear blocked-renewal status once renewal works The blocked-renewal condition on a listener was never removed, so one transient failure read as a permanent one on the listener and on the HTTPProxy hostname. It now goes away on the next pass that does not block, and the hostname status no longer claims an issued certificate that never reached the edge. Key changes: - Remove CertificateRenewalBlocked from a listener whose certificate step no longer reports a problem - Report a hostname whose issued certificate could not be taken, and which has nothing serving, as ProvisioningFailed rather than issued; an issued certificate not yet mirrored is Pending - Keep one cached message per listener while a gateway is in backoff - Count a service rejection once per transition and drop a deleted gateway's failure series Related to datum-cloud/enhancements#913 --- .../controller/gateway_certificate_service.go | 42 +++-- .../gateway_certificate_service_test.go | 156 +++++++++++++++++- internal/controller/gateway_controller.go | 7 +- .../httpproxy_certificate_service.go | 43 +++-- .../httpproxy_certificate_service_test.go | 24 ++- 5 files changed, 240 insertions(+), 32 deletions(-) diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go index 327abbea..5e3a90a3 100644 --- a/internal/controller/gateway_certificate_service.go +++ b/internal/controller/gateway_certificate_service.go @@ -126,10 +126,9 @@ func (r *GatewayReconciler) listenerIssuerResolvable(upstreamGateway *gatewayv1. // the listeners, so an event-driven reconcile inside the window repeats the // message rather than the calls. type certificateServiceBackoff struct { - attempts int - nextTry time.Time - message string - listeners []gatewayv1.SectionName + attempts int + nextTry time.Time + issues map[gatewayv1.SectionName]string } // certificateServiceRequeue records the outcome of a pass and hands back the @@ -148,10 +147,9 @@ func (r *GatewayReconciler) certificateServiceRequeue(gateway types.UID, failed if attempts > 10 || delay > certificateServiceBackoffMax { delay = certificateServiceBackoffMax } - backoff := certificateServiceBackoff{attempts: attempts, nextTry: now.Add(delay)} + backoff := certificateServiceBackoff{attempts: attempts, nextTry: now.Add(delay), issues: make(map[gatewayv1.SectionName]string, len(issues))} for listener, message := range issues { - backoff.listeners = append(backoff.listeners, listener) - backoff.message = message + backoff.issues[listener] = message } r.certificateServiceFailures.Store(gateway, backoff) return delay @@ -175,6 +173,28 @@ func recordCertificateServiceFailure(gateway *gatewayv1.Gateway, listener gatewa certificateServiceFailuresTotal.WithLabelValues(gateway.Namespace, gateway.Name, string(listener), reason).Inc() } +// recordCertificateServiceRejection counts a service rejection once per +// transition into the rejected state, since the rejection is read back on every +// pass for as long as it lasts. +func (r *GatewayReconciler) recordCertificateServiceRejection(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, rejected bool) { + key := gateway.UID + current := map[gatewayv1.SectionName]bool{} + if previous, ok := r.certificateServiceRejections.Load(key); ok { + for k, v := range previous.(map[gatewayv1.SectionName]bool) { + current[k] = v + } + } + if rejected && !current[listener] { + recordCertificateServiceFailure(gateway, listener, certificateServiceReasonRejected) + } + if rejected { + current[listener] = true + } else { + delete(current, listener) + } + r.certificateServiceRejections.Store(key, current) +} + const ( certificateServiceReasonStepFailed = "StepFailed" certificateServiceReasonNotOwned = "NotOwned" @@ -229,8 +249,8 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( } if backoff, remaining, cooling := r.certificateServiceInBackoff(upstreamGateway.UID, now); cooling { - for _, listener := range backoff.listeners { - issues[listener] = backoff.message + for listener, message := range backoff.issues { + issues[listener] = message } requeueSooner(remaining) return result, issues @@ -963,6 +983,7 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( if !apierrors.IsNotFound(err) { logger.Error(err, "failed to get listener TLSCertificate", "tlscertificate", certName) } + r.recordCertificateServiceRejection(upstreamGateway, listenerName, false) if secretStatus.healthy { return secretStatus } @@ -979,10 +1000,11 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( rejected = nil } + r.recordCertificateServiceRejection(upstreamGateway, listenerName, rejected != nil) + if secretStatus.healthy { if rejected != nil { secretStatus.renewalBlocked = tlsCertificateRejectedMessage(hostname, rejected.Message) - recordCertificateServiceFailure(upstreamGateway, listenerName, certificateServiceReasonRejected) } return secretStatus } diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go index 5e093fee..c7a66fc0 100644 --- a/internal/controller/gateway_certificate_service_test.go +++ b/internal/controller/gateway_certificate_service_test.go @@ -720,8 +720,11 @@ func TestCertificateServiceRequeueBackoff(t *testing.T) { r := &GatewayReconciler{} uid := types.UID("gw") now := time.Now() - issues := map[gatewayv1.SectionName]string{"https-0": "trying"} + issues := map[gatewayv1.SectionName]string{"https-0": "trying", "https-1": "other"} assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) + kept, _, _ := r.certificateServiceInBackoff(uid, now) + assert.Equal(t, "other", kept.issues["https-1"], "each listener keeps its own message") + issues = map[gatewayv1.SectionName]string{"https-0": "trying"} assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) for range 10 { @@ -731,8 +734,7 @@ func TestCertificateServiceRequeueBackoff(t *testing.T) { backoff, remaining, cooling := r.certificateServiceInBackoff(uid, now.Add(time.Minute)) assert.True(t, cooling, "an event inside the window repeats the message instead of the calls") - assert.Equal(t, []gatewayv1.SectionName{"https-0"}, backoff.listeners) - assert.Equal(t, "trying", backoff.message) + assert.Equal(t, map[gatewayv1.SectionName]string{"https-0": "trying"}, backoff.issues) assert.Equal(t, certificateServiceBackoffMax-time.Minute, remaining) _, _, cooling = r.certificateServiceInBackoff(uid, now.Add(certificateServiceBackoffMax)) assert.False(t, cooling) @@ -1024,3 +1026,151 @@ func TestCertificateServiceRollbackKeepsHandedOverSecret(t *testing.T) { }) } } + +func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "recovery", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + const hostname = "custom.example.com" + const gatewayName = "recovery-gw" + const listenerName = gatewayv1.SectionName("https-hostname-0") + const serviceNS = "certificates-system" + secretName := listenerCertificateSecretName(gatewayName, listenerName) + certName := tlsCertificateName(gatewayName, listenerName) + + testCfg := config.NetworkServicesOperator{Gateway: config.GatewayConfig{ + DownstreamGatewayClassName: "test-suite", + DownstreamHostnameAccountingNamespace: "default", + TargetDomain: "test-suite.com", + DefaultListenerTLSSecretName: "wildcard-test-suite-tls", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, + ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer"}, + CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: serviceNS}, + }} + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.Spec.Listeners = append(g.Spec.Listeners, gatewayv1.Listener{ + Name: listenerName, Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + TLS: &gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModeTerminate), Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer", + }}, + }) + }) + domain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { + d.Spec.DomainName = hostname + apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue}) + }) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + + now := time.Now() + servingCrt, servingKey := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + issuedCrt, issuedKey := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(90*24*time.Hour)) + + cert := &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: certName}, + Spec: certificatesv1alpha1.TLSCertificateSpec{DNSNames: []certificatesv1alpha1.DNSName{hostname}, Issuance: certificatesv1alpha1.IssuanceModeAuto, SecretName: secretName}, + Status: certificatesv1alpha1.TLSCertificateStatus{ + NotAfter: &metav1.Time{Time: now.Add(90 * 24 * time.Hour)}, + ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}}, + }, + } + require.NoError(t, controllerutil.SetControllerReference(upstreamGateway, cert, testScheme)) + serving := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: secretName}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": servingCrt, "tls.key": servingKey}} + issued := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: "issued"}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}} + for _, obj := range []client.Object{domain, gatewayClass, cert, serving, issued} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + fakeUpstreamClient := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, cert). + WithStatusSubresource(upstreamGateway, cert).Build() + fakeDownstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(serving).WithStatusSubresource(&gatewayv1.Gateway{}).Build() + + forbidden := true + serviceClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(issued).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if forbidden { + return apierrors.NewForbidden(corev1.Resource("secrets"), key.Name, nil) + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + + reconciler := &GatewayReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + Config: testCfg, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + CertificateServiceReader: serviceClient, + } + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", fakeUpstreamClient, fakeDownstreamClient) + reconciler.prepareUpstreamGateway(upstreamGateway) + + reconcile := func() *gatewayv1.Gateway { + var current gatewayv1.Gateway + require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), ¤t)) + result, _ := reconciler.ensureDownstreamGateway(ctx, "test", fakeUpstreamClient, ¤t, downstreamStrategy) + require.NoError(t, result.Err) + _, err := result.Complete(ctx) + require.NoError(t, err) + require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), ¤t)) + + var claims corev1.ConfigMapList + require.NoError(t, fakeDownstreamClient.List(ctx, &claims, client.InNamespace(testCfg.Gateway.DownstreamHostnameAccountingNamespace))) + for i := range claims.Items { + stampCreated(t, ctx, fakeDownstreamClient, &claims.Items[i]) + } + var downstreamGateways gatewayv1.GatewayList + require.NoError(t, fakeDownstreamClient.List(ctx, &downstreamGateways, client.InNamespace(downstreamNamespaceName))) + for i := range downstreamGateways.Items { + stampCreated(t, ctx, fakeDownstreamClient, &downstreamGateways.Items[i]) + } + return ¤t + } + + first := reconcile() + assertListenerRenewalBlocked(t, first, listenerName, "keep trying") + assert.Equal(t, 1.0, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed)) + + forbidden = false + reconciler.certificateServiceFailures.Delete(upstreamGateway.UID) + second := reconcile() + for _, ls := range second.Status.Listeners { + if ls.Name == listenerName { + assert.Nil(t, apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked), "a recovered renewal no longer reads as blocked") + } + } + var mirror corev1.Secret + require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &mirror)) + assert.Equal(t, issuedCrt, mirror.Data["tls.crt"], "the issued certificate replaces the serving one once readable") + assert.Equal(t, 1.0, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), "a success adds nothing") +} + +func TestCertificateServiceRejectionCountedPerTransition(t *testing.T) { + r := &GatewayReconciler{} + gw := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: "count", Name: "gw", UID: uuid.NewUUID()}} + before := counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected) + r.recordCertificateServiceRejection(gw, "https-0", true) + r.recordCertificateServiceRejection(gw, "https-0", true) + r.recordCertificateServiceRejection(gw, "https-0", true) + assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected)) + r.recordCertificateServiceRejection(gw, "https-0", false) + r.recordCertificateServiceRejection(gw, "https-0", true) + assert.Equal(t, before+2, counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected), "a new rejection after recovery counts again") +} + +// stampCreated gives an object the fake client created a creation timestamp, +// which the real API server sets and the controller reads to tell a fresh +// object from an existing one. +func stampCreated(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) { + t.Helper() + if created := obj.GetCreationTimestamp(); !created.IsZero() { + return + } + obj.SetCreationTimestamp(metav1.Now()) + require.NoError(t, cl.Update(ctx, obj)) +} diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 06de897b..87a3bdd1 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -101,7 +101,8 @@ type GatewayReconciler struct { // Config.Gateway.CertificateService.Enabled. CertificateServiceReader client.Reader - certificateServiceFailures sync.Map + certificateServiceFailures sync.Map + certificateServiceRejections sync.Map } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -1839,6 +1840,8 @@ func (r *GatewayReconciler) finalizeGateway( // Clear this gateway's cert-health series now that it is gone. clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) r.certificateServiceFailures.Delete(upstreamGateway.UID) + r.certificateServiceRejections.Delete(upstreamGateway.UID) + certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) // Clean up DNS records created by this gateway if r.Config.Gateway.EnableDNSIntegration { @@ -2229,6 +2232,8 @@ func (r *GatewayReconciler) ensureDownstreamGatewayHTTPRoutes( Message: certStatus.renewalBlocked, ObservedGeneration: upstreamGateway.Generation, }) + } else { + apimeta.RemoveStatusCondition(&status.Conditions, listenerConditionCertificateRenewalBlocked) } listenerStatus = append(listenerStatus, status) diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go index b6741a89..72b2185b 100644 --- a/internal/controller/httpproxy_certificate_service.go +++ b/internal/controller/httpproxy_certificate_service.go @@ -53,24 +53,35 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( switch { case err == nil: condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) - if condition.Status != metav1.ConditionTrue { - if serving := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()); serving.healthy { - failing := condition.Reason == networkingv1alpha.CertificateReadyReasonProvisioningFailed - condition.Status = metav1.ConditionTrue - condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued - condition.Message = "Certificate is ready; a renewal is in progress" - if failing { - condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing - condition.Message = "Certificate is ready but cannot be renewed" - if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Message != "" { - condition.Message += ": " + accepted.Message - } + serving := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()) + blocked := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateRenewalBlocked) + if blocked != nil && blocked.Status != metav1.ConditionTrue { + blocked = nil + } + + switch { + case serving.healthy && (condition.Status != metav1.ConditionTrue || blocked != nil): + condition.Status = metav1.ConditionTrue + condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued + condition.Message = "Certificate is ready; a renewal is in progress" + if blocked != nil { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed: " + blocked.Message + } else if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed" + if accepted.Message != "" { + condition.Message += ": " + accepted.Message } } - } - if blocked := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateRenewalBlocked); blocked != nil && blocked.Status == metav1.ConditionTrue && condition.Status == metav1.ConditionTrue { - condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing - condition.Message = "Certificate is ready but cannot be renewed: " + blocked.Message + case !serving.healthy && blocked != nil: + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonProvisioningFailed + condition.Message = blocked.Message + case !serving.healthy && condition.Status == metav1.ConditionTrue: + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = "The certificate has been issued and is being applied to this hostname" } return condition case !apierrors.IsNotFound(err): diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go index ba5aa272..97c55e70 100644 --- a/internal/controller/httpproxy_certificate_service_test.go +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -86,6 +86,7 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { { name: "Ready maps to CertificateIssued", upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + downstream: []client.Object{servingSecret}, wantStatus: metav1.ConditionTrue, wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, }, @@ -144,8 +145,9 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { wantMessage: "Certificate is ready but cannot be renewed: names under datum.net are denied", }, { - name: "a blocked renewal the gateway reports surfaces as RenewalFailing", - upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + name: "a blocked renewal the gateway reports surfaces as RenewalFailing", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + downstream: []client.Object{servingSecret}, listenerConditions: []metav1.Condition{{ Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", }}, @@ -153,6 +155,24 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { wantReason: networkingv1alpha.CertificateReadyReasonRenewalFailing, wantMessage: "Certificate is ready but cannot be renewed: We couldn't request a TLS certificate for app.example.com just now", }, + { + name: "Ready but the issued Secret could not be taken and nothing serves is a provisioning failure", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + listenerConditions: []metav1.Condition{{ + Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", + }}, + downstream: nil, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, + wantMessage: "We couldn't request a TLS certificate for app.example.com just now", + }, + { + name: "Ready but not yet mirrored is pending, not issued", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonPending, + wantMessage: "The certificate has been issued and is being applied to this hostname", + }, { name: "no TLSCertificate yet but a ready legacy Certificate answers", downstream: []client.Object{legacyReady()}, From 13b36e63ad4147779bb88e5b2e4998eac1f6106f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 2 Oct 2026 06:49:21 -0500 Subject: [PATCH 5/8] fix: report blocked issuance on new hostnames A new hostname whose issued certificate could not be read was reported as "being applied" for as long as the failure lasted, because the gateway only flagged blocked renewals on listeners that already served. It now flags blocked issuance on the others as well, and HTTPProxy reads both. Key changes: - Add the CertificateIssuanceBlocked listener condition for a listener with nothing serving whose certificate step failed; HTTPProxy maps it to ProvisioningFailed - Drop a removed listener's failure series and tracker entries while the gateway lives on - Compare counter deltas in tests so repeated runs pass Related to datum-cloud/enhancements#913 --- .../controller/gateway_certificate_service.go | 28 ++++++++ .../gateway_certificate_service_test.go | 65 +++++++++++++------ internal/controller/gateway_controller.go | 19 ++++++ .../httpproxy_certificate_service.go | 5 ++ .../httpproxy_certificate_service_test.go | 2 +- 5 files changed, 99 insertions(+), 20 deletions(-) diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go index 5e3a90a3..753cf708 100644 --- a/internal/controller/gateway_certificate_service.go +++ b/internal/controller/gateway_certificate_service.go @@ -17,6 +17,7 @@ import ( cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -339,10 +340,37 @@ func (r *GatewayReconciler) ensureListenerTLSCertificates( } requeueSooner(r.certificateServiceRequeue(upstreamGateway.UID, failed, now, issues)) + r.forgetRemovedListeners(upstreamGateway) return result, issues } +// forgetRemovedListeners drops the failure series and tracker entries of +// listeners the gateway no longer has, so a removed hostname stops reporting. +func (r *GatewayReconciler) forgetRemovedListeners(upstreamGateway *gatewayv1.Gateway) { + current := sets.New[gatewayv1.SectionName]() + for _, l := range upstreamGateway.Spec.Listeners { + current.Insert(l.Name) + } + if previous, ok := r.certificateServiceListeners.Load(upstreamGateway.UID); ok { + for listener := range previous.(sets.Set[gatewayv1.SectionName]) { + if current.Has(listener) { + continue + } + certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{ + jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name, metricLabelListener: string(listener), + }) + if rejected, ok := r.certificateServiceRejections.Load(upstreamGateway.UID); ok { + delete(rejected.(map[gatewayv1.SectionName]bool), listener) + } + if backoff, ok := r.certificateServiceFailures.Load(upstreamGateway.UID); ok { + delete(backoff.(certificateServiceBackoff).issues, listener) + } + } + } + r.certificateServiceListeners.Store(upstreamGateway.UID, current) +} + var errTLSCertificateNotOwned = errors.New("TLSCertificate exists but is not controlled by this Gateway") // mirrorRefusal says why issued material was not taken, for the customer and diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go index c7a66fc0..03e7d3e3 100644 --- a/internal/controller/gateway_certificate_service_test.go +++ b/internal/controller/gateway_certificate_service_test.go @@ -435,7 +435,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { require.NoError(t, e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) assert.Equal(t, []certificatesv1alpha1.DNSName{"previous.example.com"}, cert.Spec.DNSNames, "someone else's TLSCertificate is left alone") assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter, "the clash is retried with backoff, not an error") - assertListenerMessageContains(t, upstreamGateway, listenerName, "keep trying") + assertListenerSaysKeepTrying(t, upstreamGateway) }, }, { @@ -449,7 +449,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { serviceObjects: []client.Object{tlsSecret("victim-namespace", serviceSecret, serviceCertPEM, serviceKeyPEM)}, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) - assertListenerMessageContains(t, upstreamGateway, listenerName, "keep trying") + assertListenerSaysKeepTrying(t, upstreamGateway) }, }, { @@ -491,7 +491,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") - assertListenerRenewalBlocked(t, upstreamGateway, listenerName, "names under datum.net are denied") + assertListenerRenewalBlocked(t, upstreamGateway, "names under datum.net are denied") assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRejected), 1.0) }, }, @@ -506,11 +506,32 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { serviceForbidden: true, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") - assertListenerRenewalBlocked(t, upstreamGateway, listenerName, "keep trying") + assertListenerRenewalBlocked(t, upstreamGateway, "keep trying") assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), 1.0) }, }, + { + name: "a forbidden service-side read with nothing serving is reported as blocked issuance", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceForbidden: true, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + assertListenerSaysKeepTrying(t, upstreamGateway) + for _, ls := range upstreamGateway.Status.Listeners { + if ls.Name != listenerName { + continue + } + blocked := apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateIssuanceBlocked) + require.NotNil(t, blocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Equal(t, listenerReasonIssuanceFailing, blocked.Reason) + assert.Contains(t, blocked.Message, "keep trying") + } + }, + }, { name: "refused material while the previous certificate serves is reported as a blocked renewal", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { @@ -527,7 +548,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { var secret corev1.Secret require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"]) - assertListenerRenewalBlocked(t, upstreamGateway, listenerName, "keep trying") + assertListenerRenewalBlocked(t, upstreamGateway, "keep trying") assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonMaterialRefused), 1.0) }, }, @@ -664,10 +685,10 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { } } -func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, want string) { +func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, want string) { t.Helper() for _, ls := range gateway.Status.Listeners { - if ls.Name != listener { + if apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked) == nil { continue } resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) @@ -680,7 +701,7 @@ func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, list assert.Contains(t, blocked.Message, want) return } - t.Fatalf("listener %s not found in status", listener) + t.Fatal("no listener reports a blocked renewal") } func counterValue(t *testing.T, vec *prometheus.CounterVec, labels ...string) float64 { @@ -690,19 +711,17 @@ func counterValue(t *testing.T, vec *prometheus.CounterVec, labels ...string) fl return m.GetCounter().GetValue() } -func assertListenerMessageContains(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, want string) { +func assertListenerSaysKeepTrying(t *testing.T, gateway *gatewayv1.Gateway) { t.Helper() for _, ls := range gateway.Status.Listeners { - if ls.Name != listener { + resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) + if resolved == nil || resolved.Status != metav1.ConditionFalse { continue } - resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) - require.NotNil(t, resolved) - assert.Equal(t, metav1.ConditionFalse, resolved.Status) - assert.Contains(t, resolved.Message, want) + assert.Contains(t, resolved.Message, "keep trying") return } - t.Fatalf("listener %s not found in status", listener) + t.Fatal("no listener reports an unresolved certificate") } func TestTLSCertificateName(t *testing.T) { @@ -889,7 +908,7 @@ func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { var updated gatewayv1.Gateway require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), &updated)) assert.NotEmpty(t, updated.Status.Listeners, "gateway status is still written") - assertListenerMessageContains(t, &updated, "https-hostname-0", "keep trying") + assertListenerSaysKeepTrying(t, &updated) var downstreamRoutes gatewayv1.HTTPRouteList require.NoError(t, fakeDownstreamClient.List(ctx, &downstreamRoutes, client.InNamespace(downstreamGateway.Namespace))) @@ -1132,9 +1151,10 @@ func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { return ¤t } + before := counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed) first := reconcile() - assertListenerRenewalBlocked(t, first, listenerName, "keep trying") - assert.Equal(t, 1.0, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed)) + assertListenerRenewalBlocked(t, first, "keep trying") + assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed)) forbidden = false reconciler.certificateServiceFailures.Delete(upstreamGateway.UID) @@ -1147,7 +1167,14 @@ func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { var mirror corev1.Secret require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &mirror)) assert.Equal(t, issuedCrt, mirror.Data["tls.crt"], "the issued certificate replaces the serving one once readable") - assert.Equal(t, 1.0, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), "a success adds nothing") + assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), "a success adds nothing") + + removed := second.DeepCopy() + removed.Spec.Listeners = removed.Spec.Listeners[:len(removed.Spec.Listeners)-1] + reconciler.forgetRemovedListeners(removed) + var m dto.Metric + require.NoError(t, certificateServiceFailuresTotal.WithLabelValues(upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed).Write(&m)) + assert.Zero(t, m.GetCounter().GetValue(), "a removed listener's series is dropped") } func TestCertificateServiceRejectionCountedPerTransition(t *testing.T) { diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 87a3bdd1..4f9a849b 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -103,6 +103,7 @@ type GatewayReconciler struct { certificateServiceFailures sync.Map certificateServiceRejections sync.Map + certificateServiceListeners sync.Map } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -407,6 +408,7 @@ func (r *GatewayReconciler) ensureDownstreamGateway( status.renewalBlocked = message } else { status.message = message + status.issuanceBlocked = true } listenerCertHealth[name] = status } @@ -539,10 +541,15 @@ type listenerCertStatus struct { // serves cannot be replaced and why, so the customer hears about it before // the expiry turns it into an outage. renewalBlocked string + // issuanceBlocked, when set on an unhealthy listener, says the certificate + // step itself failed rather than issuance merely being underway. + issuanceBlocked bool } const listenerConditionCertificateRenewalBlocked = "CertificateRenewalBlocked" const listenerReasonRenewalFailing = "RenewalFailing" +const listenerConditionCertificateIssuanceBlocked = "CertificateIssuanceBlocked" +const listenerReasonIssuanceFailing = "IssuanceFailing" // clearListenerCertMetrics removes every certificate-health gauge series for a // gateway. Used both before re-recording each reconcile and on gateway deletion @@ -1841,6 +1848,7 @@ func (r *GatewayReconciler) finalizeGateway( clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) r.certificateServiceFailures.Delete(upstreamGateway.UID) r.certificateServiceRejections.Delete(upstreamGateway.UID) + r.certificateServiceListeners.Delete(upstreamGateway.UID) certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) // Clean up DNS records created by this gateway @@ -2235,6 +2243,17 @@ func (r *GatewayReconciler) ensureDownstreamGatewayHTTPRoutes( } else { apimeta.RemoveStatusCondition(&status.Conditions, listenerConditionCertificateRenewalBlocked) } + if certStatus, gated := listenerCertHealth[listener.Name]; gated && !certStatus.healthy && certStatus.issuanceBlocked { + apimeta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: listenerConditionCertificateIssuanceBlocked, + Status: metav1.ConditionTrue, + Reason: listenerReasonIssuanceFailing, + Message: certStatus.message, + ObservedGeneration: upstreamGateway.Generation, + }) + } else { + apimeta.RemoveStatusCondition(&status.Conditions, listenerConditionCertificateIssuanceBlocked) + } listenerStatus = append(listenerStatus, status) } diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go index 72b2185b..dad81d48 100644 --- a/internal/controller/httpproxy_certificate_service.go +++ b/internal/controller/httpproxy_certificate_service.go @@ -58,6 +58,11 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( if blocked != nil && blocked.Status != metav1.ConditionTrue { blocked = nil } + if blocked == nil { + if issuance := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateIssuanceBlocked); issuance != nil && issuance.Status == metav1.ConditionTrue { + blocked = issuance + } + } switch { case serving.healthy && (condition.Status != metav1.ConditionTrue || blocked != nil): diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go index 97c55e70..2841c301 100644 --- a/internal/controller/httpproxy_certificate_service_test.go +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -159,7 +159,7 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { name: "Ready but the issued Secret could not be taken and nothing serves is a provisioning failure", upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, listenerConditions: []metav1.Condition{{ - Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", + Type: listenerConditionCertificateIssuanceBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonIssuanceFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", }}, downstream: nil, wantStatus: metav1.ConditionFalse, From c1e8b52eb4c535fd820e9a412119544c0492d793 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 2 Oct 2026 18:16:24 -0500 Subject: [PATCH 6/8] refactor: limit certificate service to wildcards Only single-label wildcard hostnames are issued by the Milo certificate service, always over DNS-01. Exact hostnames stay on cert-manager whether the flag is on or off, so the service never takes over a certificate that already serves. Migrating exact hostnames waits until the wildcard path is proven; the full migration is kept on backup/certificate-service-full- migration. Removed, since only the exact-hostname migration needed them: - the hand-over of cert-manager Certificates 48h before renewal, with its hold, stuck-renewal and expiry-floor timing - rollback cleanup when the flag goes off, and the cert-manager path's check for a handed-over Secret before reissuing - retire and delete of legacy Certificates on the service path - the certificate-service Secret label used to track hand-overs - HTTP-01 answering for TLSCertificates and the solver refactor that shared route code with the cert-manager solver Key changes: - surface every failing or stalled renewal, not only rejections: a failed ACME order, a request short of Ready past an hour, and a served certificate past its renewal point raise CertificateRenewalBlocked or CertificateIssuanceBlocked and count once in the failure metric - add nso_certificate_service_listener_failing so a lasting failure stays visible after its single count - refuse issued chains the configured roots do not trust, behind verifyChain and an optional trustedRootsFile - back off per listener, so one failing hostname no longer delays the others on its gateway - tell a hostname the API refused why, instead of promising to retry - recheck wildcard listeners hourly so a quiet stall is noticed --- internal/cmd/manager/manager.go | 9 + internal/config/config.go | 46 +- internal/config/config_test.go | 30 + .../controller/gateway_certificate_service.go | 883 +++++------ .../gateway_certificate_service_test.go | 1318 +++++++++-------- internal/controller/gateway_controller.go | 171 +-- ...ownstream_certificate_solver_controller.go | 97 +- internal/controller/http01_solver.go | 152 -- internal/controller/http01_solver_test.go | 70 - .../httpproxy_certificate_service.go | 104 +- .../httpproxy_certificate_service_test.go | 55 +- internal/controller/httpproxy_controller.go | 4 +- internal/controller/metrics.go | 38 +- 13 files changed, 1452 insertions(+), 1525 deletions(-) delete mode 100644 internal/controller/http01_solver.go delete mode 100644 internal/controller/http01_solver_test.go diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 19000086..56768bb0 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -4,6 +4,7 @@ package managercmd import ( "context" + "crypto/x509" "errors" "flag" "fmt" @@ -391,6 +392,11 @@ func NewCommand(build BuildInfo) *cobra.Command { setupLog.Error(err, "unable to build certificate service client") os.Exit(1) } + certificateServiceRoots, err := serverConfig.Gateway.CertificateService.TrustedRoots() + if err != nil { + setupLog.Error(err, "unable to load certificate service trusted roots") + os.Exit(1) + } registeredControllers, err := setupControllers(mgr, serverConfig, controllerDeps{ downstreamCluster: downstreamCluster, @@ -398,6 +404,7 @@ func NewCommand(build BuildInfo) *cobra.Command { irohDownstream: irohDownstream, ipamClients: ipamClients, certificateServiceReader: certificateServiceReader, + certificateServiceRoots: certificateServiceRoots, }) if err != nil { setupLog.Error(err, "unable to set up controllers") @@ -575,6 +582,7 @@ type controllerDeps struct { irohDownstream cluster.Cluster ipamClients controller.IPAMClientFactory certificateServiceReader client.Reader + certificateServiceRoots *x509.CertPool } // newCertificateServiceReader returns an uncached client for the cluster the @@ -708,6 +716,7 @@ func controllerRegistrations( Config: serverConfig, DownstreamCluster: deps.downstreamCluster, CertificateServiceReader: deps.certificateServiceReader, + CertificateServiceRoots: deps.certificateServiceRoots, }).SetupWithManager(mgr) }}, {"gatewayclass", true, func() error { diff --git a/internal/config/config.go b/internal/config/config.go index d108d1d4..2cecd3b8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "context" "crypto/tls" + "crypto/x509" "errors" "fmt" "os" @@ -906,12 +907,11 @@ type GatewayConfig struct { // Challenge, solver resources). CertificateReissuance CertificateReissuanceConfig `json:"certificateReissuance,omitempty"` - // CertificateService hands per-hostname certificate issuance to the Milo - // certificate service. When enabled, the gateway controller requests a - // TLSCertificate in the project control plane for each custom hostname, - // serves its HTTP-01 challenges on the downstream gateway, and mirrors the - // issued Secret downstream, instead of creating a cert-manager Certificate - // on the downstream cluster. + // CertificateService hands certificate issuance for wildcard hostnames to + // the Milo certificate service. When enabled, the gateway controller + // requests a DNS-01 TLSCertificate in the project control plane for each + // single-label wildcard listener and mirrors the issued Secret downstream. + // Exact hostnames stay on cert-manager either way. CertificateService CertificateServiceConfig `json:"certificateService,omitempty"` } @@ -920,8 +920,8 @@ type GatewayConfig struct { // CertificateServiceConfig controls consumption of the Milo certificate // service (certificates.miloapis.com). type CertificateServiceConfig struct { - // Enabled switches custom hostname certificate issuance from downstream - // cert-manager Certificates to upstream TLSCertificates. + // Enabled issues certificates for wildcard hostnames through upstream + // TLSCertificates. // // Defaults to false. Enabled bool `json:"enabled,omitempty"` @@ -936,6 +936,33 @@ type CertificateServiceConfig struct { // // +default="certificates-system" SecretNamespace string `json:"secretNamespace,omitempty"` + + // VerifyChain refuses issued material whose chain does not build to + // TrustedRootsFile, or to the system roots when that is empty. + // + // Defaults to false. + VerifyChain bool `json:"verifyChain,omitempty"` + + // TrustedRootsFile is a PEM bundle of the roots an issued chain must + // build to when VerifyChain is set. + TrustedRootsFile string `json:"trustedRootsFile,omitempty"` +} + +// TrustedRoots returns the roots an issued chain must build to, or nil when +// chain verification is off or uses the system roots. +func (c *CertificateServiceConfig) TrustedRoots() (*x509.CertPool, error) { + if !c.VerifyChain || c.TrustedRootsFile == "" { + return nil, nil + } + data, err := os.ReadFile(c.TrustedRootsFile) + if err != nil { + return nil, fmt.Errorf("failed to read trusted roots: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(data) { + return nil, fmt.Errorf("trusted roots file %s holds no certificates", c.TrustedRootsFile) + } + return pool, nil } func SetDefaults_CertificateServiceConfig(obj *CertificateServiceConfig) { @@ -1557,6 +1584,9 @@ func (c *GatewayConfig) validate() error { if c.CertificateService.Enabled && strings.TrimSpace(c.CertificateService.SecretNamespace) == "" { errs = append(errs, errors.New("certificateService.secretNamespace is required when certificateService.enabled")) } + if c.CertificateService.TrustedRootsFile != "" && !c.CertificateService.VerifyChain { + errs = append(errs, errors.New("certificateService.trustedRootsFile requires certificateService.verifyChain")) + } return errors.Join(errs...) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index caf38a02..5780a786 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,8 @@ package config import ( + "os" + "path/filepath" "strings" "testing" ) @@ -231,6 +233,11 @@ func TestGatewayConfig_ValidateCertificateService(t *testing.T) { }, wantSub: "certificateService.enabled requires hostname verification", }, + { + name: "a trusted roots file without chain verification is rejected", + gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system", TrustedRootsFile: "/roots.pem"}}, + wantSub: "certificateService.trustedRootsFile requires certificateService.verifyChain", + }, } for _, tt := range tests { @@ -262,4 +269,27 @@ func TestSetObjectDefaults_CertificateService(t *testing.T) { if got, want := cfg.Gateway.CertificateService.SecretNamespace, "certificates-system"; got != want { t.Errorf("certificateService.secretNamespace = %q, want %q", got, want) } + if cfg.Gateway.CertificateService.VerifyChain { + t.Error("certificateService.verifyChain should default to false") + } +} + +func TestCertificateServiceConfig_TrustedRoots(t *testing.T) { + off := CertificateServiceConfig{TrustedRootsFile: "/does/not/exist"} + if pool, err := off.TrustedRoots(); pool != nil || err != nil { + t.Fatalf("verification off should load nothing, got %v, %v", pool, err) + } + + system := CertificateServiceConfig{VerifyChain: true} + if pool, err := system.TrustedRoots(); pool != nil || err != nil { + t.Fatalf("no roots file means the system roots, got %v, %v", pool, err) + } + + empty := filepath.Join(t.TempDir(), "empty.pem") + if err := os.WriteFile(empty, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := (&CertificateServiceConfig{VerifyChain: true, TrustedRootsFile: empty}).TrustedRoots(); err == nil { + t.Fatal("a roots file with no certificates should be refused") + } } diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go index 753cf708..7a125820 100644 --- a/internal/controller/gateway_certificate_service.go +++ b/internal/controller/gateway_certificate_service.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/hex" + "encoding/pem" "errors" "fmt" "regexp" @@ -15,8 +16,6 @@ import ( "strings" "time" - cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,13 +40,6 @@ import ( // +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates/status,verbs=get -const tlsCertificateSolverLabel = "networking.datumapis.com/tlscertificate-solver" - -// tlsCertificateManagedLabel marks a downstream Secret the certificate service -// path owns: either handed over from cert-manager at the switch or written by -// the mirror. The cert-manager path reads it to avoid reissuing on rollback. -const tlsCertificateManagedLabel = "networking.datumapis.com/certificate-service" - const KindTLSCertificate = "TLSCertificate" // tlsCertificateMirrorAdmitDelay is how soon the listener is re-evaluated after @@ -55,17 +47,22 @@ const KindTLSCertificate = "TLSCertificate" // mirror in the same pass. const tlsCertificateMirrorAdmitDelay = time.Second -// tlsCertificateSwitchLead is how far ahead of cert-manager's own renewal time -// a hostname moves to the service, so the two never order for the same name. -const tlsCertificateSwitchLead = 48 * time.Hour - const ( certificateServiceBackoffBase = 5 * time.Second certificateServiceBackoffMax = 5 * time.Minute - legacyRecheckMax = 6 * time.Hour -) -var acmeTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`) + // certificateServiceRecheck is how often a wildcard listener is looked at + // again without any event, so a renewal that quietly stalls is noticed. + certificateServiceRecheck = time.Hour + + // tlsCertificateIssueGrace is how long a TLSCertificate may stay not Ready + // before it is reported as failing rather than in progress. + tlsCertificateIssueGrace = time.Hour + + // renewalOverdueDivisor sets the share of a served certificate's lifetime + // below which its renewal is overdue. The service renews with a third left. + renewalOverdueDivisor = 4 +) var nonDNSLabelChars = regexp.MustCompile(`[^a-z0-9-]+`) @@ -88,9 +85,11 @@ func tlsCertificateName(gatewayName string, listenerName gatewayv1.SectionName) return prefix + "-" + suffix } -func tlsCertificateSolverName(certName, token string) string { - sum := sha256.Sum256([]byte(token)) - return fmt.Sprintf("%s-acme-%s", certName, hex.EncodeToString(sum[:])[:10]) +// isSingleLabelWildcard reports whether a hostname is a wildcard over exactly +// one label, the only shape the certificate service is used for. +func isSingleLabelWildcard(hostname string) bool { + base, ok := strings.CutPrefix(hostname, "*.") + return ok && base != "" && !strings.Contains(base, "*") } // listenerWantsOwnCertificate reports whether a listener is one the controller @@ -111,6 +110,20 @@ func (r *GatewayReconciler) listenerWantsOwnCertificate(l gatewayv1.Listener, cl return hostname, true } +// listenerUsesCertificateService reports whether the certificate service, not +// cert-manager, issues for this listener: only wildcard hostnames, and only +// while the service is enabled. Every exact hostname stays on cert-manager. +func (r *GatewayReconciler) listenerUsesCertificateService(l gatewayv1.Listener, claimedHostnames []string) (string, bool) { + if !r.Config.Gateway.CertificateService.Enabled { + return "", false + } + hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) + if !wanted || !isSingleLabelWildcard(hostname) { + return "", false + } + return hostname, true +} + // listenerIssuerResolvable applies the same rule the cert-manager path does to // the certificate-issuer option: an `auto` that maps to nothing and inherits // from no other listener leaves the listener un-programmed. @@ -122,44 +135,47 @@ func (r *GatewayReconciler) listenerIssuerResolvable(upstreamGateway *gatewayv1. return issuer != autoIssuerSentinel || r.resolveAutoIssuer(upstreamGateway) != "" } -// certificateServiceBackoff is what the step remembers about a gateway whose +// certificateServiceKey identifies one listener of one gateway. An empty +// listener stands for the gateway-wide cleanup. +type certificateServiceKey struct { + gateway types.UID + listener gatewayv1.SectionName +} + +// certificateServiceBackoff is what the step remembers about a listener whose // last pass failed: how many in a row, when it may try again, and what it told -// the listeners, so an event-driven reconcile inside the window repeats the +// the customer, so an event-driven reconcile inside the window repeats the // message rather than the calls. type certificateServiceBackoff struct { attempts int nextTry time.Time - issues map[gatewayv1.SectionName]string + issue certificateServiceIssue } -// certificateServiceRequeue records the outcome of a pass and hands back the -// delay before the next attempt, doubling per consecutive failure up to a cap, -// and forgets the gateway once a pass succeeds. -func (r *GatewayReconciler) certificateServiceRequeue(gateway types.UID, failed bool, now time.Time, issues map[gatewayv1.SectionName]string) time.Duration { +// certificateServiceRequeue records the outcome of a listener's pass and hands +// back the delay before the next attempt, doubling per consecutive failure up +// to a cap, and forgets the listener once a pass succeeds. +func (r *GatewayReconciler) certificateServiceRequeue(key certificateServiceKey, failed bool, now time.Time, issue certificateServiceIssue) time.Duration { if !failed { - r.certificateServiceFailures.Delete(gateway) + r.certificateServiceFailures.Delete(key) return 0 } attempts := 1 - if previous, ok := r.certificateServiceFailures.Load(gateway); ok { + if previous, ok := r.certificateServiceFailures.Load(key); ok { attempts = previous.(certificateServiceBackoff).attempts + 1 } delay := certificateServiceBackoffBase << (attempts - 1) if attempts > 10 || delay > certificateServiceBackoffMax { delay = certificateServiceBackoffMax } - backoff := certificateServiceBackoff{attempts: attempts, nextTry: now.Add(delay), issues: make(map[gatewayv1.SectionName]string, len(issues))} - for listener, message := range issues { - backoff.issues[listener] = message - } - r.certificateServiceFailures.Store(gateway, backoff) + r.certificateServiceFailures.Store(key, certificateServiceBackoff{attempts: attempts, nextTry: now.Add(delay), issue: issue}) return delay } -// certificateServiceInBackoff reports whether the gateway's last failure is -// still cooling off, and if so the listeners it concerned and how long is left. -func (r *GatewayReconciler) certificateServiceInBackoff(gateway types.UID, now time.Time) (certificateServiceBackoff, time.Duration, bool) { - previous, ok := r.certificateServiceFailures.Load(gateway) +// certificateServiceInBackoff reports whether the listener's last failure is +// still cooling off, and if so what it was told and how long is left. +func (r *GatewayReconciler) certificateServiceInBackoff(key certificateServiceKey, now time.Time) (certificateServiceBackoff, time.Duration, bool) { + previous, ok := r.certificateServiceFailures.Load(key) if !ok { return certificateServiceBackoff{}, 0, false } @@ -174,177 +190,215 @@ func recordCertificateServiceFailure(gateway *gatewayv1.Gateway, listener gatewa certificateServiceFailuresTotal.WithLabelValues(gateway.Namespace, gateway.Name, string(listener), reason).Inc() } -// recordCertificateServiceRejection counts a service rejection once per -// transition into the rejected state, since the rejection is read back on every -// pass for as long as it lasts. -func (r *GatewayReconciler) recordCertificateServiceRejection(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, rejected bool) { - key := gateway.UID - current := map[gatewayv1.SectionName]bool{} - if previous, ok := r.certificateServiceRejections.Load(key); ok { - for k, v := range previous.(map[gatewayv1.SectionName]bool) { - current[k] = v - } +// recordCertificateServiceState counts a TLSCertificate failure once per +// transition into it, since the same failure is read back on every pass for +// as long as it lasts. An empty reason means the listener is healthy. +func (r *GatewayReconciler) recordCertificateServiceState(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, reason string) { + key := certificateServiceKey{gateway: gateway.UID, listener: listener} + previous, _ := r.certificateServiceStates.Load(key) + if reason != "" && previous != reason { + recordCertificateServiceFailure(gateway, listener, reason) } - if rejected && !current[listener] { - recordCertificateServiceFailure(gateway, listener, certificateServiceReasonRejected) - } - if rejected { - current[listener] = true - } else { - delete(current, listener) + if reason == "" { + r.certificateServiceStates.Delete(key) + return } - r.certificateServiceRejections.Store(key, current) + r.certificateServiceStates.Store(key, reason) } const ( certificateServiceReasonStepFailed = "StepFailed" certificateServiceReasonNotOwned = "NotOwned" + certificateServiceReasonRefused = "Refused" certificateServiceReasonRejected = "Rejected" + certificateServiceReasonIssuanceFailed = "IssuanceFailed" + certificateServiceReasonNotReady = "NotReady" + certificateServiceReasonRenewalOverdue = "RenewalOverdue" certificateServiceReasonNamespaceRefused = "NamespaceRefused" certificateServiceReasonMaterialRefused = "MaterialRefused" - certificateServiceReasonTokenRefused = "TokenRefused" + certificateServiceReasonUntrustedChain = "UntrustedChain" ) func certificateServiceUnavailableMessage(hostname string) string { return fmt.Sprintf("We couldn't request a TLS certificate for %s just now and will keep trying. HTTPS for this hostname stays as it is in the meantime.", hostname) } +func certificateRequestRefusedMessage(hostname, detail string) string { + return fmt.Sprintf("A TLS certificate cannot be issued for %s: %s", hostname, detail) +} + +func certificateRequestNotOwnedMessage(hostname string) string { + return fmt.Sprintf("A TLS certificate cannot be requested for %s because another resource holds the certificate request this hostname would use.", hostname) +} + +func certificateMaterialRefusedMessage(hostname string) string { + return fmt.Sprintf("The TLS certificate issued for %s did not pass our checks and was not applied. HTTPS for this hostname stays as it is in the meantime.", hostname) +} + func certificateBeingReplacedMessage(hostname string) string { return fmt.Sprintf("The TLS certificate request for %s is being replaced. HTTPS for this hostname stays as it is in the meantime.", hostname) } +func certificateRenewalOverdueMessage(hostname string, notAfter time.Time) string { + return fmt.Sprintf("The TLS certificate for %s expires on %s and has not been renewed.", hostname, notAfter.UTC().Format(time.DateOnly)) +} + +func certificateNotIssuedMessage(hostname, detail string) string { + if detail == "" { + return fmt.Sprintf("The TLS certificate for %s has not been issued after more than an hour.", hostname) + } + return fmt.Sprintf("The TLS certificate for %s has not been issued: %s", hostname, detail) +} + +// certificateServiceIssue is what one listener's pass could not do: the +// counter reason and the message for the customer. +type certificateServiceIssue struct { + reason string + message string +} + // ensureListenerTLSCertificates is the certificate-service counterpart of -// ensureListenerCertificates: one TLSCertificate per custom hostname in the -// project control plane, its HTTP-01 challenges answered on the downstream -// gateway, and its issued Secret mirrored downstream under the name the -// listener references. Nothing here fails the gateway reconcile: a listener -// whose step failed keeps whatever Secret it has, the failure is returned as a -// message for its status, and the whole step retries with backoff. +// ensureListenerCertificates for wildcard listeners: one TLSCertificate each in +// the project control plane, issued over DNS-01, and its issued Secret +// mirrored downstream under the name the listener references. Nothing here +// fails the gateway reconcile: a listener whose step failed keeps whatever +// Secret it has, the failure is returned as a message for its status, and that +// listener alone retries with backoff. func (r *GatewayReconciler) ensureListenerTLSCertificates( ctx context.Context, upstreamClient client.Client, upstreamGateway *gatewayv1.Gateway, downstreamGateway *gatewayv1.Gateway, - downstreamClient client.Client, downstreamStrategy downstreamclient.ResourceStrategy, claimedHostnames []string, ) (result Result, issues map[gatewayv1.SectionName]string) { logger := log.FromContext(ctx) now := time.Now() issues = make(map[gatewayv1.SectionName]string) + failing := make(map[gatewayv1.SectionName]string) desiredCerts := sets.New[string]() - legacyKeep := sets.New[string]() - liveSolvers := sets.New[string]() - failed := false requeueSooner := func(d time.Duration) { if d > 0 && (result.RequeueAfter == 0 || d < result.RequeueAfter) { result.RequeueAfter = d } } - fail := func(l gatewayv1.Listener, hostname string, err error) { - logger.Error(err, "certificate service step failed", "listener", l.Name, "hostname", hostname) - issues[l.Name] = certificateServiceUnavailableMessage(hostname) - recordCertificateServiceFailure(upstreamGateway, l.Name, certificateServiceReasonStepFailed) - failed = true - } - - if backoff, remaining, cooling := r.certificateServiceInBackoff(upstreamGateway.UID, now); cooling { - for listener, message := range backoff.issues { - issues[listener] = message - } - requeueSooner(remaining) - return result, issues - } for _, l := range upstreamGateway.Spec.Listeners { - hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) - if !wanted || !r.listenerIssuerResolvable(upstreamGateway, l) { + hostname, ok := r.listenerUsesCertificateService(l, claimedHostnames) + if !ok || !r.listenerIssuerResolvable(upstreamGateway, l) { continue } - - legacyName := listenerCertificateName(upstreamGateway.Name, l.Name) certName := tlsCertificateName(upstreamGateway.Name, l.Name) - secretName := listenerCertificateSecretName(upstreamGateway.Name, l.Name) - - legacy, err := r.ownedLegacyCertificate(ctx, downstreamClient, upstreamGateway, downstreamGateway, legacyName) - if err != nil { - fail(l, hostname, err) - continue - } - if legacy != nil { - if holds, recheck := legacyCertificateHolds(legacy, now); holds { - legacyKeep.Insert(legacyName) - requeueSooner(min(recheck, legacyRecheckMax)) - continue - } - logger.Info("legacy certificate nearing renewal, switching hostname to the certificate service", "certificate", legacyName, "hostname", hostname) - if err := r.retireLegacyCertificate(ctx, downstreamStrategy, upstreamGateway, legacy, secretName); err != nil { - fail(l, hostname, err) - continue - } - } - desiredCerts.Insert(certName) + requeueSooner(certificateServiceRecheck) - cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) - if err != nil { - if errors.Is(err, errTLSCertificateNotOwned) { - recordCertificateServiceFailure(upstreamGateway, l.Name, certificateServiceReasonNotOwned) - } - fail(l, hostname, err) - continue - } - if state != tlsCertificateSettled { - issues[l.Name] = certificateBeingReplacedMessage(hostname) - failed = true + key := certificateServiceKey{gateway: upstreamGateway.UID, listener: l.Name} + if backoff, remaining, cooling := r.certificateServiceInBackoff(key, now); cooling { + issues[l.Name] = backoff.issue.message + failing[l.Name] = backoff.issue.reason + requeueSooner(remaining) continue } - solvers, rejected, err := r.serveTLSCertificateChallenges(ctx, downstreamClient, downstreamGateway, cert, hostname) + mirrored, issue, err := r.ensureListenerTLSCertificate(ctx, upstreamClient, upstreamGateway, downstreamGateway, downstreamStrategy, l.Name, certName, hostname, now) if err != nil { - fail(l, hostname, err) - continue - } - liveSolvers.Insert(solvers...) - if rejected != "" { - issues[l.Name] = rejected - recordCertificateServiceFailure(upstreamGateway, l.Name, certificateServiceReasonTokenRefused) + logger.Error(err, "certificate service step failed", "listener", l.Name, "hostname", hostname) + if issue == nil { + issue = &certificateServiceIssue{reason: certificateServiceReasonStepFailed, message: certificateServiceUnavailableMessage(hostname)} + } } - - mirrored, refusal, err := r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName, hostname, now) - if err != nil { - fail(l, hostname, err) + if issue != nil { + issues[l.Name] = issue.message + failing[l.Name] = issue.reason + recordCertificateServiceFailure(upstreamGateway, l.Name, issue.reason) + requeueSooner(r.certificateServiceRequeue(key, true, now, *issue)) continue } - if refusal.reason != "" { - issues[l.Name] = refusal.message - recordCertificateServiceFailure(upstreamGateway, l.Name, refusal.reason) + r.certificateServiceRequeue(key, false, now, certificateServiceIssue{}) + if state, ok := r.certificateServiceStates.Load(key); ok { + failing[l.Name] = state.(string) } if mirrored { requeueSooner(tlsCertificateMirrorAdmitDelay) } } - if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, desiredCerts); err != nil { + cleanupKey := certificateServiceKey{gateway: upstreamGateway.UID} + if _, remaining, cooling := r.certificateServiceInBackoff(cleanupKey, now); cooling { + requeueSooner(remaining) + } else if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, desiredCerts); err != nil { logger.Error(err, "failed to clean up TLSCertificates") - failed = true - } - if err := r.deleteStaleLegacyCertificates(ctx, downstreamClient, upstreamGateway, downstreamGateway, legacyKeep); err != nil { - logger.Error(err, "failed to clean up legacy Certificates") - failed = true - } - if err := r.deleteStaleTLSCertificateSolvers(ctx, downstreamClient, downstreamGateway, liveSolvers); err != nil { - logger.Error(err, "failed to clean up HTTP-01 solvers") - failed = true + requeueSooner(r.certificateServiceRequeue(cleanupKey, true, now, certificateServiceIssue{})) + } else { + r.certificateServiceRequeue(cleanupKey, false, now, certificateServiceIssue{}) } - requeueSooner(r.certificateServiceRequeue(upstreamGateway.UID, failed, now, issues)) r.forgetRemovedListeners(upstreamGateway) + publishCertificateServiceFailing(upstreamGateway, failing) return result, issues } +// publishCertificateServiceFailing replaces the gateway's failing-listener +// series with the listeners failing now, so one that recovered or left the +// service stops reporting. +func publishCertificateServiceFailing(upstreamGateway *gatewayv1.Gateway, failing map[gatewayv1.SectionName]string) { + clearCertificateServiceFailing(upstreamGateway) + for listener, reason := range failing { + certificateServiceListenerFailing.WithLabelValues(upstreamGateway.Namespace, upstreamGateway.Name, string(listener), reason).Set(1) + } +} + +func clearCertificateServiceFailing(upstreamGateway *gatewayv1.Gateway) { + certificateServiceListenerFailing.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) +} + +// ensureListenerTLSCertificate does one listener's pass: request or confirm +// its TLSCertificate, then mirror what the service issued. An issue says what +// the customer is told; an error alone is a transient step failure. +func (r *GatewayReconciler) ensureListenerTLSCertificate( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + downstreamStrategy downstreamclient.ResourceStrategy, + listenerName gatewayv1.SectionName, + certName string, + hostname string, + now time.Time, +) (bool, *certificateServiceIssue, error) { + secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) + + cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) + switch { + case errors.Is(err, errTLSCertificateNotOwned): + return false, &certificateServiceIssue{reason: certificateServiceReasonNotOwned, message: certificateRequestNotOwnedMessage(hostname)}, err + case err != nil: + if detail, refused := requestRefusal(err); refused { + return false, &certificateServiceIssue{reason: certificateServiceReasonRefused, message: certificateRequestRefusedMessage(hostname, detail)}, err + } + return false, nil, err + case state != tlsCertificateSettled: + return false, &certificateServiceIssue{reason: certificateServiceReasonStepFailed, message: certificateBeingReplacedMessage(hostname)}, nil + } + + return r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName, hostname, now) +} + +// requestRefusal reports whether the API refused the request itself rather +// than failing to process it, and the reason it gave. +func requestRefusal(err error) (string, bool) { + if !apierrors.IsInvalid(err) && !apierrors.IsForbidden(err) && !apierrors.IsBadRequest(err) { + return "", false + } + var status apierrors.APIStatus + if errors.As(err, &status) && status.Status().Message != "" { + return status.Status().Message, true + } + return err.Error(), true +} + // forgetRemovedListeners drops the failure series and tracker entries of // listeners the gateway no longer has, so a removed hostname stops reporting. func (r *GatewayReconciler) forgetRemovedListeners(upstreamGateway *gatewayv1.Gateway) { @@ -360,27 +414,35 @@ func (r *GatewayReconciler) forgetRemovedListeners(upstreamGateway *gatewayv1.Ga certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{ jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name, metricLabelListener: string(listener), }) - if rejected, ok := r.certificateServiceRejections.Load(upstreamGateway.UID); ok { - delete(rejected.(map[gatewayv1.SectionName]bool), listener) - } - if backoff, ok := r.certificateServiceFailures.Load(upstreamGateway.UID); ok { - delete(backoff.(certificateServiceBackoff).issues, listener) - } + key := certificateServiceKey{gateway: upstreamGateway.UID, listener: listener} + r.certificateServiceStates.Delete(key) + r.certificateServiceFailures.Delete(key) } } r.certificateServiceListeners.Store(upstreamGateway.UID, current) } -var errTLSCertificateNotOwned = errors.New("TLSCertificate exists but is not controlled by this Gateway") - -// mirrorRefusal says why issued material was not taken, for the customer and -// for the failure counter. -type mirrorRefusal struct { - reason string - message string +// forgetGateway drops everything the certificate service step remembers about +// a gateway that is gone. +func (r *GatewayReconciler) forgetGateway(upstreamGateway *gatewayv1.Gateway) { + r.certificateServiceStates.Range(func(k, _ any) bool { + if k.(certificateServiceKey).gateway == upstreamGateway.UID { + r.certificateServiceStates.Delete(k) + } + return true + }) + r.certificateServiceFailures.Range(func(k, _ any) bool { + if k.(certificateServiceKey).gateway == upstreamGateway.UID { + r.certificateServiceFailures.Delete(k) + } + return true + }) + r.certificateServiceListeners.Delete(upstreamGateway.UID) + certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) + clearCertificateServiceFailing(upstreamGateway) } -func (m mirrorRefusal) String() string { return m.reason } +var errTLSCertificateNotOwned = errors.New("TLSCertificate exists but is not controlled by this Gateway") type tlsCertificateState int @@ -403,7 +465,7 @@ func (r *GatewayReconciler) ensureTLSCertificate( desiredSpec := certificatesv1alpha1.TLSCertificateSpec{ DNSNames: []certificatesv1alpha1.DNSName{certificatesv1alpha1.DNSName(hostname)}, - Issuance: certificatesv1alpha1.IssuanceModeAuto, + Issuance: certificatesv1alpha1.IssuanceModeDNS01, SecretName: secretName, } @@ -458,63 +520,13 @@ func (r *GatewayReconciler) ensureTLSCertificate( return cert, tlsCertificateSettled, nil } -// serveTLSCertificateChallenges publishes every pending HTTP-01 challenge the -// service reports for the hostname this listener claimed and returns the solver -// names for every such challenge still in status, so a challenge that has moved -// on to Valid keeps its answer until the service drops it. A challenge for any -// other name, or with a token that is not an ACME token, is not ours to answer, -// whatever status says. -func (r *GatewayReconciler) serveTLSCertificateChallenges( - ctx context.Context, - downstreamClient client.Client, - downstreamGateway *gatewayv1.Gateway, - cert *certificatesv1alpha1.TLSCertificate, - hostname string, -) (live []string, rejected string, err error) { - logger := log.FromContext(ctx) - - for _, challenge := range cert.Status.Challenges { - if challenge.Type != certificatesv1alpha1.ChallengeTypeHTTP01 { - continue - } - if challenge.DNSName != hostname { - logger.Info("ignoring HTTP-01 challenge for a hostname this listener did not claim", - "tlscertificate", cert.Name, "claimed", hostname, "challenge_dns_name", challenge.DNSName) - continue - } - if !acmeTokenPattern.MatchString(challenge.Token) { - logger.Info("ignoring HTTP-01 challenge whose token is not an ACME token", "tlscertificate", cert.Name, "hostname", hostname) - rejected = certificateServiceUnavailableMessage(hostname) - continue - } - name := tlsCertificateSolverName(cert.Name, challenge.Token) - live = append(live, name) - - if challenge.State != certificatesv1alpha1.ChallengeStatePending { - continue - } - - err := ensureHTTP01SolverRoutes(ctx, downstreamClient, downstreamClient.Scheme(), downstreamGateway, downstreamGateway, http01SolverRoute{ - name: name, - token: challenge.Token, - key: challenge.Key, - hostnames: []gatewayv1.Hostname{gatewayv1.Hostname(hostname)}, - labels: map[string]string{tlsCertificateSolverLabel: cert.Name}, - }) - if err != nil { - return nil, "", fmt.Errorf("failed to serve HTTP-01 challenge for TLSCertificate %s: %w", cert.Name, err) - } - } - - return live, rejected, nil -} - // mirrorTLSCertificateSecret copies the service-side issued Secret, read with // the operator's own credentials from the configured service namespace, into // the downstream gateway namespace under the listener's secret name, stamped // with the upstream-owner labels the federation policy selects. The material is -// parsed, matched, checked against the hostname and its expiry before it may -// replace what is serving; the project-namespace copy is never read. +// parsed, matched, checked against the hostname, its expiry and, when +// configured, the trusted roots before it may replace what is serving; the +// project-namespace copy is never read. func (r *GatewayReconciler) mirrorTLSCertificateSecret( ctx context.Context, downstreamStrategy downstreamclient.ResourceStrategy, @@ -524,24 +536,28 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( secretName string, hostname string, now time.Time, -) (mirrored bool, refusal mirrorRefusal, err error) { +) (bool, *certificateServiceIssue, error) { logger := log.FromContext(ctx) if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { - return false, mirrorRefusal{}, nil + return false, nil, nil } ref := cert.Status.ServiceSecretRef if ref == nil || ref.Name == "" { logger.Info("TLSCertificate is Ready without a service-side Secret reference", "tlscertificate", cert.Name) - return false, mirrorRefusal{}, nil + return false, nil, nil } if ref.Namespace != r.Config.Gateway.CertificateService.SecretNamespace { logger.Info("refusing service-side Secret outside the certificate service namespace", "tlscertificate", cert.Name, "namespace", ref.Namespace, "expected", r.Config.Gateway.CertificateService.SecretNamespace) - return false, mirrorRefusal{reason: certificateServiceReasonNamespaceRefused, message: certificateServiceUnavailableMessage(hostname)}, nil + return false, &certificateServiceIssue{reason: certificateServiceReasonNamespaceRefused, message: certificateMaterialRefusedMessage(hostname)}, nil } if r.CertificateServiceReader == nil { - return false, mirrorRefusal{}, fmt.Errorf("certificate service enabled without a client for its cluster") + return false, nil, fmt.Errorf("certificate service enabled without a client for its cluster") + } + roots, err := r.certificateServiceRoots() + if err != nil { + return false, nil, fmt.Errorf("failed to load trusted roots: %w", err) } downstreamClient := downstreamStrategy.GetClient() @@ -552,10 +568,10 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( }, } if err := downstreamClient.Get(ctx, client.ObjectKeyFromObject(mirror), mirror); client.IgnoreNotFound(err) != nil { - return false, mirrorRefusal{}, fmt.Errorf("failed to get Secret %s: %w", secretName, err) + return false, nil, fmt.Errorf("failed to get Secret %s: %w", secretName, err) } if mirrorHoldsIssuance(mirror, cert) { - return false, mirrorRefusal{}, nil + return false, nil, nil } var source corev1.Secret @@ -563,15 +579,22 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( if err := r.CertificateServiceReader.Get(ctx, sourceKey, &source); err != nil { if apierrors.IsNotFound(err) { logger.Info("TLSCertificate is Ready but its service-side Secret is not readable yet", "tlscertificate", cert.Name, "secret", sourceKey) - return false, mirrorRefusal{}, nil + return false, nil, nil } - return false, mirrorRefusal{}, fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) + return false, nil, fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) } if err := validateIssuedMaterial(source.Data["tls.crt"], source.Data["tls.key"], hostname, now); err != nil { logger.Info("refusing service-side Secret that does not hold a usable certificate for the hostname", "tlscertificate", cert.Name, "secret", sourceKey, "reason", err.Error()) - return false, mirrorRefusal{reason: certificateServiceReasonMaterialRefused, message: certificateServiceUnavailableMessage(hostname)}, nil + return false, &certificateServiceIssue{reason: certificateServiceReasonMaterialRefused, message: certificateMaterialRefusedMessage(hostname)}, nil + } + if roots != nil { + if err := verifyIssuedChain(source.Data["tls.crt"], hostname, roots, now); err != nil { + logger.Info("refusing service-side Secret whose chain is not trusted", + "tlscertificate", cert.Name, "secret", sourceKey, "reason", err.Error()) + return false, &certificateServiceIssue{reason: certificateServiceReasonUntrustedChain, message: certificateMaterialRefusedMessage(hostname)}, nil + } } op, err := controllerutil.CreateOrUpdate(ctx, downstreamClient, mirror, func() error { @@ -581,7 +604,6 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, mirror); err != nil { return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) } - mirror.Labels[tlsCertificateManagedLabel] = labelValueTrue mirror.Data = map[string][]byte{ "tls.crt": source.Data["tls.crt"], "tls.key": source.Data["tls.key"], @@ -592,20 +614,32 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( return nil }) if err != nil { - return false, mirrorRefusal{}, fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) + return false, nil, fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) } if op != controllerutil.OperationResultNone { logger.Info("issued Secret mirrored downstream", "secret", secretName, "operation", op) } - return op != controllerutil.OperationResultNone, mirrorRefusal{}, nil + return op != controllerutil.OperationResultNone, nil, nil +} + +// certificateServiceRoots returns the roots an issued chain must verify +// against, or nil when chain verification is off. +func (r *GatewayReconciler) certificateServiceRoots() (*x509.CertPool, error) { + if !r.Config.Gateway.CertificateService.VerifyChain { + return nil, nil + } + if r.CertificateServiceRoots != nil { + return r.CertificateServiceRoots, nil + } + return x509.SystemCertPool() } // mirrorHoldsIssuance reports whether the downstream Secret already carries the // issuance the TLSCertificate describes, so the service cluster is not read // again for it. func mirrorHoldsIssuance(mirror *corev1.Secret, cert *certificatesv1alpha1.TLSCertificate) bool { - if mirror.CreationTimestamp.IsZero() || mirror.Labels[tlsCertificateManagedLabel] != labelValueTrue || cert.Status.NotAfter == nil { + if mirror.CreationTimestamp.IsZero() || cert.Status.NotAfter == nil { return false } leaf, err := parseLeafCertificate(mirror.Data["tls.crt"], mirror.Data["tls.key"]) @@ -648,143 +682,40 @@ func validateIssuedMaterial(certPEM, keyPEM []byte, hostname string, now time.Ti return nil } -// ownedLegacyCertificate returns the listener's cert-manager Certificate from -// the downstream cluster when one exists and belongs to this gateway. -func (r *GatewayReconciler) ownedLegacyCertificate( - ctx context.Context, - downstreamClient client.Client, - upstreamGateway *gatewayv1.Gateway, - downstreamGateway *gatewayv1.Gateway, - certName string, -) (*cmv1.Certificate, error) { - var legacy cmv1.Certificate - if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamGateway.Namespace, Name: certName}, &legacy); err != nil { - if apierrors.IsNotFound(err) { - return nil, nil +// verifyIssuedChain checks that the PEM chain, leaf first, builds to one of +// the trusted roots for server authentication on the hostname. +func verifyIssuedChain(certPEM []byte, hostname string, roots *x509.CertPool, now time.Time) error { + var chain []*x509.Certificate + for rest := certPEM; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break } - return nil, fmt.Errorf("failed to get Certificate %s: %w", certName, err) - } - if !certificateOwnedByGateway(&legacy, upstreamGateway, downstreamGateway) { - return nil, nil - } - return &legacy, nil -} - -// legacyHoldWhileIssuing bounds how long a cert-manager renewal in flight can -// defer the hand-over; past it the renewal is taken as stuck. -const legacyHoldWhileIssuing = 24 * time.Hour - -// legacyHandOverFloor is the remaining lifetime below which the hand-over goes -// ahead whatever cert-manager is doing, so the service has time to issue. -const legacyHandOverFloor = 7 * 24 * time.Hour - -// legacyCertificateHolds reports whether a cert-manager Certificate should keep -// its hostname for now: it is serving and not yet within the switch lead of its -// renewal time, or cert-manager is mid-renewal and must not be interrupted. A -// renewal that has failed, has been in flight for too long, or has too little -// lifetime left to wait for does not hold. The returned duration is when to -// look again. -func legacyCertificateHolds(cert *cmv1.Certificate, now time.Time) (bool, time.Duration) { - if !certIsServing(cert, now) { - return false, 0 - } - if cert.Status.NotAfter != nil && cert.Status.NotAfter.Sub(now) < legacyHandOverFloor { - return false, 0 - } - for _, c := range cert.Status.Conditions { - if c.Type != cmv1.CertificateConditionIssuing || c.Status != cmmeta.ConditionTrue { + if block.Type != "CERTIFICATE" { continue } - if cert.Status.LastFailureTime != nil { - return false, 0 - } - if !c.LastTransitionTime.IsZero() && now.Sub(c.LastTransitionTime.Time) > legacyHoldWhileIssuing { - return false, 0 - } - return true, 10 * time.Minute - } - if cert.Status.RenewalTime == nil { - return false, 0 - } - switchAt := cert.Status.RenewalTime.Add(-tlsCertificateSwitchLead) - if now.Before(switchAt) { - return true, switchAt.Sub(now) - } - return false, 0 -} - -// retireLegacyCertificate hands the listener over to the service: the Secret -// gains the gateway's own owner so cert-manager's owner reference no longer -// takes it along and the hand-over label the cert-manager path honours on -// rollback, then the Certificate goes so cert-manager does not renew a -// hostname the service is about to issue for. -func (r *GatewayReconciler) retireLegacyCertificate( - ctx context.Context, - downstreamStrategy downstreamclient.ResourceStrategy, - upstreamGateway *gatewayv1.Gateway, - legacy *cmv1.Certificate, - secretName string, -) error { - downstreamClient := downstreamStrategy.GetClient() - - var secret corev1.Secret - err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: legacy.Namespace, Name: secretName}, &secret) - switch { - case apierrors.IsNotFound(err): - case err != nil: - return fmt.Errorf("failed to get Secret %s: %w", secretName, err) - default: - if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, &secret); err != nil { - return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) - } - secret.Labels[tlsCertificateManagedLabel] = labelValueTrue - if err := downstreamClient.Update(ctx, &secret); err != nil { - return fmt.Errorf("failed to keep Secret %s past its Certificate: %w", secretName, err) + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate chain: %w", err) } + chain = append(chain, c) } - - if err := downstreamClient.Delete(ctx, legacy, client.Preconditions{UID: &legacy.UID}); client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to delete legacy Certificate %s: %w", legacy.Name, err) - } - return nil -} - -// deleteStaleLegacyCertificates removes the gateway's cert-manager Certificates -// for listeners that no longer carry one, mirroring what the cert-manager path -// does for removed hostnames. -func (r *GatewayReconciler) deleteStaleLegacyCertificates( - ctx context.Context, - downstreamClient client.Client, - upstreamGateway *gatewayv1.Gateway, - downstreamGateway *gatewayv1.Gateway, - legacyKeep sets.Set[string], -) error { - logger := log.FromContext(ctx) - - var certList cmv1.CertificateList - if err := downstreamClient.List(ctx, &certList, client.InNamespace(downstreamGateway.Namespace)); err != nil { - return fmt.Errorf("failed to list Certificates: %w", err) + if len(chain) == 0 { + return fmt.Errorf("secret holds no certificate") } - - for i := range certList.Items { - cert := &certList.Items[i] - if legacyKeep.Has(cert.Name) || !certificateOwnedByGateway(cert, upstreamGateway, downstreamGateway) { - continue - } - logger.Info("deleting stale Certificate", "certificate", cert.Name) - if err := downstreamClient.Delete(ctx, cert); client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to delete stale Certificate %s: %w", cert.Name, err) - } + intermediates := x509.NewCertPool() + for _, c := range chain[1:] { + intermediates.AddCert(c) } - - return nil -} - -func certificateOwnedByGateway(cert *cmv1.Certificate, upstreamGateway, downstreamGateway *gatewayv1.Gateway) bool { - ownedByStrategy := cert.Labels[downstreamclient.UpstreamOwnerKindLabel] == KindGateway && - cert.Labels[downstreamclient.UpstreamOwnerNameLabel] == upstreamGateway.Name && - cert.Labels[downstreamclient.UpstreamOwnerNamespaceLabel] == upstreamGateway.Namespace - return ownedByStrategy || metav1.IsControlledBy(cert, downstreamGateway) + _, err := chain[0].Verify(x509.VerifyOptions{ + DNSName: hostname, + Roots: roots, + Intermediates: intermediates, + CurrentTime: now, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + return err } // deleteStaleTLSCertificates removes TLSCertificates this gateway controls that @@ -817,178 +748,85 @@ func (r *GatewayReconciler) deleteStaleTLSCertificates( return nil } -func (r *GatewayReconciler) deleteStaleTLSCertificateSolvers( - ctx context.Context, - downstreamClient client.Client, - downstreamGateway *gatewayv1.Gateway, - liveSolvers sets.Set[string], -) error { - logger := log.FromContext(ctx) - - var routes gatewayv1.HTTPRouteList - if err := downstreamClient.List(ctx, &routes, - client.InNamespace(downstreamGateway.Namespace), - client.HasLabels{tlsCertificateSolverLabel}, - ); err != nil { - return fmt.Errorf("failed to list TLSCertificate solver routes: %w", err) - } - - for i := range routes.Items { - route := &routes.Items[i] - if liveSolvers.Has(route.Name) || !metav1.IsControlledBy(route, downstreamGateway) { - continue - } - logger.Info("removing finished HTTP-01 solver", "solver", route.Name) - if err := deleteHTTP01SolverRoutes(ctx, downstreamClient, route.Namespace, route.Name); err != nil { - return err - } - } - - return nil -} - -// cleanupCertificateServiceLeftovers runs on the cert-manager path for a -// gateway that was once on the service: it drops the solver routes and the -// TLSCertificates this gateway controls, so the service stops renewing names -// cert-manager has taken back. Nothing here fails the reconcile, and the -// project control plane is only asked about TLSCertificates when a handed-over -// Secret shows the gateway was ever on the service. -func (r *GatewayReconciler) cleanupCertificateServiceLeftovers( - ctx context.Context, - upstreamClient client.Client, - upstreamGateway *gatewayv1.Gateway, - downstreamGateway *gatewayv1.Gateway, - downstreamClient client.Client, -) { - logger := log.FromContext(ctx) - - if err := r.deleteStaleTLSCertificateSolvers(ctx, downstreamClient, downstreamGateway, sets.New[string]()); err != nil { - logger.Error(err, "failed to remove certificate service solvers after rollback") - } - - var handedOver corev1.SecretList - if err := downstreamClient.List(ctx, &handedOver, - client.InNamespace(downstreamGateway.Namespace), - client.MatchingLabels{ - tlsCertificateManagedLabel: labelValueTrue, - downstreamclient.UpstreamOwnerKindLabel: KindGateway, - downstreamclient.UpstreamOwnerNameLabel: upstreamGateway.Name, - downstreamclient.UpstreamOwnerNamespaceLabel: upstreamGateway.Namespace, - }, - ); err != nil { - logger.Error(err, "failed to list handed-over Secrets after rollback") - return - } - if len(handedOver.Items) == 0 { - return - } - - if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, sets.New[string]()); err != nil { - logger.Error(err, "failed to remove TLSCertificates after rollback") - } - - for i := range handedOver.Items { - secret := &handedOver.Items[i] - var legacy cmv1.Certificate - if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: secret.Namespace, Name: secret.Name}, &legacy); err != nil || !certIsReady(&legacy) { - continue - } - delete(secret.Labels, tlsCertificateManagedLabel) - if err := downstreamClient.Update(ctx, secret); err != nil { - logger.Error(err, "failed to clear hand-over marker after cert-manager retook the Secret", "secret", secret.Name) - } - } -} - -// handedOverSecretHealth reports whether the listener's downstream Secret was -// handed to or written by the certificate service and still holds a usable -// certificate for the hostname. renewalDue says the certificate is in the last -// third of its life, where cert-manager would renew it anyway. -func handedOverSecretHealth( +// servingSecretHealth is listenerSecretHealth plus a check that the leaf +// actually covers the hostname, so a listener whose hostname changed cannot +// keep serving the previous name's certificate. It also returns the served +// leaf when there is one. +func servingSecretHealth( ctx context.Context, downstreamClient client.Client, downstreamNamespace string, secretName string, hostname string, now time.Time, -) (status listenerCertStatus, handedOver bool, renewalDue bool) { +) (listenerCertStatus, *x509.Certificate) { + status := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + if !status.healthy { + return status, nil + } var secret corev1.Secret if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { - return listenerCertStatus{}, false, false + return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), pending: true, secretName: secretName}, nil } - if secret.Labels[tlsCertificateManagedLabel] != labelValueTrue { - return listenerCertStatus{}, false, false + if err := validateIssuedMaterial(secret.Data["tls.crt"], secret.Data["tls.key"], hostname, now); err != nil { + return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), secretName: secretName}, nil } leaf, err := parseLeafCertificate(secret.Data["tls.crt"], secret.Data["tls.key"]) - if err != nil || leaf.VerifyHostname(hostname) != nil || now.Before(leaf.NotBefore) || !leaf.NotAfter.After(now.Add(listenerCertExpiryMargin)) { - return listenerCertStatus{}, true, true + if err != nil { + return status, nil } - lifetime := leaf.NotAfter.Sub(leaf.NotBefore) - renewalDue = !now.Before(leaf.NotAfter.Add(-lifetime / 3)) - return listenerCertStatus{healthy: true, secretName: secretName, notAfter: &metav1.Time{Time: leaf.NotAfter}}, true, renewalDue + return status, leaf } -// evaluateListenerTLSCertificateHealth is evaluateListenerCertHealth for the -// certificate-service path: conditions come from the upstream TLSCertificate, -// the served material from the mirrored downstream Secret. -func (r *GatewayReconciler) evaluateListenerTLSCertificateHealth( - ctx context.Context, - upstreamClient client.Client, - downstreamClient client.Client, - downstreamNamespace string, - upstreamGateway *gatewayv1.Gateway, - claimedHostnames []string, -) map[gatewayv1.SectionName]listenerCertStatus { - logger := log.FromContext(ctx) - health := make(map[gatewayv1.SectionName]listenerCertStatus) - now := time.Now() - - clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) - - for _, l := range upstreamGateway.Spec.Listeners { - hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) - if !wanted { - continue - } - - status := r.listenerTLSCertificateHealth(ctx, upstreamClient, downstreamClient, downstreamNamespace, upstreamGateway, l.Name, hostname, now) - health[l.Name] = status - recordListenerCertHealth(logger, upstreamGateway, l.Name, hostname, status) +// renewalOverdue reports whether a served certificate is well past the point +// it should have been replaced, whatever the TLSCertificate's status says. +func renewalOverdue(leaf *x509.Certificate, now time.Time) bool { + if leaf == nil { + return false } - - return health + lifetime := leaf.NotAfter.Sub(leaf.NotBefore) + return leaf.NotAfter.Sub(now) < lifetime/renewalOverdueDivisor } -// servingSecretHealth is listenerSecretHealth plus a check that the leaf -// actually covers the hostname, so a listener whose hostname changed cannot -// keep serving the previous name's certificate. -func servingSecretHealth( - ctx context.Context, - downstreamClient client.Client, - downstreamNamespace string, - secretName string, - hostname string, - now time.Time, -) listenerCertStatus { - status := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) - if !status.healthy { - return status +// tlsCertificateFailure says why a TLSCertificate is not producing a usable +// certificate, as a counter reason and the service's own explanation, or "" +// while it is healthy, still within its grace, or waiting on DNS records the +// customer has to publish. +func tlsCertificateFailure(cert *certificatesv1alpha1.TLSCertificate, now time.Time) (string, string) { + if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + return certificateServiceReasonRejected, accepted.Message } - var secret corev1.Secret - if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { - return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), pending: true, secretName: secretName} + if !cert.DeletionTimestamp.IsZero() { + return "", "" } - if err := validateIssuedMaterial(secret.Data["tls.crt"], secret.Data["tls.key"], hostname, now); err != nil { - return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), secretName: secretName} + if issuing := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionIssuing); issuing != nil && + issuing.Status == metav1.ConditionFalse && issuing.Reason == "IssuanceFailed" { + return certificateServiceReasonIssuanceFailed, issuing.Message } - return status + ready := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) + if ready != nil && ready.Status == metav1.ConditionTrue { + return "", "" + } + if delegation := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionDNSDelegationReady); delegation != nil && delegation.Status == metav1.ConditionFalse { + return "", "" + } + since := cert.CreationTimestamp.Time + detail := "" + if ready != nil { + since = ready.LastTransitionTime.Time + detail = ready.Message + } + if since.IsZero() || now.Sub(since) < tlsCertificateIssueGrace { + return "", "" + } + return certificateServiceReasonNotReady, detail } // listenerTLSCertificateHealth keeps a listener serving whenever its downstream -// Secret holds a usable certificate, whatever the TLSCertificate is doing: a -// renewal in flight or a first issuance replacing a legacy certificate must -// not take HTTPS down. Only when nothing usable is downstream do the -// TLSCertificate's conditions decide what the customer is told. +// Secret holds a usable certificate, whatever the TLSCertificate is doing. A +// replacement that is failing, stalled or overdue is reported as a blocked +// renewal while the listener serves, and as blocked issuance when nothing does, +// and counted once per transition. func (r *GatewayReconciler) listenerTLSCertificateHealth( ctx context.Context, upstreamClient client.Client, @@ -1004,14 +842,19 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( certName := tlsCertificateName(upstreamGateway.Name, listenerName) secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) - secretStatus := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + secretStatus, leaf := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) var cert certificatesv1alpha1.TLSCertificate if err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamGateway.Namespace, Name: certName}, &cert); err != nil { if !apierrors.IsNotFound(err) { logger.Error(err, "failed to get listener TLSCertificate", "tlscertificate", certName) } - r.recordCertificateServiceRejection(upstreamGateway, listenerName, false) + reason := "" + if secretStatus.healthy && renewalOverdue(leaf, now) { + reason = certificateServiceReasonRenewalOverdue + secretStatus.renewalBlocked = certificateRenewalOverdueMessage(hostname, leaf.NotAfter) + } + r.recordCertificateServiceState(upstreamGateway, listenerName, reason) if secretStatus.healthy { return secretStatus } @@ -1023,26 +866,40 @@ func (r *GatewayReconciler) listenerTLSCertificateHealth( } } - rejected := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted) - if rejected != nil && rejected.Status != metav1.ConditionFalse { - rejected = nil + reason, detail := tlsCertificateFailure(&cert, now) + if reason == "" && secretStatus.healthy && renewalOverdue(leaf, now) { + reason = certificateServiceReasonRenewalOverdue } - - r.recordCertificateServiceRejection(upstreamGateway, listenerName, rejected != nil) + r.recordCertificateServiceState(upstreamGateway, listenerName, reason) if secretStatus.healthy { - if rejected != nil { - secretStatus.renewalBlocked = tlsCertificateRejectedMessage(hostname, rejected.Message) + switch reason { + case "": + case certificateServiceReasonRenewalOverdue: + secretStatus.renewalBlocked = certificateRenewalOverdueMessage(hostname, leaf.NotAfter) + case certificateServiceReasonRejected: + secretStatus.renewalBlocked = tlsCertificateRejectedMessage(hostname, detail) + default: + secretStatus.renewalBlocked = certificateNotIssuedMessage(hostname, detail) } return secretStatus } - if accepted := rejected; accepted != nil { + switch reason { + case certificateServiceReasonRejected: return listenerCertStatus{ reason: gatewayv1.ListenerReasonInvalidCertificateRef, - message: tlsCertificateRejectedMessage(hostname, accepted.Message), + message: tlsCertificateRejectedMessage(hostname, detail), secretName: secretName, } + case certificateServiceReasonIssuanceFailed, certificateServiceReasonNotReady: + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certificateNotIssuedMessage(hostname, detail), + pending: true, + issuanceBlocked: true, + secretName: secretName, + } } if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go index 03e7d3e3..9eaba195 100644 --- a/internal/controller/gateway_certificate_service_test.go +++ b/internal/controller/gateway_certificate_service_test.go @@ -4,12 +4,19 @@ package controller import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" "strings" + "sync/atomic" "testing" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -21,8 +28,10 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -53,43 +62,57 @@ func newCertificateServiceTestScheme(t *testing.T) *runtime.Scheme { return testScheme } -func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { - testScheme := newCertificateServiceTestScheme(t) - logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + pool *x509.CertPool +} - upstreamNamespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}, +func newTestCA(t *testing.T) testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test root"}, + NotBefore: time.Now().Add(-24 * time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, } - downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) - - const ( - gatewayName = "test-gw" - listenerName = gatewayv1.SectionName("https-hostname-0") - hostname = "custom.example.com" - serviceNS = "certificates-system" - serviceSecret = "svc-secret" - upstreamCluster = "test" - ) - certName := tlsCertificateName(gatewayName, listenerName) - legacyName := listenerCertificateName(gatewayName, listenerName) - secretName := listenerCertificateSecretName(gatewayName, listenerName) + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + pool := x509.NewCertPool() + pool.AddCert(cert) + return testCA{cert: cert, key: key, pool: pool} +} - baseConfig := config.GatewayConfig{ - DownstreamGatewayClassName: "test-suite", - DownstreamHostnameAccountingNamespace: "default", - TargetDomain: "test-suite.com", - DefaultListenerTLSSecretName: "wildcard-test-suite-tls", - IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, - ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ - gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, - }, - ClusterIssuerMap: map[string]string{autoIssuerSentinel: "datum-gateway-http"}, - CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: serviceNS}, +func (ca testCA) issue(t *testing.T, hostname string, notBefore, notAfter time.Time) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: hostname}, + DNSNames: []string{hostname}, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, } - testCfg := config.NetworkServicesOperator{Gateway: baseConfig} + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, &key.PublicKey, ca.key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) +} - customListener := gatewayv1.Listener{ - Name: listenerName, +func tlsListener(name gatewayv1.SectionName, hostname string) gatewayv1.Listener { + return gatewayv1.Listener{ + Name: name, Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, Hostname: ptr.To(gatewayv1.Hostname(hostname)), @@ -103,19 +126,60 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, }, } +} - verifiedDomain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { - d.Spec.DomainName = hostname - apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{ - Type: networkingv1alpha.DomainConditionVerified, - Status: metav1.ConditionTrue, - }) - }) +func dnsProvenDomain(namespace string) *networkingv1alpha.Domain { + return &networkingv1alpha.Domain{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "example.com"}, + Spec: networkingv1alpha.DomainSpec{DomainName: "example.com"}, + Status: networkingv1alpha.DomainStatus{Conditions: []metav1.Condition{ + {Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue, Reason: networkingv1alpha.DomainReasonVerified}, + {Type: networkingv1alpha.DomainConditionVerifiedDNS, Status: metav1.ConditionTrue, Reason: networkingv1alpha.DomainReasonVerified}, + }}, + } +} + +func certificateServiceGatewayConfig() config.GatewayConfig { + return config.GatewayConfig{ + DownstreamGatewayClassName: "test-suite", + DownstreamHostnameAccountingNamespace: "default", + TargetDomain: "test-suite.com", + DefaultListenerTLSSecretName: "wildcard-test-suite-tls", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, + }, + ClusterIssuerMap: map[string]string{autoIssuerSentinel: "datum-gateway-http"}, + CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system", VerifyChain: true}, + } +} + +func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + + upstreamNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}, + } + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + + const ( + gatewayName = "test-gw" + listenerName = gatewayv1.SectionName("https-hostname-0") + hostname = "*.shop.example.com" + serviceNS = "certificates-system" + serviceSecret = "svc-secret" + upstreamCluster = "test" + ) + certName := tlsCertificateName(gatewayName, listenerName) + secretName := listenerCertificateSecretName(gatewayName, listenerName) + testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + ca := newTestCA(t) now := time.Now() - serviceCertPEM, serviceKeyPEM := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) - projectCertPEM, projectKeyPEM := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) - legacyCertPEM, legacyKeyPEM := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + serviceCertPEM, serviceKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + projectCertPEM, projectKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + servingCertPEM, servingKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) tlsSecret := func(namespace, name string, certPEM, keyPEM []byte) *corev1.Secret { return &corev1.Secret{ @@ -130,7 +194,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: name, UID: uuid.NewUUID()}, Spec: certificatesv1alpha1.TLSCertificateSpec{ DNSNames: []certificatesv1alpha1.DNSName{hostname}, - Issuance: certificatesv1alpha1.IssuanceModeAuto, + Issuance: certificatesv1alpha1.IssuanceModeDNS01, SecretName: secretName, }, } @@ -152,52 +216,13 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}) } - legacyLabels := map[string]string{ - downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + upstreamCluster, - downstreamclient.UpstreamOwnerGroupLabel: gatewayv1.GroupName, - downstreamclient.UpstreamOwnerKindLabel: KindGateway, - downstreamclient.UpstreamOwnerNameLabel: gatewayName, - downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace.Name, - } - newLegacyCertificate := func(renewalTime time.Time) *cmv1.Certificate { - return &cmv1.Certificate{ - ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: legacyName, UID: uuid.NewUUID(), Labels: legacyLabels}, - Spec: cmv1.CertificateSpec{SecretName: secretName, DNSNames: []string{hostname}}, - Status: cmv1.CertificateStatus{ - NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, - NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, - RenewalTime: &metav1.Time{Time: renewalTime}, - Conditions: []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionTrue}}, - }, - } - } - - downstreamGatewaySeed := func() *gatewayv1.Gateway { - return &gatewayv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: downstreamNamespaceName, - Name: gatewayName, - UID: uuid.NewUUID(), - Labels: legacyLabels, - }, - Spec: gatewayv1.GatewaySpec{GatewayClassName: "test-suite"}, - } - } - - solverObjects := func(owner *gatewayv1.Gateway, token string) []client.Object { - name := tlsCertificateSolverName(certName, token) - labels := map[string]string{http01SolverLabel: labelValueTrue, tlsCertificateSolverLabel: certName} - route := &gatewayv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: name, Labels: labels}} - filter := &envoygatewayv1alpha1.HTTPRouteFilter{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: name, Labels: labels}} - require.NoError(t, controllerutil.SetControllerReference(owner, route, testScheme)) - require.NoError(t, controllerutil.SetControllerReference(owner, filter, testScheme)) - return []client.Object{route, filter} + serving := func() []client.Object { + return []client.Object{tlsSecret(downstreamNamespaceName, secretName, servingCertPEM, servingKeyPEM)} } type env struct { upstream client.Client downstream client.Client - service client.Client result Result reconciler *GatewayReconciler } @@ -205,95 +230,47 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { tests := []struct { name string upstreamObjects func(gw *gatewayv1.Gateway) []client.Object + upstreamCreate func(obj client.Object) error downstreamObjects func() []client.Object serviceObjects []client.Object serviceForbidden bool assert func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) }{ { - name: "requests a TLSCertificate upstream and withholds the listener until it issues", + name: "requests a DNS-01 TLSCertificate upstream and withholds the listener until it issues", assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { ctx := context.Background() var cert certificatesv1alpha1.TLSCertificate require.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) assert.Equal(t, []certificatesv1alpha1.DNSName{hostname}, cert.Spec.DNSNames) - assert.Equal(t, certificatesv1alpha1.IssuanceModeAuto, cert.Spec.Issuance) + assert.Equal(t, certificatesv1alpha1.IssuanceModeDNS01, cert.Spec.Issuance) assert.Equal(t, secretName, cert.Spec.SecretName) assert.True(t, metav1.IsControlledBy(&cert, upstreamGateway), "TLSCertificate should be controlled by the upstream Gateway") - var legacy cmv1.Certificate - err := e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &legacy) - assert.True(t, apierrors.IsNotFound(err), "no cert-manager Certificate should be created downstream") + var legacy cmv1.CertificateList + require.NoError(t, e.downstream.List(ctx, &legacy)) + assert.Empty(t, legacy.Items, "no cert-manager Certificate is created for a wildcard") + + var routes gatewayv1.HTTPRouteList + require.NoError(t, e.downstream.List(ctx, &routes, client.HasLabels{"meta.datumapis.com/http01-solver"})) + assert.Empty(t, routes.Items, "a wildcard is never answered over HTTP") assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener must be withheld until a certificate is issued") assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, gatewayutil.DefaultHTTPSListenerName), "shared wildcard listener is untouched") + assert.Equal(t, time.Minute, e.result.RequeueAfter, "a listener waiting on issuance is looked at again soon") }, }, { - name: "serves pending HTTP-01 challenges only for the hostname the listener claimed", - upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { - return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { - c.Status.Challenges = []certificatesv1alpha1.ACMEChallenge{ - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-aaaaaaaaaaaaaa", Key: "key-a", State: certificatesv1alpha1.ChallengeStatePending}, - {DNSName: "evil.example.com", Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-bbbbbbbbbbbbbb", Key: "key-b", State: certificatesv1alpha1.ChallengeStatePending}, - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeDNS01, Token: "token-cccccccccccccc", Key: "key-c", State: certificatesv1alpha1.ChallengeStatePending}, - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-dddddddddddddd", Key: "key-d", State: certificatesv1alpha1.ChallengeStateInvalid}, - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "bad token/../x", Key: "key-e", State: certificatesv1alpha1.ChallengeStatePending}, - } - })} - }, - assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - ctx := context.Background() - served := tlsCertificateSolverName(certName, "token-aaaaaaaaaaaaaa") - - var route gatewayv1.HTTPRoute - require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: served}, &route)) - assert.Equal(t, labelValueTrue, route.Labels[http01SolverLabel]) - assert.Equal(t, certName, route.Labels[tlsCertificateSolverLabel]) - require.Len(t, route.Spec.Rules, 1) - require.Len(t, route.Spec.Rules[0].Matches, 1) - assert.Equal(t, "/.well-known/acme-challenge/token-aaaaaaaaaaaaaa", ptr.Deref(route.Spec.Rules[0].Matches[0].Path.Value, "")) - assert.Equal(t, gatewayv1.ObjectName(gatewayName), route.Spec.ParentRefs[0].Name) - assert.Equal(t, []gatewayv1.Hostname{hostname}, route.Spec.Hostnames, "solver route is scoped to the claimed hostname") - owner := metav1.GetControllerOf(&route) - require.NotNil(t, owner) - assert.Equal(t, KindGateway, owner.Kind) - - var filter envoygatewayv1alpha1.HTTPRouteFilter - require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: served}, &filter)) - assert.Equal(t, "key-a", ptr.Deref(filter.Spec.DirectResponse.Body.Inline, "")) - - for _, token := range []string{"token-bbbbbbbbbbbbbb", "token-cccccccccccccc", "token-dddddddddddddd", "bad token/../x"} { - err := e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, token)}, &gatewayv1.HTTPRoute{}) - assert.True(t, apierrors.IsNotFound(err), "no solver route for challenge %s", token) - } - assert.NotEqual(t, certificateServiceBackoffBase, e.result.RequeueAfter, "a rejected token is not a failure of the step") - }, - }, - { - name: "removes solver routes once their challenge leaves status", + name: "moves an existing request onto DNS-01", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { - c.Status.Challenges = []certificatesv1alpha1.ACMEChallenge{ - {DNSName: hostname, Type: certificatesv1alpha1.ChallengeTypeHTTP01, Token: "token-keepkeepkeepkeep", Key: "key-keep", State: certificatesv1alpha1.ChallengeStateValid}, - } + c.Spec.Issuance = certificatesv1alpha1.IssuanceModeAuto })} }, - downstreamObjects: func() []client.Object { - gw := downstreamGatewaySeed() - objs := make([]client.Object, 0, 5) - objs = append(objs, gw) - objs = append(objs, solverObjects(gw, "token-keepkeepkeepkeep")...) - objs = append(objs, solverObjects(gw, "token-oldoldoldoldold")...) - return objs - }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - ctx := context.Background() - assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: tlsCertificateSolverName(certName, "token-keepkeepkeepkeep")}, &gatewayv1.HTTPRoute{}), "a Valid challenge keeps its answer until the service drops it") - - old := tlsCertificateSolverName(certName, "token-oldoldoldoldold") - assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: old}, &gatewayv1.HTTPRoute{}))) - assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: old}, &envoygatewayv1alpha1.HTTPRouteFilter{}))) + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) + assert.Equal(t, certificatesv1alpha1.IssuanceModeDNS01, cert.Spec.Issuance) }, }, { @@ -319,10 +296,11 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { assert.Equal(t, tlsCertificateMirrorAdmitDelay, e.result.RequeueAfter, "a fresh mirror asks for a prompt re-evaluation") - health := e.reconciler.evaluateListenerTLSCertificateHealth(ctx, e.upstream, e.downstream, downstreamNamespaceName, upstreamGateway, []string{hostname}) + health := e.reconciler.evaluateListenerCertHealth(ctx, e.upstream, e.downstream, downstreamNamespaceName, upstreamGateway, []string{hostname}) status, gated := health[listenerName] require.True(t, gated) assert.True(t, status.healthy, "listener is admitted on the next pass: %s", status.message) + assert.Empty(t, status.renewalBlocked) assert.Equal(t, secretName, status.secretName) require.NotNil(t, status.notAfter) assert.WithinDuration(t, now.Add(60*24*time.Hour), status.notAfter.Time, time.Minute) @@ -340,86 +318,29 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) - - var found bool - for _, ls := range upstreamGateway.Status.Listeners { - if ls.Name != listenerName { - continue - } - found = true - resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) - require.NotNil(t, resolved) - assert.Equal(t, metav1.ConditionFalse, resolved.Status) - assert.Contains(t, resolved.Message, "names under datum.net are denied") - } - assert.True(t, found, "listener status should be reported") - }, - }, - { - name: "keeps a valid cert-manager certificate that is not due for renewal", - downstreamObjects: func() []client.Object { - return []client.Object{ - newLegacyCertificate(now.Add(30 * 24 * time.Hour)), - tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM), - } - }, - assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - ctx := context.Background() - var certs certificatesv1alpha1.TLSCertificateList - require.NoError(t, e.upstream.List(ctx, &certs, client.InNamespace(upstreamNamespace.Name))) - assert.Empty(t, certs.Items, "no TLSCertificate is requested while the legacy certificate holds") - - assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{}), "legacy Certificate stays") - assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener keeps serving on the legacy certificate") + resolved := listenerCondition(t, upstreamGateway, listenerName, string(gatewayv1.ListenerConditionResolvedRefs)) + assert.Equal(t, metav1.ConditionFalse, resolved.Status) + assert.Contains(t, resolved.Message, "names under datum.net are denied") + assert.NotContains(t, resolved.Message, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRejected), 1.0) }, }, { - name: "switches to the service ahead of cert-manager's renewal time", - downstreamObjects: func() []client.Object { - legacy := newLegacyCertificate(now.Add(24 * time.Hour)) - secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) - require.NoError(t, controllerutil.SetControllerReference(legacy, secret, testScheme)) - return []client.Object{legacy, secret} - }, - assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - ctx := context.Background() - assert.True(t, apierrors.IsNotFound(e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{})), "legacy Certificate is retired so cert-manager does not renew it") - - var secret corev1.Secret - require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) - assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"], "the serving Secret is kept as-is until the service issues") - assert.Equal(t, upstreamNamespace.Name, secret.Labels[downstreamclient.UpstreamOwnerNamespaceLabel]) - assert.Equal(t, labelValueTrue, secret.Labels[tlsCertificateManagedLabel], "the hand-over is recorded for rollback") - var anchored bool - for _, ref := range secret.OwnerReferences { - if ref.Kind == "ConfigMap" { - anchored = true - } + name: "a request the API refuses tells the customer why instead of promising to keep trying", + upstreamCreate: func(obj client.Object) error { + if _, ok := obj.(*certificatesv1alpha1.TLSCertificate); !ok { + return nil } - assert.True(t, anchored, "Secret gains the gateway anchor so the Certificate's owner reference no longer takes it along") - - assert.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{}), "TLSCertificate is requested") - assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener keeps serving on the retained Secret") - }, - }, - { - name: "does not retire a cert-manager certificate while cert-manager is renewing it", - downstreamObjects: func() []client.Object { - legacy := newLegacyCertificate(now.Add(-time.Hour)) - legacy.Status.Conditions = append(legacy.Status.Conditions, cmv1.CertificateCondition{Type: cmv1.CertificateConditionIssuing, Status: cmmeta.ConditionTrue}) - return []client.Object{legacy, tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM)} + return apierrors.NewInvalid(schema.GroupKind{Group: "certificates.miloapis.com", Kind: KindTLSCertificate}, certName, + field.ErrorList{field.Forbidden(field.NewPath("spec", "dnsNames"), "names under datum.net are denied")}) }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - ctx := context.Background() - assert.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{}), "legacy Certificate stays while its renewal is in flight") - var certs certificatesv1alpha1.TLSCertificateList - require.NoError(t, e.upstream.List(ctx, &certs, client.InNamespace(upstreamNamespace.Name))) - assert.Empty(t, certs.Items) - - strategy := downstreamclient.NewMappedNamespaceResourceStrategy(upstreamCluster, e.upstream, e.downstream) - step, issues := e.reconciler.ensureListenerTLSCertificates(ctx, e.upstream, upstreamGateway, downstreamGateway, e.downstream, strategy, []string{hostname}) - assert.Empty(t, issues) - assert.Equal(t, 10*time.Minute, step.RequeueAfter, "the hold is re-checked while cert-manager renews") + blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Contains(t, blocked.Message, "names under datum.net are denied") + assert.NotContains(t, blocked.Message, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRefused), 1.0) + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) }, }, { @@ -427,15 +348,16 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { other := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: "other-gw", UID: uuid.NewUUID()}} return []client.Object{newTLSCertificate(other, certName, func(c *certificatesv1alpha1.TLSCertificate) { - c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"previous.example.com"} + c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"*.previous.example.com"} })} }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { var cert certificatesv1alpha1.TLSCertificate require.NoError(t, e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) - assert.Equal(t, []certificatesv1alpha1.DNSName{"previous.example.com"}, cert.Spec.DNSNames, "someone else's TLSCertificate is left alone") + assert.Equal(t, []certificatesv1alpha1.DNSName{"*.previous.example.com"}, cert.Spec.DNSNames, "someone else's TLSCertificate is left alone") assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter, "the clash is retried with backoff, not an error") - assertListenerSaysKeepTrying(t, upstreamGateway) + resolved := listenerCondition(t, upstreamGateway, listenerName, string(gatewayv1.ListenerConditionResolvedRefs)) + assert.Contains(t, resolved.Message, "another resource holds the certificate request") }, }, { @@ -449,7 +371,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { serviceObjects: []client.Object{tlsSecret("victim-namespace", serviceSecret, serviceCertPEM, serviceKeyPEM)}, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) - assertListenerSaysKeepTrying(t, upstreamGateway) + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNamespaceRefused), 1.0) }, }, { @@ -468,11 +390,33 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { return []client.Object{newTLSCertificate(gw, certName, readyStatus)} }, serviceObjects: func() []client.Object { - crt, key := generateTLSKeyPair(t, "other.example.com", now.Add(-time.Hour), now.Add(60*24*time.Hour)) + crt, key := ca.issue(t, "*.example.com", now.Add(-time.Hour), now.Add(60*24*time.Hour)) return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} }(), assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{})), "*.example.com does not cover *.shop.example.com") + }, + }, + { + name: "refuses a chain the configured roots do not trust and keeps the serving Secret", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(61 * 24 * time.Hour)} + })} + }, + downstreamObjects: serving, + serviceObjects: func() []client.Object { + crt, key := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var secret corev1.Secret + require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.Equal(t, servingCertPEM, secret.Data["tls.crt"], "an untrusted chain never replaces what serves") + blocked := assertListenerRenewalBlocked(t, upstreamGateway, "did not pass our checks") + assert.NotContains(t, blocked.Message, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonUntrustedChain), 1.0) }, }, { @@ -484,11 +428,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }) })} }, - downstreamObjects: func() []client.Object { - secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) - secret.Labels = map[string]string{tlsCertificateManagedLabel: labelValueTrue} - return []client.Object{secret} - }, + downstreamObjects: serving, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") assertListenerRenewalBlocked(t, upstreamGateway, "names under datum.net are denied") @@ -496,80 +436,128 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, }, { - name: "a forbidden service-side read while the previous certificate serves is reported as a blocked renewal", + name: "a failed renewal order while the previous certificate serves is reported and counted", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { - return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(60 * 24 * time.Hour).Add(time.Second)} + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionFalse, Reason: "IssuanceFailed", Message: "DNS problem: NXDOMAIN looking up TXT", + }) + })} }, - downstreamObjects: func() []client.Object { - return []client.Object{tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM)} + downstreamObjects: serving, + serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, serviceKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + assertListenerRenewalBlocked(t, upstreamGateway, "NXDOMAIN") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonIssuanceFailed), 1.0) + }, + }, + { + name: "a request stuck short of Ready with nothing serving is reported as blocked issuance", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.Conditions = []metav1.Condition{{ + Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "SecretConflict", + Message: `Secret "x" exists and is not managed by this TLSCertificate.`, LastTransitionTime: metav1.NewTime(now.Add(-2 * time.Hour)), + }} + })} }, - serviceForbidden: true, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") - assertListenerRenewalBlocked(t, upstreamGateway, "keep trying") - assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) - assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), 1.0) + blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Contains(t, blocked.Message, "is not managed by this TLSCertificate") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNotReady), 1.0) + assert.Equal(t, 1.0, gaugeValue(t, certificateServiceListenerFailing, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNotReady), "a lasting failure stays visible after its one count") }, }, { - name: "a forbidden service-side read with nothing serving is reported as blocked issuance", + name: "waiting on the customer's DNS records is progress, not a failure", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { - return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.Conditions = []metav1.Condition{ + {Type: certificatesv1alpha1.ConditionDNSDelegationReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.NewTime(now.Add(-48 * time.Hour))}, + {Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.NewTime(now.Add(-48 * time.Hour))}, + } + })} }, - serviceForbidden: true, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) - assertListenerSaysKeepTrying(t, upstreamGateway) for _, ls := range upstreamGateway.Status.Listeners { - if ls.Name != listenerName { - continue + if ls.Name == listenerName { + assert.Nil(t, apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateIssuanceBlocked)) } - blocked := apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateIssuanceBlocked) - require.NotNil(t, blocked) - assert.Equal(t, metav1.ConditionTrue, blocked.Status) - assert.Equal(t, listenerReasonIssuanceFailing, blocked.Reason) - assert.Contains(t, blocked.Message, "keep trying") } }, }, { - name: "refused material while the previous certificate serves is reported as a blocked renewal", + name: "a served certificate past its renewal point with no newer issuance is overdue", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { - return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotBefore = &metav1.Time{Time: now.Add(-80 * 24 * time.Hour)} + c.Status.NotAfter = &metav1.Time{Time: now.Add(10 * 24 * time.Hour)} + })} }, downstreamObjects: func() []client.Object { - return []client.Object{tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM)} + crt, key := ca.issue(t, hostname, now.Add(-80*24*time.Hour), now.Add(10*24*time.Hour)) + return []client.Object{tlsSecret(downstreamNamespaceName, secretName, crt, key)} }, - serviceObjects: func() []client.Object { - crt, key := generateTLSKeyPair(t, "other.example.com", now.Add(-time.Hour), now.Add(60*24*time.Hour)) - return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} - }(), assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { - var secret corev1.Secret - require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) - assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"]) + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + assertListenerRenewalBlocked(t, upstreamGateway, "has not been renewed") + assert.Equal(t, 1.0, gaugeValue(t, certificateServiceListenerFailing, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRenewalOverdue)) + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRenewalOverdue), 1.0) + }, + }, + { + name: "a forbidden service-side read while the previous certificate serves is reported as a blocked renewal", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(90 * 24 * time.Hour)} + })} + }, + downstreamObjects: serving, + serviceForbidden: true, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") assertListenerRenewalBlocked(t, upstreamGateway, "keep trying") - assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonMaterialRefused), 1.0) + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), 1.0) }, }, { - name: "refuses expired service-side material and keeps the serving Secret", + name: "a forbidden service-side read with nothing serving is reported as blocked issuance", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, readyStatus)} }, - downstreamObjects: func() []client.Object { - secret := tlsSecret(downstreamNamespaceName, secretName, legacyCertPEM, legacyKeyPEM) - secret.Labels = map[string]string{tlsCertificateManagedLabel: labelValueTrue} - return []client.Object{secret} + serviceForbidden: true, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Equal(t, listenerReasonIssuanceFailing, blocked.Reason) + assert.Contains(t, blocked.Message, "keep trying") + }, + }, + { + name: "refuses expired service-side material and keeps the serving Secret", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(-time.Hour)} + })} }, + downstreamObjects: serving, serviceObjects: func() []client.Object { - crt, key := generateTLSKeyPair(t, hostname, now.Add(-48*time.Hour), now.Add(-time.Hour)) + crt, key := ca.issue(t, hostname, now.Add(-48*time.Hour), now.Add(-time.Hour)) return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} }(), assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { var secret corev1.Secret require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) - assert.Equal(t, legacyCertPEM, secret.Data["tls.crt"], "a valid serving Secret is never replaced by material that fails validation") + assert.Equal(t, servingCertPEM, secret.Data["tls.crt"], "a valid serving Secret is never replaced by material that fails validation") assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) }, }, @@ -592,7 +580,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { name: "requests the TLSCertificate again when its immutable spec no longer matches", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { - c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"previous.example.com"} + c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"*.previous.example.com"} })} }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { @@ -605,11 +593,12 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { - g.Spec.Listeners = append(g.Spec.Listeners, customListener) + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(listenerName, hostname)) }) upstreamObjects := []client.Object{ - verifiedDomain.DeepCopy(), + dnsProvenDomain(upstreamNamespace.Name), &gatewayv1.GatewayClass{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: gatewayv1.GatewayController("test")}, @@ -629,13 +618,23 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { obj.SetCreationTimestamp(metav1.Now()) } - fakeUpstreamClient := fake.NewClientBuilder(). + upstreamBuilder := fake.NewClientBuilder(). WithScheme(testScheme). WithObjects(upstreamGateway, upstreamNamespace). WithObjects(upstreamObjects...). WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). - WithStatusSubresource(upstreamObjects...). - Build() + WithStatusSubresource(upstreamObjects...) + if tt.upstreamCreate != nil { + upstreamBuilder = upstreamBuilder.WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if err := tt.upstreamCreate(obj); err != nil { + return err + } + return cl.Create(ctx, obj, opts...) + }, + }) + } + fakeUpstreamClient := upstreamBuilder.Build() fakeDownstreamClient := fake.NewClientBuilder(). WithScheme(testScheme). @@ -653,7 +652,6 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, }) } - fakeServiceClient := serviceBuilder.Build() ctx := log.IntoContext(context.Background(), logger) @@ -661,7 +659,8 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { mgr: &fakeMockManager{cl: fakeUpstreamClient}, Config: testCfg, DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, - CertificateServiceReader: fakeServiceClient, + CertificateServiceReader: serviceBuilder.Build(), + CertificateServiceRoots: ca.pool, } downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy(upstreamCluster, fakeUpstreamClient, fakeDownstreamClient) @@ -677,7 +676,6 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { tt.assert(t, env{ upstream: fakeUpstreamClient, downstream: fakeDownstreamClient, - service: fakeServiceClient, result: result, reconciler: reconciler, }, updatedUpstream, downstreamGateway) @@ -685,23 +683,62 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { } } -func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, want string) { +func listenerCondition(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, conditionType string) *metav1.Condition { + t.Helper() + for _, ls := range gateway.Status.Listeners { + if ls.Name != listener { + continue + } + condition := apimeta.FindStatusCondition(ls.Conditions, conditionType) + require.NotNil(t, condition, "listener %s has no %s condition", listener, conditionType) + return condition + } + t.Fatalf("no status for listener %s", listener) + return nil +} + +func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, want string) *metav1.Condition { t.Helper() for _, ls := range gateway.Status.Listeners { - if apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked) == nil { + blocked := apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked) + if blocked == nil { continue } resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) require.NotNil(t, resolved) assert.Equal(t, metav1.ConditionTrue, resolved.Status, "the listener itself stays resolved") - blocked := apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked) - require.NotNil(t, blocked, "a blocked renewal is reported while the listener serves") assert.Equal(t, metav1.ConditionTrue, blocked.Status) assert.Equal(t, listenerReasonRenewalFailing, blocked.Reason) assert.Contains(t, blocked.Message, want) - return + return blocked } t.Fatal("no listener reports a blocked renewal") + return nil +} + +func seriesInNamespace(t *testing.T, vec prometheus.Collector, namespace string) int { + t.Helper() + ch := make(chan prometheus.Metric, 1024) + vec.Collect(ch) + close(ch) + count := 0 + for metric := range ch { + var m dto.Metric + require.NoError(t, metric.Write(&m)) + for _, label := range m.GetLabel() { + if label.GetName() == jsonKeyNamespace && label.GetValue() == namespace { + count++ + } + } + } + return count +} + +func gaugeValue(t *testing.T, vec *prometheus.GaugeVec, labels ...string) float64 { + t.Helper() + var m dto.Metric + require.NoError(t, vec.WithLabelValues(labels...).Write(&m)) + return m.GetGauge().GetValue() } func counterValue(t *testing.T, vec *prometheus.CounterVec, labels ...string) float64 { @@ -711,17 +748,283 @@ func counterValue(t *testing.T, vec *prometheus.CounterVec, labels ...string) fl return m.GetCounter().GetValue() } -func assertListenerSaysKeepTrying(t *testing.T, gateway *gatewayv1.Gateway) { - t.Helper() - for _, ls := range gateway.Status.Listeners { - resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) - if resolved == nil || resolved.Status != metav1.ConditionFalse { - continue +// certificateServiceHarness drives full gateway reconciles against fake +// clients that persist across passes, so a test can change the operator's +// config between them as a rollout would. +type certificateServiceHarness struct { + t *testing.T + ctx context.Context + cfg config.NetworkServicesOperator + upstream client.Client + downstream client.Client + service client.Reader + roots *x509.CertPool + gateway client.ObjectKey + namespace string + reconciler *GatewayReconciler +} + +func (h *certificateServiceHarness) reconcile() (*gatewayv1.Gateway, *gatewayv1.Gateway, Result) { + h.t.Helper() + if h.reconciler == nil || h.reconciler.Config.Gateway.CertificateService.Enabled != h.cfg.Gateway.CertificateService.Enabled { + h.reconciler = &GatewayReconciler{ + mgr: &fakeMockManager{cl: h.upstream}, + Config: h.cfg, + DownstreamCluster: &fakeCluster{cl: h.downstream}, + CertificateServiceReader: h.service, + CertificateServiceRoots: h.roots, } - assert.Contains(t, resolved.Message, "keep trying") - return } - t.Fatal("no listener reports an unresolved certificate") + var current gatewayv1.Gateway + require.NoError(h.t, h.upstream.Get(h.ctx, h.gateway, ¤t)) + h.reconciler.prepareUpstreamGateway(¤t) + strategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", h.upstream, h.downstream) + result, downstreamGateway := h.reconciler.ensureDownstreamGateway(h.ctx, "test", h.upstream, ¤t, strategy) + require.NoError(h.t, result.Err) + _, err := result.Complete(h.ctx) + require.NoError(h.t, err) + require.NoError(h.t, h.upstream.Get(h.ctx, h.gateway, ¤t)) + + var claims corev1.ConfigMapList + require.NoError(h.t, h.downstream.List(h.ctx, &claims, client.InNamespace(h.cfg.Gateway.DownstreamHostnameAccountingNamespace))) + for i := range claims.Items { + stampCreated(h.t, h.ctx, h.downstream, &claims.Items[i]) + } + var gateways gatewayv1.GatewayList + require.NoError(h.t, h.downstream.List(h.ctx, &gateways, client.InNamespace(h.namespace))) + for i := range gateways.Items { + stampCreated(h.t, h.ctx, h.downstream, &gateways.Items[i]) + } + var certs cmv1.CertificateList + require.NoError(h.t, h.downstream.List(h.ctx, &certs, client.InNamespace(h.namespace))) + for i := range certs.Items { + stampCreated(h.t, h.ctx, h.downstream, &certs.Items[i]) + } + var secrets corev1.SecretList + require.NoError(h.t, h.downstream.List(h.ctx, &secrets, client.InNamespace(h.namespace))) + for i := range secrets.Items { + stampCreated(h.t, h.ctx, h.downstream, &secrets.Items[i]) + } + var tlsCerts certificatesv1alpha1.TLSCertificateList + require.NoError(h.t, h.upstream.List(h.ctx, &tlsCerts)) + for i := range tlsCerts.Items { + stampCreated(h.t, h.ctx, h.upstream, &tlsCerts.Items[i]) + } + return ¤t, downstreamGateway, result +} + +func TestCertificateServiceLeavesExactHostnamesOnCertManager(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + const ( + gatewayName = "mixed-gw" + serviceNS = "certificates-system" + exact = "app.example.com" + wildcard = "*.shop.example.com" + ) + exactListener := gatewayv1.SectionName("https-hostname-0") + wildcardListenerName := gatewayv1.SectionName("https-hostname-1") + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "mixed", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + ca := newTestCA(t) + now := time.Now() + + cfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + upstreamGateway := newGateway(cfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(exactListener, exact), tlsListener(wildcardListenerName, wildcard)) + }) + domain := dnsProvenDomain(upstreamNamespace.Name) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + for _, obj := range []client.Object{domain, gatewayClass} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + var tlsCertificateCreates, tlsCertificateDeletes atomic.Int32 + var requestedNames []string + upstream := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass). + WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if cert, ok := obj.(*certificatesv1alpha1.TLSCertificate); ok { + tlsCertificateCreates.Add(1) + for _, name := range cert.Spec.DNSNames { + requestedNames = append(requestedNames, string(name)) + } + } + return cl.Create(ctx, obj, opts...) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*certificatesv1alpha1.TLSCertificate); ok { + tlsCertificateDeletes.Add(1) + } + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + certificateWrites := map[string]int{} + countCertificate := func(obj client.Object, verb string) { + if _, ok := obj.(*cmv1.Certificate); ok { + certificateWrites[verb+" "+obj.GetName()]++ + } + } + downstream := fake.NewClientBuilder().WithScheme(testScheme). + WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + countCertificate(obj, "create") + return cl.Create(ctx, obj, opts...) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + countCertificate(obj, "delete") + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + + issuedCrt, issuedKey := ca.issue(t, wildcard, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + service := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: "issued"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, + }).Build() + + h := &certificateServiceHarness{ + t: t, ctx: ctx, cfg: cfg, upstream: upstream, downstream: downstream, service: service, roots: ca.pool, + gateway: client.ObjectKeyFromObject(upstreamGateway), namespace: downstreamNamespaceName, + } + + exactCertificate := func() *cmv1.Certificate { + var cert cmv1.Certificate + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateName(gatewayName, exactListener)}, &cert)) + return &cert + } + + h.reconcile() + + assert.Equal(t, []string{wildcard}, requestedNames, "only the wildcard is handed to the certificate service") + assert.Equal(t, []string{exact}, exactCertificate().Spec.DNSNames, "the exact hostname gets its cert-manager Certificate exactly as before") + assert.True(t, apierrors.IsNotFound(downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateName(gatewayName, wildcardListenerName)}, &cmv1.Certificate{})), + "cert-manager never orders for the wildcard while the service is on") + exactName := listenerCertificateName(gatewayName, exactListener) + require.Equal(t, 1, certificateWrites["create "+exactName]) + + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, wildcardListenerName)}, &cert)) + cert.Status = certificatesv1alpha1.TLSCertificateStatus{ + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, + } + require.NoError(t, upstream.Status().Update(ctx, &cert)) + h.reconcile() + _, downstreamGateway, _ := h.reconcile() + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, wildcardListenerName), "the wildcard serves the service-issued certificate") + + var mirrored corev1.Secret + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName(gatewayName, wildcardListenerName)}, &mirrored)) + require.Equal(t, issuedCrt, mirrored.Data["tls.crt"]) + + h.cfg.Gateway.CertificateService.Enabled = false + h.reconcile() + assert.Zero(t, seriesInNamespace(t, certificateServiceListenerFailing, upstreamNamespace.Name), "nothing reports as failing once the service is off") + h.cfg.Gateway.CertificateService.Enabled = true + h.reconcile() + + assert.Equal(t, int32(1), tlsCertificateCreates.Load(), "turning the service off and on again requests nothing new") + assert.Zero(t, tlsCertificateDeletes.Load(), "turning the service off withdraws no request") + assert.Equal(t, []string{wildcard}, requestedNames, "an exact hostname is never handed to the service, whatever the flag did") + assert.Equal(t, 1, certificateWrites["create "+exactName], "the exact hostname's Certificate is never replaced") + assert.Zero(t, certificateWrites["delete "+exactName]) + assert.Equal(t, []string{exact}, exactCertificate().Spec.DNSNames) + require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, wildcardListenerName)}, &cert)) + assert.Equal(t, []certificatesv1alpha1.DNSName{wildcard}, cert.Spec.DNSNames) + + var stillMirrored corev1.Secret + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName(gatewayName, wildcardListenerName)}, &stillMirrored)) + assert.Equal(t, issuedCrt, stillMirrored.Data["tls.crt"], "the issued certificate is still the one served") +} + +func TestCertificateServiceOneFailingListenerDoesNotDelayOthers(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + const ( + gatewayName = "two-gw" + serviceNS = "certificates-system" + failing = "*.broken.example.com" + healthy = "*.shop.example.com" + ) + failingListener := gatewayv1.SectionName("https-hostname-0") + healthyListener := gatewayv1.SectionName("https-hostname-1") + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "two", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + ca := newTestCA(t) + now := time.Now() + + cfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + upstreamGateway := newGateway(cfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(failingListener, failing), tlsListener(healthyListener, healthy)) + }) + domain := dnsProvenDomain(upstreamNamespace.Name) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + for _, obj := range []client.Object{domain, gatewayClass} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + failingName := tlsCertificateName(gatewayName, failingListener) + upstream := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass). + WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetName() == failingName { + return apierrors.NewServiceUnavailable("webhook unavailable") + } + return cl.Create(ctx, obj, opts...) + }, + }).Build() + downstream := fake.NewClientBuilder().WithScheme(testScheme).WithStatusSubresource(&gatewayv1.Gateway{}).Build() + issuedCrt, issuedKey := ca.issue(t, healthy, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + service := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: "issued"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, + }).Build() + + h := &certificateServiceHarness{ + t: t, ctx: ctx, cfg: cfg, upstream: upstream, downstream: downstream, service: service, roots: ca.pool, + gateway: client.ObjectKeyFromObject(upstreamGateway), namespace: downstreamNamespaceName, + } + + first, _, _ := h.reconcile() + assert.Contains(t, listenerCondition(t, first, failingListener, listenerConditionCertificateIssuanceBlocked).Message, "keep trying") + + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, healthyListener)}, &cert), "the healthy wildcard is requested in the same pass") + cert.Status = certificatesv1alpha1.TLSCertificateStatus{ + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, + } + require.NoError(t, upstream.Status().Update(ctx, &cert)) + + second, _, result := h.reconcile() + _, _, cooling := h.reconciler.certificateServiceInBackoff(certificateServiceKey{gateway: upstreamGateway.UID, listener: failingListener}, time.Now()) + assert.True(t, cooling, "the failing listener is still backing off") + + var mirrored corev1.Secret + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName(gatewayName, healthyListener)}, &mirrored), "the healthy wildcard is mirrored while the other backs off") + assert.Equal(t, issuedCrt, mirrored.Data["tls.crt"]) + assert.Contains(t, listenerCondition(t, second, failingListener, listenerConditionCertificateIssuanceBlocked).Message, "keep trying", "the failing listener keeps its message") + assert.LessOrEqual(t, result.RequeueAfter, certificateServiceBackoffMax) } func TestTLSCertificateName(t *testing.T) { @@ -735,35 +1038,6 @@ func TestTLSCertificateName(t *testing.T) { assert.True(t, strings.HasPrefix(tlsCertificateName("gw", "https-0"), "gw-https-0-")) } -func TestCertificateServiceRequeueBackoff(t *testing.T) { - r := &GatewayReconciler{} - uid := types.UID("gw") - now := time.Now() - issues := map[gatewayv1.SectionName]string{"https-0": "trying", "https-1": "other"} - assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) - kept, _, _ := r.certificateServiceInBackoff(uid, now) - assert.Equal(t, "other", kept.issues["https-1"], "each listener keeps its own message") - issues = map[gatewayv1.SectionName]string{"https-0": "trying"} - assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) - assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(uid, true, now, issues)) - for range 10 { - r.certificateServiceRequeue(uid, true, now, issues) - } - assert.Equal(t, certificateServiceBackoffMax, r.certificateServiceRequeue(uid, true, now, issues)) - - backoff, remaining, cooling := r.certificateServiceInBackoff(uid, now.Add(time.Minute)) - assert.True(t, cooling, "an event inside the window repeats the message instead of the calls") - assert.Equal(t, map[gatewayv1.SectionName]string{"https-0": "trying"}, backoff.issues) - assert.Equal(t, certificateServiceBackoffMax-time.Minute, remaining) - _, _, cooling = r.certificateServiceInBackoff(uid, now.Add(certificateServiceBackoffMax)) - assert.False(t, cooling) - - assert.Zero(t, r.certificateServiceRequeue(uid, false, now, nil)) - assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(uid, true, now, issues), "a success resets the backoff") - _, _, cooling = r.certificateServiceInBackoff(types.UID("other"), now) - assert.False(t, cooling) -} - func TestTLSCertificateNameGolden(t *testing.T) { for _, tt := range []struct{ gateway, listener, want string }{ {"gw", "https-0", "gw-https-0-bb35a504d1"}, @@ -776,6 +1050,42 @@ func TestTLSCertificateNameGolden(t *testing.T) { } } +func TestIsSingleLabelWildcard(t *testing.T) { + assert.True(t, isSingleLabelWildcard("*.example.com")) + assert.True(t, isSingleLabelWildcard("*.shop.example.com")) + assert.False(t, isSingleLabelWildcard("app.example.com")) + assert.False(t, isSingleLabelWildcard("*.*.example.com")) + assert.False(t, isSingleLabelWildcard("*.")) + assert.False(t, isSingleLabelWildcard("a.*.example.com")) +} + +func TestCertificateServiceRequeueBackoff(t *testing.T) { + r := &GatewayReconciler{} + key := certificateServiceKey{gateway: types.UID("gw"), listener: "https-0"} + other := certificateServiceKey{gateway: types.UID("gw"), listener: "https-1"} + now := time.Now() + trying := certificateServiceIssue{reason: certificateServiceReasonStepFailed, message: "trying"} + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(key, true, now, trying)) + assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(key, true, now, trying)) + assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(key, true, now, trying)) + for range 10 { + r.certificateServiceRequeue(key, true, now, trying) + } + assert.Equal(t, certificateServiceBackoffMax, r.certificateServiceRequeue(key, true, now, trying)) + + backoff, remaining, cooling := r.certificateServiceInBackoff(key, now.Add(time.Minute)) + assert.True(t, cooling, "an event inside the window repeats the message instead of the calls") + assert.Equal(t, trying, backoff.issue) + assert.Equal(t, certificateServiceBackoffMax-time.Minute, remaining) + _, _, cooling = r.certificateServiceInBackoff(key, now.Add(certificateServiceBackoffMax)) + assert.False(t, cooling) + _, _, cooling = r.certificateServiceInBackoff(other, now) + assert.False(t, cooling, "another listener on the same gateway is not held back") + + assert.Zero(t, r.certificateServiceRequeue(key, false, now, certificateServiceIssue{})) + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(key, true, now, trying), "a success resets the backoff") +} + func TestValidateIssuedMaterialWildcard(t *testing.T) { now := time.Now() wildcardCrt, wildcardKey := generateTLSKeyPair(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) @@ -787,53 +1097,63 @@ func TestValidateIssuedMaterialWildcard(t *testing.T) { assert.Error(t, validateIssuedMaterial(apexCrt, apexKey, "*.example.com", now), "an apex certificate does not cover a wildcard listener") } -func TestLegacyCertificateHolds(t *testing.T) { +func TestVerifyIssuedChain(t *testing.T) { now := time.Now() - base := func() *cmv1.Certificate { - return &cmv1.Certificate{Status: cmv1.CertificateStatus{ - NotBefore: &metav1.Time{Time: now.Add(-30 * 24 * time.Hour)}, - NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, - RenewalTime: &metav1.Time{Time: now.Add(-time.Hour)}, - Conditions: []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionTrue}}, - }} - } - issuing := func(since time.Time) cmv1.CertificateCondition { - return cmv1.CertificateCondition{Type: cmv1.CertificateConditionIssuing, Status: cmmeta.ConditionTrue, LastTransitionTime: &metav1.Time{Time: since}} + ca := newTestCA(t) + other := newTestCA(t) + crt, _ := ca.issue(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + selfSigned, _ := generateTLSKeyPair(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + + assert.NoError(t, verifyIssuedChain(crt, "*.example.com", ca.pool, now)) + assert.Error(t, verifyIssuedChain(crt, "*.example.com", other.pool, now), "a chain from another CA is not trusted") + assert.Error(t, verifyIssuedChain(selfSigned, "*.example.com", ca.pool, now), "a self-signed leaf is not trusted") + assert.Error(t, verifyIssuedChain(crt, "*.other.com", ca.pool, now)) + assert.Error(t, verifyIssuedChain(nil, "*.example.com", ca.pool, now)) +} + +func TestTLSCertificateFailure(t *testing.T) { + now := time.Now() + old := metav1.NewTime(now.Add(-2 * tlsCertificateIssueGrace)) + fresh := metav1.NewTime(now.Add(-time.Minute)) + for _, tt := range []struct { + name string + created metav1.Time + conditions []metav1.Condition + want string + }{ + {name: "ready", created: old, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue}}}, + {name: "rejected", created: fresh, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse}}, want: certificateServiceReasonRejected}, + {name: "renewal order failed while ready", created: old, conditions: []metav1.Condition{ + {Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue}, + {Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionFalse, Reason: "IssuanceFailed"}, + }, want: certificateServiceReasonIssuanceFailed}, + {name: "fresh request with no status yet", created: fresh}, + {name: "request the service never engaged", created: old, want: certificateServiceReasonNotReady}, + {name: "pending past the grace", created: old, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: old}}, want: certificateServiceReasonNotReady}, + {name: "pending within the grace", created: old, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: fresh}}}, + {name: "waiting on the customer's records", created: old, conditions: []metav1.Condition{ + {Type: certificatesv1alpha1.ConditionDNSDelegationReady, Status: metav1.ConditionFalse, LastTransitionTime: old}, + {Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: old}, + }}, + } { + t.Run(tt.name, func(t *testing.T) { + cert := &certificatesv1alpha1.TLSCertificate{ObjectMeta: metav1.ObjectMeta{CreationTimestamp: tt.created}} + cert.Status.Conditions = tt.conditions + got, _ := tlsCertificateFailure(cert, now) + assert.Equal(t, tt.want, got) + }) } +} - cert := base() - cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-time.Hour))) - holds, recheck := legacyCertificateHolds(cert, now) - assert.True(t, holds, "a fresh renewal in flight is left to finish") - assert.Equal(t, 10*time.Minute, recheck) - - cert = base() - cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-25*time.Hour))) - holds, _ = legacyCertificateHolds(cert, now) - assert.False(t, holds, "a renewal in flight for over a day is stuck") - - cert = base() - cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-time.Hour))) - cert.Status.LastFailureTime = &metav1.Time{Time: now.Add(-10 * time.Minute)} - holds, _ = legacyCertificateHolds(cert, now) - assert.False(t, holds, "a renewal that has failed does not hold") - - cert = base() - cert.Status.Conditions = append(cert.Status.Conditions, issuing(now.Add(-time.Hour))) - cert.Status.NotAfter = &metav1.Time{Time: now.Add(5 * 24 * time.Hour)} - holds, _ = legacyCertificateHolds(cert, now) - assert.False(t, holds, "under a week of lifetime hands over regardless") - - cert = base() - cert.Status.RenewalTime = &metav1.Time{Time: now.Add(10 * 24 * time.Hour)} - holds, recheck = legacyCertificateHolds(cert, now) - assert.True(t, holds) - assert.InDelta(t, (10*24*time.Hour - tlsCertificateSwitchLead).Seconds(), recheck.Seconds(), 1) - - cert = base() - cert.Status.Conditions = []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionFalse, Reason: "Failed"}} - holds, _ = legacyCertificateHolds(cert, now) - assert.False(t, holds, "a certificate that is not serving never holds") +func TestRenewalOverdue(t *testing.T) { + now := time.Now() + leaf := func(age, left time.Duration) *x509.Certificate { + return &x509.Certificate{NotBefore: now.Add(-age), NotAfter: now.Add(left)} + } + assert.False(t, renewalOverdue(leaf(10*24*time.Hour, 80*24*time.Hour), now)) + assert.False(t, renewalOverdue(leaf(65*24*time.Hour, 25*24*time.Hour), now), "inside the window the service renews in") + assert.True(t, renewalOverdue(leaf(70*24*time.Hour, 20*24*time.Hour), now)) + assert.False(t, renewalOverdue(nil, now)) } func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { @@ -848,29 +1168,13 @@ func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { downstreamScheme := newCertificateServiceTestScheme(t) upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}} - const hostname = "custom.example.com" - testCfg := config.NetworkServicesOperator{Gateway: config.GatewayConfig{ - DownstreamGatewayClassName: "test-suite", - DownstreamHostnameAccountingNamespace: "default", - TargetDomain: "test-suite.com", - DefaultListenerTLSSecretName: "wildcard-test-suite-tls", - IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, - ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer"}, - CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system"}, - }} + const hostname = "*.shop.example.com" + testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, "test-gw", func(g *gatewayv1.Gateway) { - g.Spec.Listeners = append(g.Spec.Listeners, gatewayv1.Listener{ - Name: "https-hostname-0", Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, - Hostname: ptr.To(gatewayv1.Hostname(hostname)), - TLS: &gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModeTerminate), Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ - gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer", - }}, - }) - }) - domain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { - d.Spec.DomainName = hostname - apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue}) + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener("https-hostname-0", hostname)) }) + domain := dnsProvenDomain(upstreamNamespace.Name) gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} route := newHTTPRoute(upstreamNamespace.Name, "app", func(r *gatewayv1.HTTPRoute) { r.Spec.ParentRefs = []gatewayv1.ParentReference{{Name: gatewayv1.ObjectName(upstreamGateway.Name)}} @@ -908,144 +1212,13 @@ func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { var updated gatewayv1.Gateway require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), &updated)) assert.NotEmpty(t, updated.Status.Listeners, "gateway status is still written") - assertListenerSaysKeepTrying(t, &updated) + assert.Contains(t, listenerCondition(t, &updated, "https-hostname-0", listenerConditionCertificateIssuanceBlocked).Message, "keep trying") var downstreamRoutes gatewayv1.HTTPRouteList require.NoError(t, fakeDownstreamClient.List(ctx, &downstreamRoutes, client.InNamespace(downstreamGateway.Namespace))) assert.NotEmpty(t, downstreamRoutes.Items, "downstream routes are still reconciled") } -func TestCertificateServiceRollbackKeepsHandedOverSecret(t *testing.T) { - testScheme := newCertificateServiceTestScheme(t) - logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) - ctx := log.IntoContext(context.Background(), logger) - - upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}} - downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) - const hostname = "custom.example.com" - const gatewayName = "test-gw" - const listenerName = gatewayv1.SectionName("https-hostname-0") - secretName := listenerCertificateSecretName(gatewayName, listenerName) - legacyName := listenerCertificateName(gatewayName, listenerName) - certName := tlsCertificateName(gatewayName, listenerName) - - testCfg := config.NetworkServicesOperator{Gateway: config.GatewayConfig{ - DownstreamGatewayClassName: "test-suite", - DownstreamHostnameAccountingNamespace: "default", - TargetDomain: "test-suite.com", - DefaultListenerTLSSecretName: "wildcard-test-suite-tls", - IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, - ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer"}, - }} - - now := time.Now() - tests := []struct { - name string - notBefore time.Time - notAfter time.Time - wantCertificate bool - wantListenerAdmit bool - }{ - {name: "fresh handed-over Secret keeps serving and is not reissued", notBefore: now.Add(-24 * time.Hour), notAfter: now.Add(80 * 24 * time.Hour), wantCertificate: false, wantListenerAdmit: true}, - {name: "handed-over Secret in its renewal window gets a cert-manager Certificate again", notBefore: now.Add(-70 * 24 * time.Hour), notAfter: now.Add(20 * 24 * time.Hour), wantCertificate: true, wantListenerAdmit: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { - g.Spec.Listeners = append(g.Spec.Listeners, gatewayv1.Listener{ - Name: listenerName, Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, - Hostname: ptr.To(gatewayv1.Hostname(hostname)), - TLS: &gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModeTerminate), Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ - gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer", - }}, - }) - }) - domain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { - d.Spec.DomainName = hostname - apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue}) - }) - gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} - leftover := &certificatesv1alpha1.TLSCertificate{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: certName}} - require.NoError(t, controllerutil.SetControllerReference(upstreamGateway, leftover, testScheme)) - foreign := &certificatesv1alpha1.TLSCertificate{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: "foreign"}} - - downstreamGateway := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: gatewayName, UID: uuid.NewUUID(), Labels: map[string]string{ - downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-test", - downstreamclient.UpstreamOwnerKindLabel: KindGateway, - downstreamclient.UpstreamOwnerNameLabel: gatewayName, - downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace.Name, - }}, Spec: gatewayv1.GatewaySpec{GatewayClassName: "test-suite"}} - crt, key := generateTLSKeyPair(t, hostname, tt.notBefore, tt.notAfter) - handedOver := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: secretName, Labels: map[string]string{ - tlsCertificateManagedLabel: labelValueTrue, - downstreamclient.UpstreamOwnerKindLabel: KindGateway, - downstreamclient.UpstreamOwnerNameLabel: gatewayName, - downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace.Name, - }}, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{"tls.crt": crt, "tls.key": key}, - } - solverName := tlsCertificateSolverName(certName, "token-leftoverleftover") - solverRoute := &gatewayv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: solverName, Labels: map[string]string{tlsCertificateSolverLabel: certName}}} - solverFilter := &envoygatewayv1alpha1.HTTPRouteFilter{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: solverName, Labels: map[string]string{tlsCertificateSolverLabel: certName}}} - require.NoError(t, controllerutil.SetControllerReference(downstreamGateway, solverRoute, testScheme)) - require.NoError(t, controllerutil.SetControllerReference(downstreamGateway, solverFilter, testScheme)) - - for _, obj := range []client.Object{domain, gatewayClass, leftover, foreign, handedOver, solverRoute, solverFilter} { - obj.SetUID(uuid.NewUUID()) - obj.SetCreationTimestamp(metav1.Now()) - } - downstreamGateway.SetCreationTimestamp(metav1.Now()) - - fakeUpstreamClient := fake.NewClientBuilder().WithScheme(testScheme). - WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, leftover, foreign). - WithStatusSubresource(upstreamGateway).Build() - fakeDownstreamClient := fake.NewClientBuilder().WithScheme(testScheme). - WithObjects(downstreamGateway, handedOver, solverRoute, solverFilter). - WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}).Build() - - reconciler := &GatewayReconciler{ - mgr: &fakeMockManager{cl: fakeUpstreamClient}, - Config: testCfg, - DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, - } - downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", fakeUpstreamClient, fakeDownstreamClient) - - reconciler.prepareUpstreamGateway(upstreamGateway) - result, downstream := reconciler.ensureDownstreamGateway(ctx, "test", fakeUpstreamClient, upstreamGateway, downstreamStrategy) - require.NoError(t, result.Err) - _, err := result.Complete(ctx) - require.NoError(t, err) - - certErr := fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &cmv1.Certificate{}) - if tt.wantCertificate { - assert.NoError(t, certErr, "cert-manager takes the hostname back once the Secret is due for renewal") - } else { - assert.True(t, apierrors.IsNotFound(certErr), "a fresh handed-over Secret is not reissued on rollback") - } - assert.Equal(t, tt.wantListenerAdmit, gatewayutil.GetListenerByName(downstream.Spec.Listeners, listenerName) != nil) - - assert.True(t, apierrors.IsNotFound(fakeUpstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{})), "the gateway's TLSCertificate is removed on rollback") - assert.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: "foreign"}, &certificatesv1alpha1.TLSCertificate{}), "a TLSCertificate the gateway does not control stays") - assert.True(t, apierrors.IsNotFound(fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: solverName}, &gatewayv1.HTTPRoute{})), "leftover solver routes are removed") - assert.True(t, apierrors.IsNotFound(fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: solverName}, &envoygatewayv1alpha1.HTTPRouteFilter{}))) - - if tt.wantCertificate { - var legacy cmv1.Certificate - require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: legacyName}, &legacy)) - legacy.Status.Conditions = []cmv1.CertificateCondition{{Type: cmv1.CertificateConditionReady, Status: cmmeta.ConditionTrue}} - require.NoError(t, fakeDownstreamClient.Status().Update(ctx, &legacy)) - reconciler.cleanupCertificateServiceLeftovers(ctx, fakeUpstreamClient, upstreamGateway, downstreamGateway, fakeDownstreamClient) - var secret corev1.Secret - require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) - assert.NotContains(t, secret.Labels, tlsCertificateManagedLabel, "the hand-over marker goes once cert-manager has the Secret back") - } - }) - } -} - func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { testScheme := newCertificateServiceTestScheme(t) logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) @@ -1053,44 +1226,29 @@ func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "recovery", UID: uuid.NewUUID()}} downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) - const hostname = "custom.example.com" + const hostname = "*.shop.example.com" const gatewayName = "recovery-gw" const listenerName = gatewayv1.SectionName("https-hostname-0") const serviceNS = "certificates-system" secretName := listenerCertificateSecretName(gatewayName, listenerName) certName := tlsCertificateName(gatewayName, listenerName) + ca := newTestCA(t) - testCfg := config.NetworkServicesOperator{Gateway: config.GatewayConfig{ - DownstreamGatewayClassName: "test-suite", - DownstreamHostnameAccountingNamespace: "default", - TargetDomain: "test-suite.com", - DefaultListenerTLSSecretName: "wildcard-test-suite-tls", - IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, - ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer"}, - CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: serviceNS}, - }} + testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { - g.Spec.Listeners = append(g.Spec.Listeners, gatewayv1.Listener{ - Name: listenerName, Protocol: gatewayv1.HTTPSProtocolType, Port: DefaultHTTPSPort, - Hostname: ptr.To(gatewayv1.Hostname(hostname)), - TLS: &gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModeTerminate), Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ - gatewayv1.AnnotationKey(certificateIssuerTLSOption): "test-issuer", - }}, - }) - }) - domain := newDomain(upstreamNamespace.Name, hostname, func(d *networkingv1alpha.Domain) { - d.Spec.DomainName = hostname - apimeta.SetStatusCondition(&d.Status.Conditions, metav1.Condition{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue}) + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(listenerName, hostname)) }) + domain := dnsProvenDomain(upstreamNamespace.Name) gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} now := time.Now() - servingCrt, servingKey := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) - issuedCrt, issuedKey := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(90*24*time.Hour)) + servingCrt, servingKey := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + issuedCrt, issuedKey := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(90*24*time.Hour)) cert := &certificatesv1alpha1.TLSCertificate{ ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: certName}, - Spec: certificatesv1alpha1.TLSCertificateSpec{DNSNames: []certificatesv1alpha1.DNSName{hostname}, Issuance: certificatesv1alpha1.IssuanceModeAuto, SecretName: secretName}, + Spec: certificatesv1alpha1.TLSCertificateSpec{DNSNames: []certificatesv1alpha1.DNSName{hostname}, Issuance: certificatesv1alpha1.IssuanceModeDNS01, SecretName: secretName}, Status: certificatesv1alpha1.TLSCertificateStatus{ NotAfter: &metav1.Time{Time: now.Add(90 * 24 * time.Hour)}, ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, @@ -1120,45 +1278,19 @@ func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { }, }).Build() - reconciler := &GatewayReconciler{ - mgr: &fakeMockManager{cl: fakeUpstreamClient}, - Config: testCfg, - DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, - CertificateServiceReader: serviceClient, - } - downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", fakeUpstreamClient, fakeDownstreamClient) - reconciler.prepareUpstreamGateway(upstreamGateway) - - reconcile := func() *gatewayv1.Gateway { - var current gatewayv1.Gateway - require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), ¤t)) - result, _ := reconciler.ensureDownstreamGateway(ctx, "test", fakeUpstreamClient, ¤t, downstreamStrategy) - require.NoError(t, result.Err) - _, err := result.Complete(ctx) - require.NoError(t, err) - require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), ¤t)) - - var claims corev1.ConfigMapList - require.NoError(t, fakeDownstreamClient.List(ctx, &claims, client.InNamespace(testCfg.Gateway.DownstreamHostnameAccountingNamespace))) - for i := range claims.Items { - stampCreated(t, ctx, fakeDownstreamClient, &claims.Items[i]) - } - var downstreamGateways gatewayv1.GatewayList - require.NoError(t, fakeDownstreamClient.List(ctx, &downstreamGateways, client.InNamespace(downstreamNamespaceName))) - for i := range downstreamGateways.Items { - stampCreated(t, ctx, fakeDownstreamClient, &downstreamGateways.Items[i]) - } - return ¤t + h := &certificateServiceHarness{ + t: t, ctx: ctx, cfg: testCfg, upstream: fakeUpstreamClient, downstream: fakeDownstreamClient, service: serviceClient, roots: ca.pool, + gateway: client.ObjectKeyFromObject(upstreamGateway), namespace: downstreamNamespaceName, } before := counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed) - first := reconcile() + first, _, _ := h.reconcile() assertListenerRenewalBlocked(t, first, "keep trying") assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed)) forbidden = false - reconciler.certificateServiceFailures.Delete(upstreamGateway.UID) - second := reconcile() + h.reconciler.certificateServiceFailures.Delete(certificateServiceKey{gateway: upstreamGateway.UID, listener: listenerName}) + second, _, _ := h.reconcile() for _, ls := range second.Status.Listeners { if ls.Name == listenerName { assert.Nil(t, apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked), "a recovered renewal no longer reads as blocked") @@ -1171,23 +1303,31 @@ func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { removed := second.DeepCopy() removed.Spec.Listeners = removed.Spec.Listeners[:len(removed.Spec.Listeners)-1] - reconciler.forgetRemovedListeners(removed) + h.reconciler.forgetRemovedListeners(removed) var m dto.Metric require.NoError(t, certificateServiceFailuresTotal.WithLabelValues(upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed).Write(&m)) assert.Zero(t, m.GetCounter().GetValue(), "a removed listener's series is dropped") } -func TestCertificateServiceRejectionCountedPerTransition(t *testing.T) { +func TestCertificateServiceStateCountedPerTransition(t *testing.T) { r := &GatewayReconciler{} gw := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: "count", Name: "gw", UID: uuid.NewUUID()}} - before := counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected) - r.recordCertificateServiceRejection(gw, "https-0", true) - r.recordCertificateServiceRejection(gw, "https-0", true) - r.recordCertificateServiceRejection(gw, "https-0", true) - assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected)) - r.recordCertificateServiceRejection(gw, "https-0", false) - r.recordCertificateServiceRejection(gw, "https-0", true) - assert.Equal(t, before+2, counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected), "a new rejection after recovery counts again") + rejected := func() float64 { + return counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected) + } + overdue := func() float64 { + return counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRenewalOverdue) + } + before, beforeOverdue := rejected(), overdue() + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + assert.Equal(t, before+1, rejected()) + r.recordCertificateServiceState(gw, "https-0", "") + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + assert.Equal(t, before+2, rejected(), "a new failure after recovery counts again") + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRenewalOverdue) + assert.Equal(t, beforeOverdue+1, overdue(), "a different failure counts under its own reason") } // stampCreated gives an object the fake client created a creation timestamp, diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 4f9a849b..1d92fc2d 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -5,6 +5,7 @@ package controller import ( "context" "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "slices" @@ -16,7 +17,6 @@ import ( cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -101,9 +101,14 @@ type GatewayReconciler struct { // Config.Gateway.CertificateService.Enabled. CertificateServiceReader client.Reader - certificateServiceFailures sync.Map - certificateServiceRejections sync.Map - certificateServiceListeners sync.Map + // CertificateServiceRoots are the roots an issued chain must verify + // against when Config.Gateway.CertificateService.VerifyChain is set. Nil + // means the system roots. + CertificateServiceRoots *x509.CertPool + + certificateServiceFailures sync.Map + certificateServiceStates sync.Map + certificateServiceListeners sync.Map } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -316,25 +321,14 @@ func (r *GatewayReconciler) ensureDownstreamGateway( // reissueFailedCertificate): this path is read-only and gates on any // unusable cert, re-issuance mutates and fires only on cert-manager // hard-fail (LastFailureTime). See #260. - var listenerCertHealth map[gatewayv1.SectionName]listenerCertStatus - if r.Config.Gateway.CertificateService.Enabled { - listenerCertHealth = r.evaluateListenerTLSCertificateHealth( - ctx, - upstreamClient, - downstreamClient, - downstreamGateway.Namespace, - upstreamGateway, - claimedHostnames, - ) - } else { - listenerCertHealth = r.evaluateListenerCertHealth( - ctx, - downstreamClient, - downstreamGateway.Namespace, - upstreamGateway, - claimedHostnames, - ) - } + listenerCertHealth := r.evaluateListenerCertHealth( + ctx, + upstreamClient, + downstreamClient, + downstreamGateway.Namespace, + upstreamGateway, + claimedHostnames, + ) desiredDownstreamGateway := r.getDesiredDownstreamGateway( ctx, @@ -390,12 +384,11 @@ func (r *GatewayReconciler) ensureDownstreamGateway( // The service is a dependency of issuance, not of routing: whatever it // does, DNS, status and routes below still reconcile, and a listener // that could not be served tells the customer why. - certResult, issues := r.ensureListenerTLSCertificates( + serviceResult, issues := r.ensureListenerTLSCertificates( ctx, upstreamClient, upstreamGateway, downstreamGateway, - downstreamClient, downstreamStrategy, claimedHostnames, ) @@ -412,20 +405,21 @@ func (r *GatewayReconciler) ensureDownstreamGateway( } listenerCertHealth[name] = status } - certificateServiceRequeue = certResult.RequeueAfter + certificateServiceRequeue = serviceResult.RequeueAfter } else { - certResult := r.ensureListenerCertificates( - ctx, - upstreamGateway, - downstreamGateway, - downstreamClient, - downstreamStrategy, - claimedHostnames, - ) - if certResult.ShouldReturn() { - return certResult.Merge(result), nil - } - r.cleanupCertificateServiceLeftovers(ctx, upstreamClient, upstreamGateway, downstreamGateway, downstreamClient) + clearCertificateServiceFailing(upstreamGateway) + } + + certResult := r.ensureListenerCertificates( + ctx, + upstreamGateway, + downstreamGateway, + downstreamClient, + downstreamStrategy, + claimedHostnames, + ) + if certResult.ShouldReturn() { + return certResult.Merge(result), nil } dnsResult := r.ensureDownstreamGatewayDNSEndpoints( @@ -569,6 +563,7 @@ func clearListenerCertMetrics(namespace, name string) { // is left out so it is never gated. func (r *GatewayReconciler) evaluateListenerCertHealth( ctx context.Context, + upstreamClient client.Client, downstreamClient client.Client, downstreamNamespace string, upstreamGateway *gatewayv1.Gateway, @@ -601,58 +596,53 @@ func (r *GatewayReconciler) evaluateListenerCertHealth( continue } - status := r.listenerCertHealth(ctx, downstreamClient, downstreamNamespace, upstreamGateway.Name, l.Name, hostname, now) + var status listenerCertStatus + if _, service := r.listenerUsesCertificateService(l, claimedHostnames); service { + status = r.listenerTLSCertificateHealth(ctx, upstreamClient, downstreamClient, downstreamNamespace, upstreamGateway, l.Name, hostname, now) + } else { + status = r.listenerCertHealth(ctx, downstreamClient, downstreamNamespace, upstreamGateway.Name, l.Name, hostname, now) + } health[l.Name] = status - recordListenerCertHealth(logger, upstreamGateway, l.Name, hostname, status) + // Mark this listener as managed regardless of its health, so the + // SLI ratio (withheld / managed) can be computed fleet-wide. + gatewayListenerCertManaged.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, string(l.Name), hostname, + ).Set(1) + + if !status.healthy { + logArgs := []any{ + "listener", l.Name, "hostname", hostname, + "reason", status.reason, "message", status.message, + } + // Include the expiry timestamp when available so log queries can + // identify exactly when the cert stopped being valid. + if status.notAfter != nil { + logArgs = append(logArgs, "cert_not_after", status.notAfter.UTC().Format(time.RFC3339)) + } + logger.Info("listener certificate unhealthy", logArgs...) + + gatewayListenerCertWithheld.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, + string(l.Name), hostname, string(status.reason), + ).Set(1) + gatewayListenerCertGatingTotal.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, + string(l.Name), hostname, string(status.reason), + ).Inc() + } else if status.notAfter != nil { + // Record the expiry timestamp for healthy certs so operators can + // alert before the next expiry rather than after. + gatewayListenerCertExpiryTime.WithLabelValues( + upstreamGateway.Namespace, upstreamGateway.Name, + string(l.Name), hostname, status.secretName, + ).Set(float64(status.notAfter.Unix())) + } } return health } -// recordListenerCertHealth publishes one listener's certificate health as -// metrics and logs, the same way for every issuance path. -func recordListenerCertHealth( - logger logr.Logger, - upstreamGateway *gatewayv1.Gateway, - listenerName gatewayv1.SectionName, - hostname string, - status listenerCertStatus, -) { - gatewayListenerCertManaged.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, string(listenerName), hostname, - ).Set(1) - - if !status.healthy { - logArgs := []any{ - "listener", listenerName, "hostname", hostname, - "reason", status.reason, "message", status.message, - } - // Include the expiry timestamp when available so log queries can - // identify exactly when the cert stopped being valid. - if status.notAfter != nil { - logArgs = append(logArgs, "cert_not_after", status.notAfter.UTC().Format(time.RFC3339)) - } - logger.Info("listener certificate unhealthy", logArgs...) - - gatewayListenerCertWithheld.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, - string(listenerName), hostname, string(status.reason), - ).Set(1) - gatewayListenerCertGatingTotal.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, - string(listenerName), hostname, string(status.reason), - ).Inc() - } else if status.notAfter != nil { - // Record the expiry timestamp for healthy certs so operators can - // alert before the next expiry rather than after. - gatewayListenerCertExpiryTime.WithLabelValues( - upstreamGateway.Namespace, upstreamGateway.Name, - string(listenerName), hostname, status.secretName, - ).Set(float64(status.notAfter.Unix())) - } -} - // listenerCertHealth reports whether a single listener's certificate can be // used to serve HTTPS right now, covering every way it can be unusable: still // being issued, not yet valid, expired, or missing. @@ -674,9 +664,6 @@ func (r *GatewayReconciler) listenerCertHealth( var cert cmv1.Certificate if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: certName}, &cert); err != nil { if apierrors.IsNotFound(err) { - if status, handedOver, _ := handedOverSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now); handedOver && status.healthy { - return status - } return listenerCertStatus{ reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certIssuanceFailingMessage(hostname), @@ -1047,6 +1034,9 @@ func (r *GatewayReconciler) ensureListenerCertificates( if hasSharedSecret && (strings.HasSuffix(hostname, wildcardSuffix) || hostname == r.Config.Gateway.TargetDomain) { continue } + if _, service := r.listenerUsesCertificateService(l, claimedHostnames); service { + continue + } // Apply ClusterIssuerMap translation first — this is the admin-level // override (e.g. mapping the `auto` sentinel to a real ClusterIssuer @@ -1086,10 +1076,6 @@ func (r *GatewayReconciler) ensureListenerCertificates( isNew := cert.CreationTimestamp.IsZero() if isNew { - if status, handedOver, renewalDue := handedOverSecretHealth(ctx, downstreamClient, downstreamGateway.Namespace, secretName, hostname, time.Now()); handedOver && status.healthy && !renewalDue { - logger.Info("keeping certificate-service issued Secret until it is due for renewal", "secret", secretName, "hostname", hostname) - continue - } if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, cert); err != nil { result.Err = fmt.Errorf("failed to set strategy reference on Certificate %s: %w", certName, err) return result @@ -1846,10 +1832,7 @@ func (r *GatewayReconciler) finalizeGateway( gatewayProgrammedTotal.DeleteLabelValues(upstreamGateway.Namespace, upstreamGateway.Name) // Clear this gateway's cert-health series now that it is gone. clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) - r.certificateServiceFailures.Delete(upstreamGateway.UID) - r.certificateServiceRejections.Delete(upstreamGateway.UID) - r.certificateServiceListeners.Delete(upstreamGateway.UID) - certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) + r.forgetGateway(upstreamGateway) // Clean up DNS records created by this gateway if r.Config.Gateway.EnableDNSIntegration { diff --git a/internal/controller/gateway_downstream_certificate_solver_controller.go b/internal/controller/gateway_downstream_certificate_solver_controller.go index edd2a260..717c8f3a 100644 --- a/internal/controller/gateway_downstream_certificate_solver_controller.go +++ b/internal/controller/gateway_downstream_certificate_solver_controller.go @@ -3,14 +3,18 @@ package controller import ( "context" "fmt" + "net/http" + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -156,13 +160,94 @@ func (r *GatewayDownstreamCertificateSolverReconciler) Reconcile(ctx context.Con logger.Info("Successfully retrieved key and token from cert-manager Challenge for downstream certificate") - if err := ensureHTTP01SolverRoutes(ctx, cl, r.DownstreamCluster.GetScheme(), challenge, &gateway, http01SolverRoute{ - name: challenge.GetName(), - token: token, - key: key, - }); err != nil { - return ctrl.Result{}, err + // Create HTTPRouteFilter with inline direct response + httpRouteFilter := &envoygatewayv1alpha1.HTTPRouteFilter{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: challenge.GetName(), + }, + } + + if err := controllerutil.SetControllerReference(challenge, httpRouteFilter, r.DownstreamCluster.GetScheme()); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set controller reference on HTTPRouteFilter: %w", err) + } + + result, err := controllerutil.CreateOrUpdate(ctx, cl, httpRouteFilter, func() error { + httpRouteFilter.Labels = map[string]string{ + "meta.datumapis.com/http01-solver": labelValueTrue, + } + httpRouteFilter.Spec = envoygatewayv1alpha1.HTTPRouteFilterSpec{ + DirectResponse: &envoygatewayv1alpha1.HTTPDirectResponseFilter{ + ContentType: ptr.To("text/plain"), + StatusCode: ptr.To(http.StatusOK), + Body: &envoygatewayv1alpha1.CustomResponseBody{ + Type: ptr.To(envoygatewayv1alpha1.ResponseValueTypeInline), + Inline: ptr.To(key), + }, + }, + } + + return nil + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create or update HTTPRouteFilter: %w", err) + } + logger.Info("HTTPRouteFilter reconciled", "result", result) + + // Attach HTTPRoute at expected path with direct response filter + httpRoute := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: challenge.GetName(), + }, + } + + if err := controllerutil.SetControllerReference(challenge, httpRoute, r.DownstreamCluster.GetScheme()); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set controller reference on HTTPRoute: %w", err) + } + + result, err = controllerutil.CreateOrUpdate(ctx, cl, httpRoute, func() error { + httpRoute.Labels = map[string]string{ + "meta.datumapis.com/http01-solver": labelValueTrue, + } + httpRoute.Spec = gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{ + { + Name: gatewayv1.ObjectName(gateway.GetName()), + }, + }, + }, + Rules: []gatewayv1.HTTPRouteRule{ + { + Matches: []gatewayv1.HTTPRouteMatch{ + { + Path: &gatewayv1.HTTPPathMatch{ + Type: ptr.To(gatewayv1.PathMatchExact), + Value: ptr.To(fmt.Sprintf("/.well-known/acme-challenge/%s", token)), + }, + }, + }, + Filters: []gatewayv1.HTTPRouteFilter{ + { + Type: gatewayv1.HTTPRouteFilterExtensionRef, + ExtensionRef: &gatewayv1.LocalObjectReference{ + Group: envoygatewayv1alpha1.GroupName, + Kind: envoygatewayv1alpha1.KindHTTPRouteFilter, + Name: gatewayv1.ObjectName(httpRouteFilter.GetName()), + }, + }, + }, + }, + }, + } + + return nil + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create or update HTTPRoute: %w", err) } + logger.Info("HTTPRoute reconciled", "result", result) return ctrl.Result{}, nil } diff --git a/internal/controller/http01_solver.go b/internal/controller/http01_solver.go deleted file mode 100644 index 8f40b04d..00000000 --- a/internal/controller/http01_solver.go +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package controller - -import ( - "context" - "fmt" - "maps" - "net/http" - - envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" -) - -const http01SolverLabel = "meta.datumapis.com/http01-solver" - -// http01SolverRoute names one ACME HTTP-01 answer to publish on a downstream -// gateway: GET /.well-known/acme-challenge/ returns . -type http01SolverRoute struct { - name string - token string - key string - hostnames []gatewayv1.Hostname - labels map[string]string -} - -func (s http01SolverRoute) objectLabels() map[string]string { - labels := map[string]string{http01SolverLabel: labelValueTrue} - maps.Copy(labels, s.labels) - return labels -} - -func (s http01SolverRoute) path() string { - return fmt.Sprintf("/.well-known/acme-challenge/%s", s.token) -} - -// ensureHTTP01SolverRoutes creates or updates the HTTPRouteFilter and HTTPRoute -// that answer a challenge on the gateway, both controlled by owner so they -// leave with it. -func ensureHTTP01SolverRoutes( - ctx context.Context, - cl client.Client, - scheme *runtime.Scheme, - owner client.Object, - gateway *gatewayv1.Gateway, - solver http01SolverRoute, -) error { - logger := log.FromContext(ctx) - - httpRouteFilter := &envoygatewayv1alpha1.HTTPRouteFilter{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: gateway.Namespace, - Name: solver.name, - }, - } - - result, err := controllerutil.CreateOrUpdate(ctx, cl, httpRouteFilter, func() error { - if err := controllerutil.SetControllerReference(owner, httpRouteFilter, scheme); err != nil { - return fmt.Errorf("failed to set controller reference on HTTPRouteFilter: %w", err) - } - httpRouteFilter.Labels = solver.objectLabels() - httpRouteFilter.Spec = envoygatewayv1alpha1.HTTPRouteFilterSpec{ - DirectResponse: &envoygatewayv1alpha1.HTTPDirectResponseFilter{ - ContentType: ptr.To("text/plain"), - StatusCode: ptr.To(http.StatusOK), - Body: &envoygatewayv1alpha1.CustomResponseBody{ - Type: ptr.To(envoygatewayv1alpha1.ResponseValueTypeInline), - Inline: ptr.To(solver.key), - }, - }, - } - return nil - }) - if err != nil { - return fmt.Errorf("failed to create or update HTTPRouteFilter: %w", err) - } - logger.Info("HTTPRouteFilter reconciled", "name", solver.name, "result", result) - - httpRoute := &gatewayv1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: gateway.Namespace, - Name: solver.name, - }, - } - - result, err = controllerutil.CreateOrUpdate(ctx, cl, httpRoute, func() error { - if err := controllerutil.SetControllerReference(owner, httpRoute, scheme); err != nil { - return fmt.Errorf("failed to set controller reference on HTTPRoute: %w", err) - } - httpRoute.Labels = solver.objectLabels() - httpRoute.Spec = gatewayv1.HTTPRouteSpec{ - Hostnames: solver.hostnames, - CommonRouteSpec: gatewayv1.CommonRouteSpec{ - ParentRefs: []gatewayv1.ParentReference{ - { - Name: gatewayv1.ObjectName(gateway.GetName()), - }, - }, - }, - Rules: []gatewayv1.HTTPRouteRule{ - { - Matches: []gatewayv1.HTTPRouteMatch{ - { - Path: &gatewayv1.HTTPPathMatch{ - Type: ptr.To(gatewayv1.PathMatchExact), - Value: ptr.To(solver.path()), - }, - }, - }, - Filters: []gatewayv1.HTTPRouteFilter{ - { - Type: gatewayv1.HTTPRouteFilterExtensionRef, - ExtensionRef: &gatewayv1.LocalObjectReference{ - Group: envoygatewayv1alpha1.GroupName, - Kind: envoygatewayv1alpha1.KindHTTPRouteFilter, - Name: gatewayv1.ObjectName(httpRouteFilter.GetName()), - }, - }, - }, - }, - }, - } - return nil - }) - if err != nil { - return fmt.Errorf("failed to create or update HTTPRoute: %w", err) - } - logger.Info("HTTPRoute reconciled", "name", solver.name, "result", result) - - return nil -} - -// deleteHTTP01SolverRoutes removes the solver objects published under name in -// the namespace, tolerating ones that are already gone. -func deleteHTTP01SolverRoutes(ctx context.Context, cl client.Client, namespace, name string) error { - meta := metav1.ObjectMeta{Namespace: namespace, Name: name} - for _, obj := range []client.Object{ - &gatewayv1.HTTPRoute{ObjectMeta: meta}, - &envoygatewayv1alpha1.HTTPRouteFilter{ObjectMeta: meta}, - } { - if err := cl.Delete(ctx, obj); client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to delete solver %T %s/%s: %w", obj, namespace, name, err) - } - } - return nil -} diff --git a/internal/controller/http01_solver_test.go b/internal/controller/http01_solver_test.go deleted file mode 100644 index 54797054..00000000 --- a/internal/controller/http01_solver_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package controller - -import ( - "context" - "testing" - - envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" -) - -func TestEnsureAndDeleteHTTP01SolverRoutes(t *testing.T) { - testScheme := runtime.NewScheme() - require.NoError(t, scheme.AddToScheme(testScheme)) - require.NoError(t, gatewayv1.Install(testScheme)) - require.NoError(t, envoygatewayv1alpha1.AddToScheme(testScheme)) - - gateway := &gatewayv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{Namespace: "ns-1", Name: "gw", UID: uuid.NewUUID()}, - } - cl := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(gateway).Build() - ctx := context.Background() - - solver := http01SolverRoute{ - name: "gw-acme-abc", - token: "the-token", - key: "the-key", - labels: map[string]string{tlsCertificateSolverLabel: "gw-https-0"}, - } - - require.NoError(t, ensureHTTP01SolverRoutes(ctx, cl, testScheme, gateway, gateway, solver)) - require.NoError(t, ensureHTTP01SolverRoutes(ctx, cl, testScheme, gateway, gateway, solver), "re-running is idempotent") - - key := client.ObjectKey{Namespace: "ns-1", Name: "gw-acme-abc"} - - var route gatewayv1.HTTPRoute - require.NoError(t, cl.Get(ctx, key, &route)) - assert.Equal(t, labelValueTrue, route.Labels[http01SolverLabel]) - assert.Equal(t, "gw-https-0", route.Labels[tlsCertificateSolverLabel]) - require.Len(t, route.Spec.ParentRefs, 1) - assert.Equal(t, gatewayv1.ObjectName("gw"), route.Spec.ParentRefs[0].Name) - require.Len(t, route.Spec.Rules, 1) - assert.Equal(t, "/.well-known/acme-challenge/the-token", ptr.Deref(route.Spec.Rules[0].Matches[0].Path.Value, "")) - assert.Equal(t, gatewayv1.ObjectName("gw-acme-abc"), route.Spec.Rules[0].Filters[0].ExtensionRef.Name) - owner := metav1.GetControllerOf(&route) - require.NotNil(t, owner) - assert.Equal(t, gateway.UID, owner.UID) - - var filter envoygatewayv1alpha1.HTTPRouteFilter - require.NoError(t, cl.Get(ctx, key, &filter)) - assert.Equal(t, "the-key", ptr.Deref(filter.Spec.DirectResponse.Body.Inline, "")) - assert.Equal(t, labelValueTrue, filter.Labels[http01SolverLabel]) - assert.Equal(t, gateway.UID, metav1.GetControllerOf(&filter).UID) - - require.NoError(t, deleteHTTP01SolverRoutes(ctx, cl, "ns-1", "gw-acme-abc")) - assert.True(t, apierrors.IsNotFound(cl.Get(ctx, key, &gatewayv1.HTTPRoute{}))) - assert.True(t, apierrors.IsNotFound(cl.Get(ctx, key, &envoygatewayv1alpha1.HTTPRouteFilter{}))) - assert.NoError(t, deleteHTTP01SolverRoutes(ctx, cl, "ns-1", "gw-acme-abc"), "deleting again is a no-op") -} diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go index dad81d48..cfb3960c 100644 --- a/internal/controller/httpproxy_certificate_service.go +++ b/internal/controller/httpproxy_certificate_service.go @@ -26,10 +26,9 @@ import ( const certificateProvisioningMessage = "We're provisioning and applying a certificate to this hostname - it may take a few minutes" -// tlsCertificateReadyCondition derives a hostname's CertificateReady condition -// from its TLSCertificate in the project control plane. A listener still served -// by a cert-manager Certificate that is not yet due for renewal has no -// TLSCertificate, so that Certificate answers for it until the switch. +// tlsCertificateReadyCondition derives a wildcard hostname's CertificateReady +// condition from its TLSCertificate in the project control plane, the Secret +// serving it, and what the gateway reports about its listener. func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( ctx context.Context, upstreamClient client.Client, @@ -37,7 +36,6 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( downstreamNamespace string, upstreamNamespace string, certName string, - legacyName string, secretName string, hostname string, listenerConditions []metav1.Condition, @@ -48,76 +46,62 @@ func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( ObservedGeneration: generation, } + blocked := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateRenewalBlocked) + if blocked != nil && blocked.Status != metav1.ConditionTrue { + blocked = nil + } + if blocked == nil { + if issuance := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateIssuanceBlocked); issuance != nil && issuance.Status == metav1.ConditionTrue { + blocked = issuance + } + } + var cert certificatesv1alpha1.TLSCertificate err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace, Name: certName}, &cert) switch { - case err == nil: - condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) - serving := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()) - blocked := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateRenewalBlocked) - if blocked != nil && blocked.Status != metav1.ConditionTrue { - blocked = nil - } - if blocked == nil { - if issuance := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateIssuanceBlocked); issuance != nil && issuance.Status == metav1.ConditionTrue { - blocked = issuance - } - } - - switch { - case serving.healthy && (condition.Status != metav1.ConditionTrue || blocked != nil): - condition.Status = metav1.ConditionTrue - condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued - condition.Message = "Certificate is ready; a renewal is in progress" - if blocked != nil { - condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing - condition.Message = "Certificate is ready but cannot be renewed: " + blocked.Message - } else if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { - condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing - condition.Message = "Certificate is ready but cannot be renewed" - if accepted.Message != "" { - condition.Message += ": " + accepted.Message - } - } - case !serving.healthy && blocked != nil: - condition.Status = metav1.ConditionFalse + case apierrors.IsNotFound(err): + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = certificateProvisioningMessage + if blocked != nil { condition.Reason = networkingv1alpha.CertificateReadyReasonProvisioningFailed condition.Message = blocked.Message - case !serving.healthy && condition.Status == metav1.ConditionTrue: - condition.Status = metav1.ConditionFalse - condition.Reason = networkingv1alpha.CertificateReadyReasonPending - condition.Message = "The certificate has been issued and is being applied to this hostname" } return condition - case !apierrors.IsNotFound(err): + case err != nil: condition.Status = metav1.ConditionUnknown condition.Reason = networkingv1alpha.CertificateReadyReasonPending condition.Message = fmt.Sprintf("Failed to get certificate: %v", err) return condition } - legacy := newUnstructuredForGVK(certificateGVK) - if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: legacyName}, legacy); err == nil { - ready, readyErr := isCertificateReady(legacy) - switch { - case readyErr != nil: - condition.Status = metav1.ConditionUnknown - condition.Reason = networkingv1alpha.CertificateReadyReasonPending - condition.Message = fmt.Sprintf("Failed to check certificate status: %v", readyErr) - case ready: - condition.Status = metav1.ConditionTrue - condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued - condition.Message = "Certificate is ready" - default: - condition.Status = metav1.ConditionFalse - condition.Reason, condition.Message = getCertificateReadyConditionReason(legacy) + condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) + serving, _ := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()) + + switch { + case serving.healthy && (condition.Status != metav1.ConditionTrue || blocked != nil): + condition.Status = metav1.ConditionTrue + condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued + condition.Message = "Certificate is ready; a renewal is in progress" + if blocked != nil { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed: " + blocked.Message + } else if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed" + if accepted.Message != "" { + condition.Message += ": " + accepted.Message + } } - return condition + case !serving.healthy && blocked != nil: + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonProvisioningFailed + condition.Message = blocked.Message + case !serving.healthy && condition.Status == metav1.ConditionTrue: + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = "The certificate has been issued and is being applied to this hostname" } - - condition.Status = metav1.ConditionFalse - condition.Reason = networkingv1alpha.CertificateReadyReasonPending - condition.Message = certificateProvisioningMessage return condition } diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go index 2841c301..d0efdec7 100644 --- a/internal/controller/httpproxy_certificate_service_test.go +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -35,20 +35,23 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) require.NoError(t, certificatesv1alpha1.AddToScheme(testScheme)) + const wildcard = "*.shop.example.com" ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-ns", UID: types.UID("ns-uid")}} downstreamNamespaceName := "ns-" + string(ns.UID) - gateway := &gatewayv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name}, - Spec: gatewayv1.GatewaySpec{Listeners: []gatewayv1.Listener{{ - Name: "https-hostname-0", - Protocol: gatewayv1.HTTPSProtocolType, - Hostname: ptr.To(gatewayv1.Hostname("app.example.com")), - }}}, + newGatewayFor := func(hostname string) *gatewayv1.Gateway { + return &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name}, + Spec: gatewayv1.GatewaySpec{Listeners: []gatewayv1.Listener{{ + Name: "https-hostname-0", + Protocol: gatewayv1.HTTPSProtocolType, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + }}}, + } } httpProxy := &networkingv1alpha.HTTPProxy{ ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name, Generation: 3}, - Spec: networkingv1alpha.HTTPProxySpec{Hostnames: []gatewayv1.Hostname{"app.example.com"}}, + Spec: networkingv1alpha.HTTPProxySpec{Hostnames: []gatewayv1.Hostname{wildcard, "app.example.com"}}, } certName := tlsCertificateName("my-proxy", "https-hostname-0") @@ -66,7 +69,7 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { return cert } - crt, key := generateTLSKeyPair(t, "app.example.com", time.Now().Add(-time.Hour), time.Now().Add(30*24*time.Hour)) + crt, key := generateTLSKeyPair(t, wildcard, time.Now().Add(-time.Hour), time.Now().Add(30*24*time.Hour)) servingSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName("my-proxy", "https-hostname-0")}, Type: corev1.SecretTypeTLS, @@ -75,6 +78,7 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { tests := []struct { name string + hostname string upstream []client.Object mutate func(*certificatesv1alpha1.TLSCertificate) listenerConditions []metav1.Condition @@ -120,11 +124,11 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue, Reason: "OrderInFlight"}, ), nil}[:1], mutate: func(c *certificatesv1alpha1.TLSCertificate) { - c.Status.RequiredDNSRecords = []certificatesv1alpha1.RequiredDNSRecord{{Name: "_acme-challenge.app.example.com", Type: "CNAME", Content: "abc.acme-dns.example.net", Purpose: certificatesv1alpha1.DNSRecordPurposeCertificate}} + c.Status.RequiredDNSRecords = []certificatesv1alpha1.RequiredDNSRecord{{Name: "_acme-challenge.shop.example.com", Type: "CNAME", Content: "abc.acme-dns.example.net", Purpose: certificatesv1alpha1.DNSRecordPurposeCertificate}} }, wantStatus: metav1.ConditionFalse, wantReason: networkingv1alpha.CertificateReadyReasonChallengeInProgress, - wantMessage: certificateProvisioningMessage + ". Publish these DNS records to continue: _acme-challenge.app.example.com CNAME abc.acme-dns.example.net", + wantMessage: certificateProvisioningMessage + ". Publish these DNS records to continue: _acme-challenge.shop.example.com CNAME abc.acme-dns.example.net", }, { name: "not Ready yet but a serving downstream Secret keeps the hostname ready", @@ -149,22 +153,22 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, downstream: []client.Object{servingSecret}, listenerConditions: []metav1.Condition{{ - Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", + Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for *.shop.example.com just now", }}, wantStatus: metav1.ConditionTrue, wantReason: networkingv1alpha.CertificateReadyReasonRenewalFailing, - wantMessage: "Certificate is ready but cannot be renewed: We couldn't request a TLS certificate for app.example.com just now", + wantMessage: "Certificate is ready but cannot be renewed: We couldn't request a TLS certificate for *.shop.example.com just now", }, { name: "Ready but the issued Secret could not be taken and nothing serves is a provisioning failure", upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, listenerConditions: []metav1.Condition{{ - Type: listenerConditionCertificateIssuanceBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonIssuanceFailing, Message: "We couldn't request a TLS certificate for app.example.com just now", + Type: listenerConditionCertificateIssuanceBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonIssuanceFailing, Message: "We couldn't request a TLS certificate for *.shop.example.com just now", }}, downstream: nil, wantStatus: metav1.ConditionFalse, wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, - wantMessage: "We couldn't request a TLS certificate for app.example.com just now", + wantMessage: "We couldn't request a TLS certificate for *.shop.example.com just now", }, { name: "Ready but not yet mirrored is pending, not issued", @@ -174,11 +178,22 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { wantMessage: "The certificate has been issued and is being applied to this hostname", }, { - name: "no TLSCertificate yet but a ready legacy Certificate answers", + name: "an exact hostname is answered by its cert-manager Certificate, never a TLSCertificate", + hostname: "app.example.com", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain"})}, downstream: []client.Object{legacyReady()}, wantStatus: metav1.ConditionTrue, wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, }, + { + name: "no TLSCertificate because the request was refused is a provisioning failure", + listenerConditions: []metav1.Condition{{ + Type: listenerConditionCertificateIssuanceBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonIssuanceFailing, Message: "A TLS certificate cannot be issued for *.shop.example.com: names under datum.net are denied", + }}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, + wantMessage: "A TLS certificate cannot be issued for *.shop.example.com: names under datum.net are denied", + }, { name: "nothing requested yet is Pending", wantStatus: metav1.ConditionFalse, @@ -205,13 +220,17 @@ func TestBuildCertificateStatusesCertificateService(t *testing.T) { DownstreamCluster: &clusterWithClient{c: downstreamClient, scheme: testScheme}, } - gw := gateway.DeepCopy() + hostname := tt.hostname + if hostname == "" { + hostname = wildcard + } + gw := newGatewayFor(hostname) if tt.listenerConditions != nil { gw.Status.Listeners = []gatewayv1.ListenerStatus{{Name: "https-hostname-0", Conditions: tt.listenerConditions}} } got := r.buildCertificateStatuses(context.Background(), upstreamClient, "local", gw, httpProxy) require.Len(t, got, 1) - assert.Equal(t, "app.example.com", got[0].Hostname) + assert.Equal(t, hostname, got[0].Hostname) cond := apimeta.FindStatusCondition(got[0].Conditions, networkingv1alpha.HostnameConditionCertificateReady) require.NotNil(t, cond) assert.Equal(t, tt.wantStatus, cond.Status) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 4891210f..f95b0d4e 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1516,8 +1516,8 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( certName := resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-%s", gateway.Name, l.Name)) - if r.Config.Gateway.CertificateService.Enabled { - apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), certName, listenerCertificateSecretName(gateway.Name, l.Name), hostname, listenerStatusConditions(gateway, l.Name), httpProxy.Generation)) + if r.Config.Gateway.CertificateService.Enabled && isSingleLabelWildcard(hostname) { + apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), listenerCertificateSecretName(gateway.Name, l.Name), hostname, listenerStatusConditions(gateway, l.Name), httpProxy.Generation)) statuses = append(statuses, hs) continue } diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go index 234892aa..84438d79 100644 --- a/internal/controller/metrics.go +++ b/internal/controller/metrics.go @@ -75,27 +75,39 @@ var ( []string{jsonKeyNamespace, jsonKeyName}, ) - // gatewayListenerCertWithheld is 1 for each upstream Gateway listener that NSO - // is currently withholding from the downstream because its TLS certificate is - // unusable. The series for a listener is removed when the listener recovers, - // is removed from the Gateway, or the Gateway is deleted. - // - // Use sum(nso_gateway_listener_cert_withheld) to count how many listeners are - // currently dark across the fleet, or filter by namespace/name/listener/hostname - // to find the specific affected object during an incident. // certificateServiceFailuresTotal counts every certificate-service step that - // could not do its work for a listener, by reason, whether or not the - // listener still serves a certificate. A hostname can serve for weeks on its - // previous certificate while every attempt to replace it fails; this is the - // signal that says so before the expiry does. + // could not do its work for a wildcard listener, and every transition of its + // TLSCertificate into a failing state, by reason, whether or not the listener + // still serves a certificate. A hostname can serve for weeks on its previous + // certificate while every attempt to replace it fails; this is the signal + // that says so before the expiry does. certificateServiceFailuresTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "nso_certificate_service_failures_total", - Help: "Total certificate-service steps that failed for a Gateway listener, by reason.", + Help: "Total certificate-service failures for a Gateway listener, by reason.", }, []string{jsonKeyNamespace, jsonKeyName, metricLabelListener, metricLabelReason}, ) + // certificateServiceListenerFailing is 1 for each wildcard listener whose + // certificate cannot currently be issued or renewed, labelled with why. The + // series goes when the listener recovers, leaves the service, or is removed. + certificateServiceListenerFailing = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "nso_certificate_service_listener_failing", + Help: "1 while a Gateway listener's certificate-service issuance or renewal is failing, by reason.", + }, + []string{jsonKeyNamespace, jsonKeyName, metricLabelListener, metricLabelReason}, + ) + + // gatewayListenerCertWithheld is 1 for each upstream Gateway listener that NSO + // is currently withholding from the downstream because its TLS certificate is + // unusable. The series for a listener is removed when the listener recovers, + // is removed from the Gateway, or the Gateway is deleted. + // + // Use sum(nso_gateway_listener_cert_withheld) to count how many listeners are + // currently dark across the fleet, or filter by namespace/name/listener/hostname + // to find the specific affected object during an incident. gatewayListenerCertWithheld = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "nso_gateway_listener_cert_withheld", From 3d45ec3a5bb33d85cfd914eac111923e5010aa87 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 2 Oct 2026 18:16:24 -0500 Subject: [PATCH 7/8] feat: alert on failing wildcard certificates Add CertificateServiceIssuanceFailing, a warning raised when a wildcard listener's certificate has failed to issue or renew for two hours. It reads the failing-listener gauge, so a failure counted once still alerts, and the failure counter, so a step that keeps failing and retrying does too. Key changes: - add the rule in its own group so existing alert state is untouched - add promtool tests for lasting, retrying, cleared and young failures - add a runbook section mapping each reason to where it failed --- config/telemetry/alerts/gateways.yaml | 21 ++++++ docs/runbooks/gateway-tls-certificates.md | 32 ++++++++ .../gateways/certificate-service-rules.yaml | 21 ++++++ .../gateways/certificate-service-tests.yaml | 74 +++++++++++++++++++ 4 files changed, 148 insertions(+) create mode 100644 test/prometheus-rules/gateways/certificate-service-rules.yaml create mode 100644 test/prometheus-rules/gateways/certificate-service-tests.yaml diff --git a/config/telemetry/alerts/gateways.yaml b/config/telemetry/alerts/gateways.yaml index 710a5fa4..c35a127a 100644 --- a/config/telemetry/alerts/gateways.yaml +++ b/config/telemetry/alerts/gateways.yaml @@ -173,3 +173,24 @@ spec: summary: "{{ $value }} edge listener(s) have every TLS certificate broken and cannot be protected" description: "The extension server left {{ $value }} edge listener(s) untouched because every certificate on them is broken. It never removes a listener entirely, so the edge will reject the configuration update for those listeners (EnvoyListenerUpdateRejected confirms it). This means the controller did not withhold the listener before it reached the edge. Check extension server logs for 'listeners_left_intact' and why the controller did not withhold the listener." runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#tlsbackstoplistenerallcertsbroken" + + - name: nso-certificate-service + interval: 30s + rules: + # Fires when a wildcard hostname's certificate cannot be issued or renewed + # through the certificate service. The hostname may still serve its previous + # certificate, so this fires weeks before that certificate expires. + - alert: CertificateServiceIssuanceFailing + expr: | + max by (namespace, name, listener, reason) (nso_certificate_service_listener_failing) > 0 + or + sum by (namespace, name, listener, reason) (increase(nso_certificate_service_failures_total[1h])) > 0 + for: 2h + labels: + severity: warning + service: network-services + team: sre + annotations: + summary: "Certificate for Gateway listener {{ $labels.namespace }}/{{ $labels.name }}/{{ $labels.listener }} cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener {{ $labels.listener }} on Gateway {{ $labels.name }} in namespace {{ $labels.namespace }} for over two hours (reason: {{ $labels.reason }}). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" diff --git a/docs/runbooks/gateway-tls-certificates.md b/docs/runbooks/gateway-tls-certificates.md index 5118827a..83699019 100644 --- a/docs/runbooks/gateway-tls-certificates.md +++ b/docs/runbooks/gateway-tls-certificates.md @@ -87,6 +87,38 @@ renewal depends on it. customer-driven gating event — no platform fix. If renewal is failing for a platform reason, fix the issuer / ACME path so cert-manager can renew. +## CertificateServiceIssuanceFailing + +**Meaning.** A wildcard hostname's certificate has not been issued or renewed by +the certificate service for over two hours. Only wildcard hostnames use the +service; exact hostnames stay on cert-manager and are covered by the alerts +above. + +**Impact.** If the listener still serves a certificate, none yet: it carries +`CertificateRenewalBlocked` and keeps serving until that certificate expires. If +it serves nothing, it carries `CertificateIssuanceBlocked` and the wildcard is +unavailable. + +**Diagnose.** The `reason` label says where it failed: + +| Reason | Where | +|---|---| +| `Rejected`, `Refused` | The service refused the request; the condition message carries its reason | +| `IssuanceFailed`, `NotReady` | The service accepted it but the ACME order failed or never completed | +| `RenewalOverdue` | The served certificate is past its renewal point and nothing newer arrived | +| `MaterialRefused`, `UntrustedChain`, `NamespaceRefused` | The operator refused what the service issued | +| `StepFailed`, `NotOwned` | The operator could not reach the service, or the request name is taken | + +Read the TLSCertificate in the project, named after the gateway and listener: + +```sh +kubectl -n get tlscertificates -o yaml +``` + +**Remediate.** A refusal or a missing DNS delegation record is for the customer. +An ACME failure, a refused or untrusted chain, or a step failure is a platform +fault in the certificate service or the operator's access to it. + ## TLSBackstopPruningChains **Meaning.** The extension server is actively dropping broken certificates from diff --git a/test/prometheus-rules/gateways/certificate-service-rules.yaml b/test/prometheus-rules/gateways/certificate-service-rules.yaml new file mode 100644 index 00000000..88eb890a --- /dev/null +++ b/test/prometheus-rules/gateways/certificate-service-rules.yaml @@ -0,0 +1,21 @@ +groups: +- name: nso-certificate-service + interval: 30s + rules: + # Fires when a wildcard hostname's certificate cannot be issued or renewed + # through the certificate service. The hostname may still serve its previous + # certificate, so this fires weeks before that certificate expires. + - alert: CertificateServiceIssuanceFailing + expr: | + max by (namespace, name, listener, reason) (nso_certificate_service_listener_failing) > 0 + or + sum by (namespace, name, listener, reason) (increase(nso_certificate_service_failures_total[1h])) > 0 + for: 2h + labels: + severity: warning + service: network-services + team: sre + annotations: + summary: "Certificate for Gateway listener {{ $labels.namespace }}/{{ $labels.name }}/{{ $labels.listener }} cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener {{ $labels.listener }} on Gateway {{ $labels.name }} in namespace {{ $labels.namespace }} for over two hours (reason: {{ $labels.reason }}). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" diff --git a/test/prometheus-rules/gateways/certificate-service-tests.yaml b/test/prometheus-rules/gateways/certificate-service-tests.yaml new file mode 100644 index 00000000..64d4f69b --- /dev/null +++ b/test/prometheus-rules/gateways/certificate-service-tests.yaml @@ -0,0 +1,74 @@ +rule_files: + - certificate-service-rules.yaml + +evaluation_interval: 30s + +tests: + # A renewal that keeps failing raises the alert after two hours, even though + # the failure was counted only once when it began. + - interval: 1m + input_series: + - series: 'nso_certificate_service_listener_failing{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="IssuanceFailed"}' + values: '1+0x180' + - series: 'nso_certificate_service_failures_total{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="IssuanceFailed"}' + values: '1+0x180' + alert_rule_test: + - eval_time: 150m + alertname: CertificateServiceIssuanceFailing + exp_alerts: + - exp_labels: + severity: warning + service: network-services + team: sre + namespace: customer-ns + name: my-gateway + listener: https-hostname-0 + reason: IssuanceFailed + exp_annotations: + summary: "Certificate for Gateway listener customer-ns/my-gateway/https-hostname-0 cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener https-hostname-0 on Gateway my-gateway in namespace customer-ns for over two hours (reason: IssuanceFailed). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" + + # A step that keeps failing and retrying raises the alert from the counter alone. + - interval: 1m + input_series: + - series: 'nso_certificate_service_failures_total{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="StepFailed"}' + values: '0+1x180' + alert_rule_test: + - eval_time: 150m + alertname: CertificateServiceIssuanceFailing + exp_alerts: + - exp_labels: + severity: warning + service: network-services + team: sre + namespace: customer-ns + name: my-gateway + listener: https-hostname-0 + reason: StepFailed + exp_annotations: + summary: "Certificate for Gateway listener customer-ns/my-gateway/https-hostname-0 cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener https-hostname-0 on Gateway my-gateway in namespace customer-ns for over two hours (reason: StepFailed). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" + + # A failure that cleared within the hour does not alert. + - interval: 1m + input_series: + - series: 'nso_certificate_service_listener_failing{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="NotReady"}' + values: '1+0x30 _x150' + - series: 'nso_certificate_service_failures_total{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="NotReady"}' + values: '1+0x180' + alert_rule_test: + - eval_time: 150m + alertname: CertificateServiceIssuanceFailing + exp_alerts: [] + + # A failure younger than two hours does not alert yet. + - interval: 1m + input_series: + - series: 'nso_certificate_service_listener_failing{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="Rejected"}' + values: '1+0x90' + alert_rule_test: + - eval_time: 90m + alertname: CertificateServiceIssuanceFailing + exp_alerts: [] From c6397c661d8ceb95a961879d3a6e8f3ea7be0287 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 5 Oct 2026 09:50:47 -0500 Subject: [PATCH 8/8] refactor: read issued key pairs by TLSCertificate UID The certificate service no longer names a Secret in its API. spec.secretName, status.secretRef and status.serviceSecretRef are gone, and the issued key pair stays on the service cluster in a Secret named after the TLSCertificate's UID. Key material is never written into a project control plane. Key changes: - resync the vendored certificates.miloapis.com/v1alpha1 types to milo-os/certificates 8f997a0 and vendor StoredSecretName with its test vector - stop setting and comparing spec.secretName on the listener's request - read the stored Secret by StoredSecretName(uid) in the configured secretNamespace instead of following status.serviceSecretRef, and drop the NamespaceRefused reason that only guarded that reference - show operators how to find the stored Secret in the runbook --- docs/runbooks/gateway-tls-certificates.md | 11 ++- .../certificates/v1alpha1/stored_secret.go | 19 ++++ .../v1alpha1/stored_secret_test.go | 23 +++++ .../v1alpha1/tlscertificate_types.go | 41 +++------ .../v1alpha1/zz_generated.deepcopy.go | 40 -------- internal/config/config.go | 8 +- .../controller/gateway_certificate_service.go | 63 ++++++------- .../gateway_certificate_service_test.go | 91 +++++++++---------- internal/controller/gateway_controller.go | 2 +- 9 files changed, 140 insertions(+), 158 deletions(-) create mode 100644 internal/certificates/v1alpha1/stored_secret.go create mode 100644 internal/certificates/v1alpha1/stored_secret_test.go diff --git a/docs/runbooks/gateway-tls-certificates.md b/docs/runbooks/gateway-tls-certificates.md index 83699019..aca683ac 100644 --- a/docs/runbooks/gateway-tls-certificates.md +++ b/docs/runbooks/gateway-tls-certificates.md @@ -106,7 +106,7 @@ unavailable. | `Rejected`, `Refused` | The service refused the request; the condition message carries its reason | | `IssuanceFailed`, `NotReady` | The service accepted it but the ACME order failed or never completed | | `RenewalOverdue` | The served certificate is past its renewal point and nothing newer arrived | -| `MaterialRefused`, `UntrustedChain`, `NamespaceRefused` | The operator refused what the service issued | +| `MaterialRefused`, `UntrustedChain` | The operator refused what the service issued | | `StepFailed`, `NotOwned` | The operator could not reach the service, or the request name is taken | Read the TLSCertificate in the project, named after the gateway and listener: @@ -115,6 +115,15 @@ Read the TLSCertificate in the project, named after the gateway and listener: kubectl -n get tlscertificates -o yaml ``` +The issued key pair never enters the project. It is on the service cluster, in +the `secretNamespace` the operator is configured with, in a Secret named `tc-` +plus the first 32 hex characters of the SHA-256 of the TLSCertificate's UID: + +```sh +uid=$(kubectl -n get tlscertificate -o jsonpath='{.metadata.uid}') +kubectl -n certificates-system get secret "tc-$(printf %s "$uid" | sha256sum | cut -c1-32)" +``` + **Remediate.** A refusal or a missing DNS delegation record is for the customer. An ACME failure, a refused or untrusted chain, or a step failure is a platform fault in the certificate service or the operator's access to it. diff --git a/internal/certificates/v1alpha1/stored_secret.go b/internal/certificates/v1alpha1/stored_secret.go new file mode 100644 index 00000000..964cf37c --- /dev/null +++ b/internal/certificates/v1alpha1/stored_secret.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha1 + +import ( + "crypto/sha256" + "encoding/hex" + + "k8s.io/apimachinery/pkg/types" +) + +// StoredSecretName returns the name of the kubernetes.io/tls Secret holding the +// issued key pair for the TLSCertificate with the given UID. The Secret lives in +// the certificate service's namespace on the service cluster, and its name +// depends on nothing but the UID. +func StoredSecretName(uid types.UID) string { + sum := sha256.Sum256([]byte(uid)) + return "tc-" + hex.EncodeToString(sum[:16]) +} diff --git a/internal/certificates/v1alpha1/stored_secret_test.go b/internal/certificates/v1alpha1/stored_secret_test.go new file mode 100644 index 00000000..2ec1a882 --- /dev/null +++ b/internal/certificates/v1alpha1/stored_secret_test.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha1 + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" +) + +func TestStoredSecretName(t *testing.T) { + uid := types.UID("6f1c2a4e-0b7d-4c1e-9a43-2d5f8e7b1c90") + got := StoredSecretName(uid) + if got != "tc-15441e1abfba82916b1ba950ace11517" { + t.Fatalf("unexpected name %q", got) + } + if again := StoredSecretName(uid); again != got { + t.Fatalf("name is not deterministic: %q then %q", got, again) + } + if other := StoredSecretName("7a2d3b5f-1c8e-4d2f-8b54-3e6a9f8c2d01"); other == got { + t.Fatalf("different UIDs share the name %q", got) + } +} diff --git a/internal/certificates/v1alpha1/tlscertificate_types.go b/internal/certificates/v1alpha1/tlscertificate_types.go index 2ab72524..0b83ac47 100644 --- a/internal/certificates/v1alpha1/tlscertificate_types.go +++ b/internal/certificates/v1alpha1/tlscertificate_types.go @@ -54,26 +54,13 @@ const ( // does not verify that the project controls the names; the caller creates a // TLSCertificate only for names whose ownership it has already verified. type TLSCertificateSpec struct { - DNSNames []DNSName `json:"dnsNames"` - Issuance IssuanceMode `json:"issuance,omitempty"` - SecretName string `json:"secretName,omitempty"` + DNSNames []DNSName `json:"dnsNames"` + Issuance IssuanceMode `json:"issuance,omitempty"` } // DNSName is a lowercase RFC 1123 hostname, optionally prefixed with "*.". type DNSName string -// SecretReference names a Secret in the TLSCertificate's namespace. -type SecretReference struct { - Name string `json:"name"` -} - -// ServiceSecretReference locates the issued Secret on the cluster that runs -// the certificate service. -type ServiceSecretReference struct { - Namespace string `json:"namespace"` - Name string `json:"name"` -} - // RequiredDNSRecord is a DNS record the name's owner must publish before // issuance can complete. type RequiredDNSRecord struct { @@ -96,22 +83,20 @@ type ACMEChallenge struct { // TLSCertificateStatus defines the observed state of TLSCertificate. type TLSCertificateStatus struct { - Issuance ChallengeType `json:"issuance,omitempty"` - SecretRef *SecretReference `json:"secretRef,omitempty"` - ServiceSecretRef *ServiceSecretReference `json:"serviceSecretRef,omitempty"` - DelegationTarget string `json:"delegationTarget,omitempty"` - NotBefore *metav1.Time `json:"notBefore,omitempty"` - NotAfter *metav1.Time `json:"notAfter,omitempty"` - RenewalTime *metav1.Time `json:"renewalTime,omitempty"` - RequiredDNSRecords []RequiredDNSRecord `json:"requiredDNSRecords,omitempty"` - Challenges []ACMEChallenge `json:"challenges,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Issuance ChallengeType `json:"issuance,omitempty"` + DelegationTarget string `json:"delegationTarget,omitempty"` + NotBefore *metav1.Time `json:"notBefore,omitempty"` + NotAfter *metav1.Time `json:"notAfter,omitempty"` + RenewalTime *metav1.Time `json:"renewalTime,omitempty"` + RequiredDNSRecords []RequiredDNSRecord `json:"requiredDNSRecords,omitempty"` + Challenges []ACMEChallenge `json:"challenges,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` } // TLSCertificate requests a publicly trusted TLS certificate for a set of -// hostnames and delivers it as a kubernetes.io/tls Secret in the same -// namespace. Its name is at most 63 characters. +// hostnames. The issued key pair stays on the service cluster and is never +// written to the project. Its name is at most 63 characters. // // +kubebuilder:object:root=true type TLSCertificate struct { diff --git a/internal/certificates/v1alpha1/zz_generated.deepcopy.go b/internal/certificates/v1alpha1/zz_generated.deepcopy.go index 9729778b..9661af44 100644 --- a/internal/certificates/v1alpha1/zz_generated.deepcopy.go +++ b/internal/certificates/v1alpha1/zz_generated.deepcopy.go @@ -41,36 +41,6 @@ func (in *RequiredDNSRecord) DeepCopy() *RequiredDNSRecord { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretReference) DeepCopyInto(out *SecretReference) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference. -func (in *SecretReference) DeepCopy() *SecretReference { - if in == nil { - return nil - } - out := new(SecretReference) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ServiceSecretReference) DeepCopyInto(out *ServiceSecretReference) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSecretReference. -func (in *ServiceSecretReference) DeepCopy() *ServiceSecretReference { - if in == nil { - return nil - } - out := new(ServiceSecretReference) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSCertificate) DeepCopyInto(out *TLSCertificate) { *out = *in @@ -153,16 +123,6 @@ func (in *TLSCertificateSpec) DeepCopy() *TLSCertificateSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSCertificateStatus) DeepCopyInto(out *TLSCertificateStatus) { *out = *in - if in.SecretRef != nil { - in, out := &in.SecretRef, &out.SecretRef - *out = new(SecretReference) - **out = **in - } - if in.ServiceSecretRef != nil { - in, out := &in.ServiceSecretRef, &out.ServiceSecretRef - *out = new(ServiceSecretReference) - **out = **in - } if in.NotBefore != nil { in, out := &in.NotBefore, &out.NotBefore *out = (*in).DeepCopy() diff --git a/internal/config/config.go b/internal/config/config.go index 2cecd3b8..f7deaf35 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -927,12 +927,12 @@ type CertificateServiceConfig struct { Enabled bool `json:"enabled,omitempty"` // KubeconfigPath reaches the cluster the certificate service runs on, - // where it keeps the service-side copy of each issued Secret. Empty - // means the cluster this operator runs in. + // where it keeps each issued key pair. Empty means the cluster this + // operator runs in. KubeconfigPath string `json:"kubeconfigPath,omitempty"` - // SecretNamespace is the only namespace on that cluster a TLSCertificate's - // serviceSecretRef may point into; a reference anywhere else is refused. + // SecretNamespace is the namespace on that cluster holding the issued key + // pairs, each in a Secret named after its TLSCertificate's UID. // // +default="certificates-system" SecretNamespace string `json:"secretNamespace,omitempty"` diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go index 7a125820..41a0f35e 100644 --- a/internal/controller/gateway_certificate_service.go +++ b/internal/controller/gateway_certificate_service.go @@ -207,16 +207,15 @@ func (r *GatewayReconciler) recordCertificateServiceState(gateway *gatewayv1.Gat } const ( - certificateServiceReasonStepFailed = "StepFailed" - certificateServiceReasonNotOwned = "NotOwned" - certificateServiceReasonRefused = "Refused" - certificateServiceReasonRejected = "Rejected" - certificateServiceReasonIssuanceFailed = "IssuanceFailed" - certificateServiceReasonNotReady = "NotReady" - certificateServiceReasonRenewalOverdue = "RenewalOverdue" - certificateServiceReasonNamespaceRefused = "NamespaceRefused" - certificateServiceReasonMaterialRefused = "MaterialRefused" - certificateServiceReasonUntrustedChain = "UntrustedChain" + certificateServiceReasonStepFailed = "StepFailed" + certificateServiceReasonNotOwned = "NotOwned" + certificateServiceReasonRefused = "Refused" + certificateServiceReasonRejected = "Rejected" + certificateServiceReasonIssuanceFailed = "IssuanceFailed" + certificateServiceReasonNotReady = "NotReady" + certificateServiceReasonRenewalOverdue = "RenewalOverdue" + certificateServiceReasonMaterialRefused = "MaterialRefused" + certificateServiceReasonUntrustedChain = "UntrustedChain" ) func certificateServiceUnavailableMessage(hostname string) string { @@ -370,7 +369,7 @@ func (r *GatewayReconciler) ensureListenerTLSCertificate( ) (bool, *certificateServiceIssue, error) { secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) - cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, secretName, hostname) + cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, hostname) switch { case errors.Is(err, errTLSCertificateNotOwned): return false, &certificateServiceIssue{reason: certificateServiceReasonNotOwned, message: certificateRequestNotOwnedMessage(hostname)}, err @@ -452,21 +451,20 @@ const ( ) // ensureTLSCertificate creates the listener's TLSCertificate or confirms the -// existing one. dnsNames and secretName are immutable on the service's API, so -// one of ours that no longer matches is deleted and requested again once it is -// gone. One this gateway does not control is never touched. +// existing one. dnsNames is immutable on the service's API, so one of ours that +// no longer matches is deleted and requested again once it is gone. One this +// gateway does not control is never touched. func (r *GatewayReconciler) ensureTLSCertificate( ctx context.Context, upstreamClient client.Client, upstreamGateway *gatewayv1.Gateway, - certName, secretName, hostname string, + certName, hostname string, ) (*certificatesv1alpha1.TLSCertificate, tlsCertificateState, error) { logger := log.FromContext(ctx) desiredSpec := certificatesv1alpha1.TLSCertificateSpec{ - DNSNames: []certificatesv1alpha1.DNSName{certificatesv1alpha1.DNSName(hostname)}, - Issuance: certificatesv1alpha1.IssuanceModeDNS01, - SecretName: secretName, + DNSNames: []certificatesv1alpha1.DNSName{certificatesv1alpha1.DNSName(hostname)}, + Issuance: certificatesv1alpha1.IssuanceModeDNS01, } cert := &certificatesv1alpha1.TLSCertificate{ @@ -500,7 +498,7 @@ func (r *GatewayReconciler) ensureTLSCertificate( return cert, tlsCertificateReplacing, nil } - if !slices.Equal(cert.Spec.DNSNames, desiredSpec.DNSNames) || cert.Spec.SecretName != desiredSpec.SecretName { + if !slices.Equal(cert.Spec.DNSNames, desiredSpec.DNSNames) { logger.Info("TLSCertificate no longer matches its listener, requesting it again", "tlscertificate", certName, "hostname", hostname) if err := upstreamClient.Delete(ctx, cert, client.Preconditions{UID: &cert.UID}); client.IgnoreNotFound(err) != nil { return nil, tlsCertificateSettled, fmt.Errorf("failed to delete TLSCertificate %s: %w", certName, err) @@ -520,13 +518,13 @@ func (r *GatewayReconciler) ensureTLSCertificate( return cert, tlsCertificateSettled, nil } -// mirrorTLSCertificateSecret copies the service-side issued Secret, read with -// the operator's own credentials from the configured service namespace, into -// the downstream gateway namespace under the listener's secret name, stamped -// with the upstream-owner labels the federation policy selects. The material is +// mirrorTLSCertificateSecret copies the service-side Secret holding the issued +// key pair, read with the operator's own credentials from the configured +// service namespace under the name the TLSCertificate's UID derives, into the +// downstream gateway namespace under the listener's secret name, stamped with +// the upstream-owner labels the federation policy selects. The material is // parsed, matched, checked against the hostname, its expiry and, when -// configured, the trusted roots before it may replace what is serving; the -// project-namespace copy is never read. +// configured, the trusted roots before it may replace what is serving. func (r *GatewayReconciler) mirrorTLSCertificateSecret( ctx context.Context, downstreamStrategy downstreamclient.ResourceStrategy, @@ -542,16 +540,6 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { return false, nil, nil } - ref := cert.Status.ServiceSecretRef - if ref == nil || ref.Name == "" { - logger.Info("TLSCertificate is Ready without a service-side Secret reference", "tlscertificate", cert.Name) - return false, nil, nil - } - if ref.Namespace != r.Config.Gateway.CertificateService.SecretNamespace { - logger.Info("refusing service-side Secret outside the certificate service namespace", - "tlscertificate", cert.Name, "namespace", ref.Namespace, "expected", r.Config.Gateway.CertificateService.SecretNamespace) - return false, &certificateServiceIssue{reason: certificateServiceReasonNamespaceRefused, message: certificateMaterialRefusedMessage(hostname)}, nil - } if r.CertificateServiceReader == nil { return false, nil, fmt.Errorf("certificate service enabled without a client for its cluster") } @@ -575,7 +563,10 @@ func (r *GatewayReconciler) mirrorTLSCertificateSecret( } var source corev1.Secret - sourceKey := client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name} + sourceKey := client.ObjectKey{ + Namespace: r.Config.Gateway.CertificateService.SecretNamespace, + Name: certificatesv1alpha1.StoredSecretName(cert.UID), + } if err := r.CertificateServiceReader.Get(ctx, sourceKey, &source); err != nil { if apierrors.IsNotFound(err) { logger.Info("TLSCertificate is Ready but its service-side Secret is not readable yet", "tlscertificate", cert.Name, "secret", sourceKey) diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go index 9eaba195..38e20446 100644 --- a/internal/controller/gateway_certificate_service_test.go +++ b/internal/controller/gateway_certificate_service_test.go @@ -168,17 +168,18 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { listenerName = gatewayv1.SectionName("https-hostname-0") hostname = "*.shop.example.com" serviceNS = "certificates-system" - serviceSecret = "svc-secret" upstreamCluster = "test" ) certName := tlsCertificateName(gatewayName, listenerName) secretName := listenerCertificateSecretName(gatewayName, listenerName) + certUID := uuid.NewUUID() + serviceSecret := certificatesv1alpha1.StoredSecretName(certUID) testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} ca := newTestCA(t) now := time.Now() serviceCertPEM, serviceKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) - projectCertPEM, projectKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + otherCertPEM, otherKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) servingCertPEM, servingKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) tlsSecret := func(namespace, name string, certPEM, keyPEM []byte) *corev1.Secret { @@ -191,11 +192,10 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { newTLSCertificate := func(owner *gatewayv1.Gateway, name string, mutate func(*certificatesv1alpha1.TLSCertificate)) *certificatesv1alpha1.TLSCertificate { cert := &certificatesv1alpha1.TLSCertificate{ - ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: name, UID: uuid.NewUUID()}, + ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: name, UID: certUID}, Spec: certificatesv1alpha1.TLSCertificateSpec{ - DNSNames: []certificatesv1alpha1.DNSName{hostname}, - Issuance: certificatesv1alpha1.IssuanceModeDNS01, - SecretName: secretName, + DNSNames: []certificatesv1alpha1.DNSName{hostname}, + Issuance: certificatesv1alpha1.IssuanceModeDNS01, }, } if owner != nil { @@ -210,8 +210,6 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { readyStatus := func(cert *certificatesv1alpha1.TLSCertificate) { cert.Status.NotBefore = &metav1.Time{Time: now.Add(-time.Hour)} cert.Status.NotAfter = &metav1.Time{Time: now.Add(60 * 24 * time.Hour)} - cert.Status.SecretRef = &certificatesv1alpha1.SecretReference{Name: secretName} - cert.Status.ServiceSecretRef = &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: serviceSecret} apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue, Reason: "Accepted"}) apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}) } @@ -244,7 +242,6 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { require.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) assert.Equal(t, []certificatesv1alpha1.DNSName{hostname}, cert.Spec.DNSNames) assert.Equal(t, certificatesv1alpha1.IssuanceModeDNS01, cert.Spec.Issuance) - assert.Equal(t, secretName, cert.Spec.SecretName) assert.True(t, metav1.IsControlledBy(&cert, upstreamGateway), "TLSCertificate should be controlled by the upstream Gateway") var legacy cmv1.CertificateList @@ -276,12 +273,12 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { { name: "mirrors the service-side Secret when Ready and admits the listener", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { - return []client.Object{ - newTLSCertificate(gw, certName, readyStatus), - tlsSecret(upstreamNamespace.Name, secretName, projectCertPEM, projectKeyPEM), - } + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: []client.Object{ + tlsSecret(serviceNS, serviceSecret, serviceCertPEM, serviceKeyPEM), + tlsSecret(serviceNS, secretName, otherCertPEM, otherKeyPEM), }, - serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, serviceKeyPEM)}, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { ctx := context.Background() var mirror corev1.Secret @@ -289,7 +286,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { assert.Equal(t, corev1.SecretTypeTLS, mirror.Type) assert.Equal(t, serviceCertPEM, mirror.Data["tls.crt"], "edge material comes from the service-side Secret") assert.Equal(t, serviceKeyPEM, mirror.Data["tls.key"]) - assert.NotEqual(t, projectCertPEM, mirror.Data["tls.crt"], "the project copy must never reach the edge") + assert.NotEqual(t, otherCertPEM, mirror.Data["tls.crt"], "only the Secret named after the TLSCertificate's UID is read") assert.Equal(t, "cluster-"+upstreamCluster, mirror.Labels[downstreamclient.UpstreamOwnerClusterNameLabel]) assert.Equal(t, upstreamNamespace.Name, mirror.Labels[downstreamclient.UpstreamOwnerNamespaceLabel]) assert.Equal(t, KindGateway, mirror.Labels[downstreamclient.UpstreamOwnerKindLabel]) @@ -361,17 +358,13 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { }, }, { - name: "refuses a service-side Secret outside the service namespace", + name: "reads the stored Secret only from the service namespace", upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { - return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { - readyStatus(c) - c.Status.ServiceSecretRef.Namespace = "victim-namespace" - })} + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} }, serviceObjects: []client.Object{tlsSecret("victim-namespace", serviceSecret, serviceCertPEM, serviceKeyPEM)}, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) - assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNamespaceRefused), 1.0) }, }, { @@ -379,7 +372,7 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, readyStatus)} }, - serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, projectKeyPEM)}, + serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, otherKeyPEM)}, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) }, @@ -459,15 +452,15 @@ func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { c.Status.Conditions = []metav1.Condition{{ - Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "SecretConflict", - Message: `Secret "x" exists and is not managed by this TLSCertificate.`, LastTransitionTime: metav1.NewTime(now.Add(-2 * time.Hour)), + Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", + Message: "The certificate has not been issued yet.", LastTransitionTime: metav1.NewTime(now.Add(-2 * time.Hour)), }} })} }, assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) assert.Equal(t, metav1.ConditionTrue, blocked.Status) - assert.Contains(t, blocked.Message, "is not managed by this TLSCertificate") + assert.Contains(t, blocked.Message, "has not been issued yet") assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNotReady), 1.0) assert.Equal(t, 1.0, gaugeValue(t, certificateServiceListenerFailing, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNotReady), "a lasting failure stays visible after its one count") }, @@ -852,6 +845,7 @@ func TestCertificateServiceLeavesExactHostnamesOnCertManager(t *testing.T) { Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { if cert, ok := obj.(*certificatesv1alpha1.TLSCertificate); ok { tlsCertificateCreates.Add(1) + cert.UID = uuid.NewUUID() for _, name := range cert.Spec.DNSNames { requestedNames = append(requestedNames, string(name)) } @@ -885,11 +879,7 @@ func TestCertificateServiceLeavesExactHostnamesOnCertManager(t *testing.T) { }).Build() issuedCrt, issuedKey := ca.issue(t, wildcard, now.Add(-time.Hour), now.Add(60*24*time.Hour)) - service := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: "issued"}, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, - }).Build() + service := fake.NewClientBuilder().WithScheme(testScheme).Build() h := &certificateServiceHarness{ t: t, ctx: ctx, cfg: cfg, upstream: upstream, downstream: downstream, service: service, roots: ca.pool, @@ -914,12 +904,16 @@ func TestCertificateServiceLeavesExactHostnamesOnCertManager(t *testing.T) { var cert certificatesv1alpha1.TLSCertificate require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, wildcardListenerName)}, &cert)) cert.Status = certificatesv1alpha1.TLSCertificateStatus{ - NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, - NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, - ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, - Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, } require.NoError(t, upstream.Status().Update(ctx, &cert)) + require.NoError(t, service.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: certificatesv1alpha1.StoredSecretName(cert.UID)}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, + })) h.reconcile() _, downstreamGateway, _ := h.reconcile() assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, wildcardListenerName), "the wildcard serves the service-issued certificate") @@ -987,16 +981,13 @@ func TestCertificateServiceOneFailingListenerDoesNotDelayOthers(t *testing.T) { if obj.GetName() == failingName { return apierrors.NewServiceUnavailable("webhook unavailable") } + obj.SetUID(uuid.NewUUID()) return cl.Create(ctx, obj, opts...) }, }).Build() downstream := fake.NewClientBuilder().WithScheme(testScheme).WithStatusSubresource(&gatewayv1.Gateway{}).Build() issuedCrt, issuedKey := ca.issue(t, healthy, now.Add(-time.Hour), now.Add(60*24*time.Hour)) - service := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: "issued"}, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, - }).Build() + service := fake.NewClientBuilder().WithScheme(testScheme).Build() h := &certificateServiceHarness{ t: t, ctx: ctx, cfg: cfg, upstream: upstream, downstream: downstream, service: service, roots: ca.pool, @@ -1009,12 +1000,16 @@ func TestCertificateServiceOneFailingListenerDoesNotDelayOthers(t *testing.T) { var cert certificatesv1alpha1.TLSCertificate require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, healthyListener)}, &cert), "the healthy wildcard is requested in the same pass") cert.Status = certificatesv1alpha1.TLSCertificateStatus{ - NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, - NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, - ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, - Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, } require.NoError(t, upstream.Status().Update(ctx, &cert)) + require.NoError(t, service.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: certificatesv1alpha1.StoredSecretName(cert.UID)}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, + })) second, _, result := h.reconcile() _, _, cooling := h.reconciler.certificateServiceInBackoff(certificateServiceKey{gateway: upstreamGateway.UID, listener: failingListener}, time.Now()) @@ -1248,20 +1243,20 @@ func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { cert := &certificatesv1alpha1.TLSCertificate{ ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: certName}, - Spec: certificatesv1alpha1.TLSCertificateSpec{DNSNames: []certificatesv1alpha1.DNSName{hostname}, Issuance: certificatesv1alpha1.IssuanceModeDNS01, SecretName: secretName}, + Spec: certificatesv1alpha1.TLSCertificateSpec{DNSNames: []certificatesv1alpha1.DNSName{hostname}, Issuance: certificatesv1alpha1.IssuanceModeDNS01}, Status: certificatesv1alpha1.TLSCertificateStatus{ - NotAfter: &metav1.Time{Time: now.Add(90 * 24 * time.Hour)}, - ServiceSecretRef: &certificatesv1alpha1.ServiceSecretReference{Namespace: serviceNS, Name: "issued"}, - Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}}, + NotAfter: &metav1.Time{Time: now.Add(90 * 24 * time.Hour)}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}}, }, } require.NoError(t, controllerutil.SetControllerReference(upstreamGateway, cert, testScheme)) serving := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: secretName}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": servingCrt, "tls.key": servingKey}} - issued := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: "issued"}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}} + issued := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}} for _, obj := range []client.Object{domain, gatewayClass, cert, serving, issued} { obj.SetUID(uuid.NewUUID()) obj.SetCreationTimestamp(metav1.Now()) } + issued.Name = certificatesv1alpha1.StoredSecretName(cert.UID) fakeUpstreamClient := fake.NewClientBuilder().WithScheme(testScheme). WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, cert). diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 1d92fc2d..3da7e0d4 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -97,7 +97,7 @@ type GatewayReconciler struct { DownstreamCluster cluster.Cluster // CertificateServiceReader reads the certificate service's cluster, where - // the service-side copy of each issued Secret lives. Required when + // each issued key pair is stored. Required when // Config.Gateway.CertificateService.Enabled. CertificateServiceReader client.Reader