diff --git a/api/v1alpha/httpproxy_types.go b/api/v1alpha/httpproxy_types.go index 489feee5..88ab9434 100644 --- a/api/v1alpha/httpproxy_types.go +++ b/api/v1alpha/httpproxy_types.go @@ -551,6 +551,10 @@ const ( // CertificateReadyReasonChallengeInProgress indicates an ACME challenge is in progress. CertificateReadyReasonChallengeInProgress = "ChallengeInProgress" + + // CertificateReadyReasonRenewalFailing indicates the hostname still serves a + // valid certificate but its replacement cannot be issued; the message says why. + CertificateReadyReasonRenewalFailing = "RenewalFailing" ) // Reasons for HostnameConditionAvailable. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d3aaeb88..f7e99f1c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -61,6 +61,24 @@ rules: - patch - update - watch +- apiGroups: + - certificates.miloapis.com + resources: + - tlscertificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.miloapis.com + resources: + - tlscertificates/status + verbs: + - get - apiGroups: - coordination.k8s.io resources: diff --git a/config/telemetry/alerts/gateways.yaml b/config/telemetry/alerts/gateways.yaml index 710a5fa4..c35a127a 100644 --- a/config/telemetry/alerts/gateways.yaml +++ b/config/telemetry/alerts/gateways.yaml @@ -173,3 +173,24 @@ spec: summary: "{{ $value }} edge listener(s) have every TLS certificate broken and cannot be protected" description: "The extension server left {{ $value }} edge listener(s) untouched because every certificate on them is broken. It never removes a listener entirely, so the edge will reject the configuration update for those listeners (EnvoyListenerUpdateRejected confirms it). This means the controller did not withhold the listener before it reached the edge. Check extension server logs for 'listeners_left_intact' and why the controller did not withhold the listener." runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#tlsbackstoplistenerallcertsbroken" + + - name: nso-certificate-service + interval: 30s + rules: + # Fires when a wildcard hostname's certificate cannot be issued or renewed + # through the certificate service. The hostname may still serve its previous + # certificate, so this fires weeks before that certificate expires. + - alert: CertificateServiceIssuanceFailing + expr: | + max by (namespace, name, listener, reason) (nso_certificate_service_listener_failing) > 0 + or + sum by (namespace, name, listener, reason) (increase(nso_certificate_service_failures_total[1h])) > 0 + for: 2h + labels: + severity: warning + service: network-services + team: sre + annotations: + summary: "Certificate for Gateway listener {{ $labels.namespace }}/{{ $labels.name }}/{{ $labels.listener }} cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener {{ $labels.listener }} on Gateway {{ $labels.name }} in namespace {{ $labels.namespace }} for over two hours (reason: {{ $labels.reason }}). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" diff --git a/docs/runbooks/gateway-tls-certificates.md b/docs/runbooks/gateway-tls-certificates.md index 5118827a..aca683ac 100644 --- a/docs/runbooks/gateway-tls-certificates.md +++ b/docs/runbooks/gateway-tls-certificates.md @@ -87,6 +87,47 @@ renewal depends on it. customer-driven gating event — no platform fix. If renewal is failing for a platform reason, fix the issuer / ACME path so cert-manager can renew. +## CertificateServiceIssuanceFailing + +**Meaning.** A wildcard hostname's certificate has not been issued or renewed by +the certificate service for over two hours. Only wildcard hostnames use the +service; exact hostnames stay on cert-manager and are covered by the alerts +above. + +**Impact.** If the listener still serves a certificate, none yet: it carries +`CertificateRenewalBlocked` and keeps serving until that certificate expires. If +it serves nothing, it carries `CertificateIssuanceBlocked` and the wildcard is +unavailable. + +**Diagnose.** The `reason` label says where it failed: + +| Reason | Where | +|---|---| +| `Rejected`, `Refused` | The service refused the request; the condition message carries its reason | +| `IssuanceFailed`, `NotReady` | The service accepted it but the ACME order failed or never completed | +| `RenewalOverdue` | The served certificate is past its renewal point and nothing newer arrived | +| `MaterialRefused`, `UntrustedChain` | The operator refused what the service issued | +| `StepFailed`, `NotOwned` | The operator could not reach the service, or the request name is taken | + +Read the TLSCertificate in the project, named after the gateway and listener: + +```sh +kubectl -n get tlscertificates -o yaml +``` + +The issued key pair never enters the project. It is on the service cluster, in +the `secretNamespace` the operator is configured with, in a Secret named `tc-` +plus the first 32 hex characters of the SHA-256 of the TLSCertificate's UID: + +```sh +uid=$(kubectl -n get tlscertificate -o jsonpath='{.metadata.uid}') +kubectl -n certificates-system get secret "tc-$(printf %s "$uid" | sha256sum | cut -c1-32)" +``` + +**Remediate.** A refusal or a missing DNS delegation record is for the customer. +An ACME failure, a refused or untrusted chain, or a step failure is a platform +fault in the certificate service or the operator's access to it. + ## TLSBackstopPruningChains **Meaning.** The extension server is actively dropping broken certificates from diff --git a/internal/agent/catalog.go b/internal/agent/catalog.go index 4541496b..ce5cd838 100644 --- a/internal/agent/catalog.go +++ b/internal/agent/catalog.go @@ -540,6 +540,18 @@ var dnsAndCertCatalog = []ReasonInfo{ "status message.", Skill: SkillCertificateNotIssued, }, + { + Reason: networkingv1alpha.CertificateReadyReasonRenewalFailing, + ConditionType: networkingv1alpha.HostnameConditionCertificateReady, + Actionability: ActionabilityUser, + Scope: ScopeOneHostname, + Explanation: "HTTPS still works on this hostname's current certificate, but the replacement " + + "cannot be issued. When the current certificate expires HTTPS will fail.", + Remediation: "The status message says what blocked the renewal. If it names the hostname " + + "itself, check that it still resolves publicly to this load balancer; otherwise " + + "it is a platform issue and support can see the same message.", + Skill: SkillCertificateNotIssued, + }, } // domainCatalog covers the Domain behind a custom hostname. This is where diff --git a/internal/certificates/v1alpha1/groupversion_info.go b/internal/certificates/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..e0bccfe9 --- /dev/null +++ b/internal/certificates/v1alpha1/groupversion_info.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package v1alpha1 carries the certificates.miloapis.com/v1alpha1 types the +// operator consumes from the Milo certificate service. It mirrors the +// service's API until its module is importable. +// +// +kubebuilder:object:generate=true +// +kubebuilder:skip +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + GroupVersion = schema.GroupVersion{Group: "certificates.miloapis.com", Version: "v1alpha1"} + + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, &TLSCertificate{}, &TLSCertificateList{}) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} diff --git a/internal/certificates/v1alpha1/stored_secret.go b/internal/certificates/v1alpha1/stored_secret.go new file mode 100644 index 00000000..964cf37c --- /dev/null +++ b/internal/certificates/v1alpha1/stored_secret.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha1 + +import ( + "crypto/sha256" + "encoding/hex" + + "k8s.io/apimachinery/pkg/types" +) + +// StoredSecretName returns the name of the kubernetes.io/tls Secret holding the +// issued key pair for the TLSCertificate with the given UID. The Secret lives in +// the certificate service's namespace on the service cluster, and its name +// depends on nothing but the UID. +func StoredSecretName(uid types.UID) string { + sum := sha256.Sum256([]byte(uid)) + return "tc-" + hex.EncodeToString(sum[:16]) +} diff --git a/internal/certificates/v1alpha1/stored_secret_test.go b/internal/certificates/v1alpha1/stored_secret_test.go new file mode 100644 index 00000000..2ec1a882 --- /dev/null +++ b/internal/certificates/v1alpha1/stored_secret_test.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha1 + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" +) + +func TestStoredSecretName(t *testing.T) { + uid := types.UID("6f1c2a4e-0b7d-4c1e-9a43-2d5f8e7b1c90") + got := StoredSecretName(uid) + if got != "tc-15441e1abfba82916b1ba950ace11517" { + t.Fatalf("unexpected name %q", got) + } + if again := StoredSecretName(uid); again != got { + t.Fatalf("name is not deterministic: %q then %q", got, again) + } + if other := StoredSecretName("7a2d3b5f-1c8e-4d2f-8b54-3e6a9f8c2d01"); other == got { + t.Fatalf("different UIDs share the name %q", got) + } +} diff --git a/internal/certificates/v1alpha1/tlscertificate_types.go b/internal/certificates/v1alpha1/tlscertificate_types.go new file mode 100644 index 00000000..0b83ac47 --- /dev/null +++ b/internal/certificates/v1alpha1/tlscertificate_types.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IssuanceMode selects how the certificate authority validates control of the +// requested names. +type IssuanceMode string + +const ( + IssuanceModeAuto IssuanceMode = "Auto" + IssuanceModeHTTP01 IssuanceMode = "HTTP01" + IssuanceModeDNS01 IssuanceMode = "DNS01" +) + +// ChallengeType is a resolved issuance mode: the ACME challenge type used to +// validate the names. +type ChallengeType string + +const ( + ChallengeTypeHTTP01 ChallengeType = "HTTP01" + ChallengeTypeDNS01 ChallengeType = "DNS01" +) + +// DNSRecordPurpose explains why a DNS record must be published. +type DNSRecordPurpose string + +const ( + DNSRecordPurposeRouting DNSRecordPurpose = "Routing" + DNSRecordPurposeCertificate DNSRecordPurpose = "Certificate" +) + +// ChallengeState is the lifecycle state of an ACME challenge. +type ChallengeState string + +const ( + ChallengeStatePending ChallengeState = "Pending" + ChallengeStateValid ChallengeState = "Valid" + ChallengeStateInvalid ChallengeState = "Invalid" +) + +// Condition types reported on a TLSCertificate. +const ( + ConditionAccepted = "Accepted" + ConditionDNSDelegationReady = "DNSDelegationReady" + ConditionIssuing = "Issuing" + ConditionReady = "Ready" +) + +// TLSCertificateSpec defines the desired state of TLSCertificate. The service +// does not verify that the project controls the names; the caller creates a +// TLSCertificate only for names whose ownership it has already verified. +type TLSCertificateSpec struct { + DNSNames []DNSName `json:"dnsNames"` + Issuance IssuanceMode `json:"issuance,omitempty"` +} + +// DNSName is a lowercase RFC 1123 hostname, optionally prefixed with "*.". +type DNSName string + +// RequiredDNSRecord is a DNS record the name's owner must publish before +// issuance can complete. +type RequiredDNSRecord struct { + Name string `json:"name"` + Type string `json:"type"` + Content string `json:"content"` + Purpose DNSRecordPurpose `json:"purpose"` +} + +// ACMEChallenge is a live ACME challenge for one name. For HTTP01, the +// consumer serves GET /.well-known/acme-challenge/ with the body +// on dnsName. +type ACMEChallenge struct { + DNSName string `json:"dnsName"` + Type ChallengeType `json:"type"` + Token string `json:"token"` + Key string `json:"key"` + State ChallengeState `json:"state"` +} + +// TLSCertificateStatus defines the observed state of TLSCertificate. +type TLSCertificateStatus struct { + Issuance ChallengeType `json:"issuance,omitempty"` + DelegationTarget string `json:"delegationTarget,omitempty"` + NotBefore *metav1.Time `json:"notBefore,omitempty"` + NotAfter *metav1.Time `json:"notAfter,omitempty"` + RenewalTime *metav1.Time `json:"renewalTime,omitempty"` + RequiredDNSRecords []RequiredDNSRecord `json:"requiredDNSRecords,omitempty"` + Challenges []ACMEChallenge `json:"challenges,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +// TLSCertificate requests a publicly trusted TLS certificate for a set of +// hostnames. The issued key pair stays on the service cluster and is never +// written to the project. Its name is at most 63 characters. +// +// +kubebuilder:object:root=true +type TLSCertificate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TLSCertificateSpec `json:"spec"` + Status TLSCertificateStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type TLSCertificateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TLSCertificate `json:"items"` +} diff --git a/internal/certificates/v1alpha1/zz_generated.deepcopy.go b/internal/certificates/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..9661af44 --- /dev/null +++ b/internal/certificates/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,165 @@ +//go:build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ACMEChallenge) DeepCopyInto(out *ACMEChallenge) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ACMEChallenge. +func (in *ACMEChallenge) DeepCopy() *ACMEChallenge { + if in == nil { + return nil + } + out := new(ACMEChallenge) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RequiredDNSRecord) DeepCopyInto(out *RequiredDNSRecord) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequiredDNSRecord. +func (in *RequiredDNSRecord) DeepCopy() *RequiredDNSRecord { + if in == nil { + return nil + } + out := new(RequiredDNSRecord) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificate) DeepCopyInto(out *TLSCertificate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificate. +func (in *TLSCertificate) DeepCopy() *TLSCertificate { + if in == nil { + return nil + } + out := new(TLSCertificate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSCertificate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificateList) DeepCopyInto(out *TLSCertificateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TLSCertificate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificateList. +func (in *TLSCertificateList) DeepCopy() *TLSCertificateList { + if in == nil { + return nil + } + out := new(TLSCertificateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSCertificateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificateSpec) DeepCopyInto(out *TLSCertificateSpec) { + *out = *in + if in.DNSNames != nil { + in, out := &in.DNSNames, &out.DNSNames + *out = make([]DNSName, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificateSpec. +func (in *TLSCertificateSpec) DeepCopy() *TLSCertificateSpec { + if in == nil { + return nil + } + out := new(TLSCertificateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCertificateStatus) DeepCopyInto(out *TLSCertificateStatus) { + *out = *in + if in.NotBefore != nil { + in, out := &in.NotBefore, &out.NotBefore + *out = (*in).DeepCopy() + } + if in.NotAfter != nil { + in, out := &in.NotAfter, &out.NotAfter + *out = (*in).DeepCopy() + } + if in.RenewalTime != nil { + in, out := &in.RenewalTime, &out.RenewalTime + *out = (*in).DeepCopy() + } + if in.RequiredDNSRecords != nil { + in, out := &in.RequiredDNSRecords, &out.RequiredDNSRecords + *out = make([]RequiredDNSRecord, len(*in)) + copy(*out, *in) + } + if in.Challenges != nil { + in, out := &in.Challenges, &out.Challenges + *out = make([]ACMEChallenge, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCertificateStatus. +func (in *TLSCertificateStatus) DeepCopy() *TLSCertificateStatus { + if in == nil { + return nil + } + out := new(TLSCertificateStatus) + in.DeepCopyInto(out) + return out +} diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 439472de..56768bb0 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -4,6 +4,7 @@ package managercmd import ( "context" + "crypto/x509" "errors" "flag" "fmt" @@ -44,6 +45,7 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" "go.datum.net/network-services-operator/internal/cmd/clusterdiscovery" "go.datum.net/network-services-operator/internal/config" "go.datum.net/network-services-operator/internal/controller" @@ -76,6 +78,7 @@ func init() { utilruntime.Must(cmv1.AddToScheme(scheme)) utilruntime.Must(dnsv1alpha1.AddToScheme(scheme)) utilruntime.Must(ipamv1alpha1.AddToScheme(scheme)) + utilruntime.Must(certificatesv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -384,11 +387,24 @@ func NewCommand(build BuildInfo) *cobra.Command { os.Exit(1) } + certificateServiceReader, err := newCertificateServiceReader(serverConfig.Gateway) + if err != nil { + setupLog.Error(err, "unable to build certificate service client") + os.Exit(1) + } + certificateServiceRoots, err := serverConfig.Gateway.CertificateService.TrustedRoots() + if err != nil { + setupLog.Error(err, "unable to load certificate service trusted roots") + os.Exit(1) + } + registeredControllers, err := setupControllers(mgr, serverConfig, controllerDeps{ - downstreamCluster: downstreamCluster, - singletonManager: singletonControllerMgr, - irohDownstream: irohDownstream, - ipamClients: ipamClients, + downstreamCluster: downstreamCluster, + singletonManager: singletonControllerMgr, + irohDownstream: irohDownstream, + ipamClients: ipamClients, + certificateServiceReader: certificateServiceReader, + certificateServiceRoots: certificateServiceRoots, }) if err != nil { setupLog.Error(err, "unable to set up controllers") @@ -561,10 +577,27 @@ func setupWebhooks(mgr mcmanager.Manager, serverConfig config.NetworkServicesOpe // controllerDeps carries the clients and managers that controllers are wired // against. Fields are only populated for the sets that need them. type controllerDeps struct { - downstreamCluster cluster.Cluster - singletonManager manager.Manager - irohDownstream cluster.Cluster - ipamClients controller.IPAMClientFactory + downstreamCluster cluster.Cluster + singletonManager manager.Manager + irohDownstream cluster.Cluster + ipamClients controller.IPAMClientFactory + certificateServiceReader client.Reader + certificateServiceRoots *x509.CertPool +} + +// newCertificateServiceReader returns an uncached client for the cluster the +// certificate service keeps its issued Secrets on, or nil when the service is +// not consumed. Uncached so the operator never holds an informer over every +// Secret on that cluster. +func newCertificateServiceReader(gatewayConfig config.GatewayConfig) (client.Reader, error) { + if !gatewayConfig.CertificateService.Enabled { + return nil, nil + } + restConfig, err := gatewayConfig.CertificateService.RestConfig() + if err != nil { + return nil, fmt.Errorf("unable to load certificate service kubeconfig: %w", err) + } + return client.New(restConfig, client.Options{Scheme: scheme}) } // newIPAMClientFactory returns nil when no IPAM connection is configured. A @@ -680,8 +713,10 @@ func controllerRegistrations( }}, {"gateway", true, func() error { return (&controller.GatewayReconciler{ - Config: serverConfig, - DownstreamCluster: deps.downstreamCluster, + Config: serverConfig, + DownstreamCluster: deps.downstreamCluster, + CertificateServiceReader: deps.certificateServiceReader, + CertificateServiceRoots: deps.certificateServiceRoots, }).SetupWithManager(mgr) }}, {"gatewayclass", true, func() error { diff --git a/internal/config/config.go b/internal/config/config.go index b72c4a07..f7deaf35 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "context" "crypto/tls" + "crypto/x509" "errors" "fmt" "os" @@ -905,6 +906,93 @@ type GatewayConfig struct { // the deletion through the entire chain (CertificateRequest, Order, // Challenge, solver resources). CertificateReissuance CertificateReissuanceConfig `json:"certificateReissuance,omitempty"` + + // CertificateService hands certificate issuance for wildcard hostnames to + // the Milo certificate service. When enabled, the gateway controller + // requests a DNS-01 TLSCertificate in the project control plane for each + // single-label wildcard listener and mirrors the issued Secret downstream. + // Exact hostnames stay on cert-manager either way. + CertificateService CertificateServiceConfig `json:"certificateService,omitempty"` +} + +// +k8s:deepcopy-gen=true + +// CertificateServiceConfig controls consumption of the Milo certificate +// service (certificates.miloapis.com). +type CertificateServiceConfig struct { + // Enabled issues certificates for wildcard hostnames through upstream + // TLSCertificates. + // + // Defaults to false. + Enabled bool `json:"enabled,omitempty"` + + // KubeconfigPath reaches the cluster the certificate service runs on, + // where it keeps each issued key pair. Empty means the cluster this + // operator runs in. + KubeconfigPath string `json:"kubeconfigPath,omitempty"` + + // SecretNamespace is the namespace on that cluster holding the issued key + // pairs, each in a Secret named after its TLSCertificate's UID. + // + // +default="certificates-system" + SecretNamespace string `json:"secretNamespace,omitempty"` + + // VerifyChain refuses issued material whose chain does not build to + // TrustedRootsFile, or to the system roots when that is empty. + // + // Defaults to false. + VerifyChain bool `json:"verifyChain,omitempty"` + + // TrustedRootsFile is a PEM bundle of the roots an issued chain must + // build to when VerifyChain is set. + TrustedRootsFile string `json:"trustedRootsFile,omitempty"` +} + +// TrustedRoots returns the roots an issued chain must build to, or nil when +// chain verification is off or uses the system roots. +func (c *CertificateServiceConfig) TrustedRoots() (*x509.CertPool, error) { + if !c.VerifyChain || c.TrustedRootsFile == "" { + return nil, nil + } + data, err := os.ReadFile(c.TrustedRootsFile) + if err != nil { + return nil, fmt.Errorf("failed to read trusted roots: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(data) { + return nil, fmt.Errorf("trusted roots file %s holds no certificates", c.TrustedRootsFile) + } + return pool, nil +} + +func SetDefaults_CertificateServiceConfig(obj *CertificateServiceConfig) { + if obj.SecretNamespace == "" { + obj.SecretNamespace = "certificates-system" + } +} + +// RestConfig returns the connection to the certificate service's cluster. +func (c *CertificateServiceConfig) RestConfig() (*rest.Config, error) { + cfg, err := c.restConfig() + if err != nil { + return nil, err + } + cfg.Timeout = certificateServiceRequestTimeout + return cfg, nil +} + +// certificateServiceRequestTimeout bounds each read of the service cluster, so +// a hung connection there cannot stall a gateway reconcile. +const certificateServiceRequestTimeout = 10 * time.Second + +func (c *CertificateServiceConfig) restConfig() (*rest.Config, error) { + if c.KubeconfigPath != "" { + return clientcmd.BuildConfigFromFlags("", c.KubeconfigPath) + } + if cfg, err := rest.InClusterConfig(); err == nil { + return cfg, nil + } + return ctrl.GetConfig() } // +k8s:deepcopy-gen=true @@ -1490,6 +1578,15 @@ func (c *GatewayConfig) validate() error { } seen = append(seen, domain) } + if c.CertificateService.Enabled && c.DisableHostnameVerification { + errs = append(errs, errors.New("certificateService.enabled requires hostname verification: the certificate service issues for any hostname it is handed")) + } + if c.CertificateService.Enabled && strings.TrimSpace(c.CertificateService.SecretNamespace) == "" { + errs = append(errs, errors.New("certificateService.secretNamespace is required when certificateService.enabled")) + } + if c.CertificateService.TrustedRootsFile != "" && !c.CertificateService.VerifyChain { + errs = append(errs, errors.New("certificateService.trustedRootsFile requires certificateService.verifyChain")) + } return errors.Join(errs...) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ff5bf93f..5780a786 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,8 @@ package config import ( + "os" + "path/filepath" "strings" "testing" ) @@ -204,3 +206,90 @@ func TestGatewayConfig_ManagedTargetDomains(t *testing.T) { } } } + +func TestGatewayConfig_ValidateCertificateService(t *testing.T) { + tests := []struct { + name string + gateway GatewayConfig + wantSub string + }{ + { + name: "disabled by default validates", + }, + { + name: "enabled with hostname verification validates", + gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system"}}, + }, + { + name: "enabled without a secret namespace is rejected", + gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true}}, + wantSub: "certificateService.secretNamespace is required", + }, + { + name: "enabled without hostname verification is rejected", + gateway: GatewayConfig{ + CertificateService: CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system"}, + DisableHostnameVerification: true, + }, + wantSub: "certificateService.enabled requires hostname verification", + }, + { + name: "a trusted roots file without chain verification is rejected", + gateway: GatewayConfig{CertificateService: CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system", TrustedRootsFile: "/roots.pem"}}, + wantSub: "certificateService.trustedRootsFile requires certificateService.verifyChain", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &NetworkServicesOperator{Gateway: tt.gateway} + err := cfg.Validate() + if tt.wantSub == "" { + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantSub) { + t.Fatalf("expected error containing %q, got %v", tt.wantSub, err) + } + }) + } +} + +func TestSetObjectDefaults_CertificateService(t *testing.T) { + cfg := &NetworkServicesOperator{} + SetObjectDefaults_NetworkServicesOperator(cfg) + if cfg.Gateway.CertificateService.Enabled { + t.Error("certificateService.enabled should default to false") + } + if cfg.Gateway.CertificateService.KubeconfigPath != "" { + t.Errorf("certificateService.kubeconfigPath should default to empty, got %q", cfg.Gateway.CertificateService.KubeconfigPath) + } + if got, want := cfg.Gateway.CertificateService.SecretNamespace, "certificates-system"; got != want { + t.Errorf("certificateService.secretNamespace = %q, want %q", got, want) + } + if cfg.Gateway.CertificateService.VerifyChain { + t.Error("certificateService.verifyChain should default to false") + } +} + +func TestCertificateServiceConfig_TrustedRoots(t *testing.T) { + off := CertificateServiceConfig{TrustedRootsFile: "/does/not/exist"} + if pool, err := off.TrustedRoots(); pool != nil || err != nil { + t.Fatalf("verification off should load nothing, got %v, %v", pool, err) + } + + system := CertificateServiceConfig{VerifyChain: true} + if pool, err := system.TrustedRoots(); pool != nil || err != nil { + t.Fatalf("no roots file means the system roots, got %v, %v", pool, err) + } + + empty := filepath.Join(t.TempDir(), "empty.pem") + if err := os.WriteFile(empty, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := (&CertificateServiceConfig{VerifyChain: true, TrustedRootsFile: empty}).TrustedRoots(); err == nil { + t.Fatal("a roots file with no certificates should be refused") + } +} diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index d6a4737c..1202dfcd 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -78,6 +78,21 @@ func (in *CertificateReissuanceConfig) DeepCopy() *CertificateReissuanceConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificateServiceConfig) DeepCopyInto(out *CertificateServiceConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificateServiceConfig. +func (in *CertificateServiceConfig) DeepCopy() *CertificateServiceConfig { + if in == nil { + return nil + } + out := new(CertificateServiceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientConnectionConfig) DeepCopyInto(out *ClientConnectionConfig) { *out = *in @@ -436,6 +451,7 @@ func (in *GatewayConfig) DeepCopyInto(out *GatewayConfig) { **out = **in } out.CertificateReissuance = in.CertificateReissuance + out.CertificateService = in.CertificateService } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayConfig. diff --git a/internal/config/zz_generated.defaults.go b/internal/config/zz_generated.defaults.go index 84aa88f5..bf274d86 100644 --- a/internal/config/zz_generated.defaults.go +++ b/internal/config/zz_generated.defaults.go @@ -292,6 +292,10 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { if in.Gateway.CertificateReissuance.MaxRetries == 0 { in.Gateway.CertificateReissuance.MaxRetries = 3 } + SetDefaults_CertificateServiceConfig(&in.Gateway.CertificateService) + if in.Gateway.CertificateService.SecretNamespace == "" { + in.Gateway.CertificateService.SecretNamespace = "certificates-system" + } if in.HTTPProxy.GatewayClassName == "" { in.HTTPProxy.GatewayClassName = "datum-external-global-proxy" } diff --git a/internal/controller/gateway_certificate_service.go b/internal/controller/gateway_certificate_service.go new file mode 100644 index 00000000..41a0f35e --- /dev/null +++ b/internal/controller/gateway_certificate_service.go @@ -0,0 +1,947 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "regexp" + "slices" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" +) + +// +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=certificates.miloapis.com,resources=tlscertificates/status,verbs=get + +const KindTLSCertificate = "TLSCertificate" + +// tlsCertificateMirrorAdmitDelay is how soon the listener is re-evaluated after +// its Secret lands downstream, since listener health was judged before the +// mirror in the same pass. +const tlsCertificateMirrorAdmitDelay = time.Second + +const ( + certificateServiceBackoffBase = 5 * time.Second + certificateServiceBackoffMax = 5 * time.Minute + + // certificateServiceRecheck is how often a wildcard listener is looked at + // again without any event, so a renewal that quietly stalls is noticed. + certificateServiceRecheck = time.Hour + + // tlsCertificateIssueGrace is how long a TLSCertificate may stay not Ready + // before it is reported as failing rather than in progress. + tlsCertificateIssueGrace = time.Hour + + // renewalOverdueDivisor sets the share of a served certificate's lifetime + // below which its renewal is overdue. The service renews with a third left. + renewalOverdueDivisor = 4 +) + +var nonDNSLabelChars = regexp.MustCompile(`[^a-z0-9-]+`) + +// tlsCertificateName names the listener's TLSCertificate: a readable prefix +// from the gateway and listener plus a hash of both, so "a-b"/"c" and "a"/"b-c" +// never meet, inside the service's 63 character cap and valid as a DNS label +// whatever the gateway was called. +func tlsCertificateName(gatewayName string, listenerName gatewayv1.SectionName) string { + sum := sha256.Sum256([]byte(gatewayName + "/" + string(listenerName))) + suffix := hex.EncodeToString(sum[:])[:10] + + prefix := nonDNSLabelChars.ReplaceAllString(strings.ToLower(gatewayName+"-"+string(listenerName)), "-") + if max := 63 - len(suffix) - 1; len(prefix) > max { + prefix = prefix[:max] + } + prefix = strings.Trim(prefix, "-") + if prefix == "" { + return suffix + } + return prefix + "-" + suffix +} + +// isSingleLabelWildcard reports whether a hostname is a wildcard over exactly +// one label, the only shape the certificate service is used for. +func isSingleLabelWildcard(hostname string) bool { + base, ok := strings.CutPrefix(hostname, "*.") + return ok && base != "" && !strings.Contains(base, "*") +} + +// listenerWantsOwnCertificate reports whether a listener is one the controller +// issues a per-hostname certificate for, under either issuance path. +func (r *GatewayReconciler) listenerWantsOwnCertificate(l gatewayv1.Listener, claimedHostnames []string) (string, bool) { + if l.TLS == nil || l.TLS.Options[certificateIssuerTLSOption] == "" || l.Hostname == nil { + return "", false + } + hostname := string(*l.Hostname) + if !slices.Contains(claimedHostnames, hostname) { + return "", false + } + wildcardSuffix := "." + r.Config.Gateway.TargetDomain + if r.Config.Gateway.HasDefaultListenerTLSSecret() && + (strings.HasSuffix(hostname, wildcardSuffix) || hostname == r.Config.Gateway.TargetDomain) { + return "", false + } + return hostname, true +} + +// listenerUsesCertificateService reports whether the certificate service, not +// cert-manager, issues for this listener: only wildcard hostnames, and only +// while the service is enabled. Every exact hostname stays on cert-manager. +func (r *GatewayReconciler) listenerUsesCertificateService(l gatewayv1.Listener, claimedHostnames []string) (string, bool) { + if !r.Config.Gateway.CertificateService.Enabled { + return "", false + } + hostname, wanted := r.listenerWantsOwnCertificate(l, claimedHostnames) + if !wanted || !isSingleLabelWildcard(hostname) { + return "", false + } + return hostname, true +} + +// listenerIssuerResolvable applies the same rule the cert-manager path does to +// the certificate-issuer option: an `auto` that maps to nothing and inherits +// from no other listener leaves the listener un-programmed. +func (r *GatewayReconciler) listenerIssuerResolvable(upstreamGateway *gatewayv1.Gateway, l gatewayv1.Listener) bool { + issuer := string(l.TLS.Options[certificateIssuerTLSOption]) + if mapped := r.Config.Gateway.ClusterIssuerMap[issuer]; mapped != "" { + issuer = mapped + } + return issuer != autoIssuerSentinel || r.resolveAutoIssuer(upstreamGateway) != "" +} + +// certificateServiceKey identifies one listener of one gateway. An empty +// listener stands for the gateway-wide cleanup. +type certificateServiceKey struct { + gateway types.UID + listener gatewayv1.SectionName +} + +// certificateServiceBackoff is what the step remembers about a listener whose +// last pass failed: how many in a row, when it may try again, and what it told +// the customer, so an event-driven reconcile inside the window repeats the +// message rather than the calls. +type certificateServiceBackoff struct { + attempts int + nextTry time.Time + issue certificateServiceIssue +} + +// certificateServiceRequeue records the outcome of a listener's pass and hands +// back the delay before the next attempt, doubling per consecutive failure up +// to a cap, and forgets the listener once a pass succeeds. +func (r *GatewayReconciler) certificateServiceRequeue(key certificateServiceKey, failed bool, now time.Time, issue certificateServiceIssue) time.Duration { + if !failed { + r.certificateServiceFailures.Delete(key) + return 0 + } + attempts := 1 + if previous, ok := r.certificateServiceFailures.Load(key); ok { + attempts = previous.(certificateServiceBackoff).attempts + 1 + } + delay := certificateServiceBackoffBase << (attempts - 1) + if attempts > 10 || delay > certificateServiceBackoffMax { + delay = certificateServiceBackoffMax + } + r.certificateServiceFailures.Store(key, certificateServiceBackoff{attempts: attempts, nextTry: now.Add(delay), issue: issue}) + return delay +} + +// certificateServiceInBackoff reports whether the listener's last failure is +// still cooling off, and if so what it was told and how long is left. +func (r *GatewayReconciler) certificateServiceInBackoff(key certificateServiceKey, now time.Time) (certificateServiceBackoff, time.Duration, bool) { + previous, ok := r.certificateServiceFailures.Load(key) + if !ok { + return certificateServiceBackoff{}, 0, false + } + backoff := previous.(certificateServiceBackoff) + if !now.Before(backoff.nextTry) { + return backoff, 0, false + } + return backoff, backoff.nextTry.Sub(now), true +} + +func recordCertificateServiceFailure(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, reason string) { + certificateServiceFailuresTotal.WithLabelValues(gateway.Namespace, gateway.Name, string(listener), reason).Inc() +} + +// recordCertificateServiceState counts a TLSCertificate failure once per +// transition into it, since the same failure is read back on every pass for +// as long as it lasts. An empty reason means the listener is healthy. +func (r *GatewayReconciler) recordCertificateServiceState(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, reason string) { + key := certificateServiceKey{gateway: gateway.UID, listener: listener} + previous, _ := r.certificateServiceStates.Load(key) + if reason != "" && previous != reason { + recordCertificateServiceFailure(gateway, listener, reason) + } + if reason == "" { + r.certificateServiceStates.Delete(key) + return + } + r.certificateServiceStates.Store(key, reason) +} + +const ( + certificateServiceReasonStepFailed = "StepFailed" + certificateServiceReasonNotOwned = "NotOwned" + certificateServiceReasonRefused = "Refused" + certificateServiceReasonRejected = "Rejected" + certificateServiceReasonIssuanceFailed = "IssuanceFailed" + certificateServiceReasonNotReady = "NotReady" + certificateServiceReasonRenewalOverdue = "RenewalOverdue" + certificateServiceReasonMaterialRefused = "MaterialRefused" + certificateServiceReasonUntrustedChain = "UntrustedChain" +) + +func certificateServiceUnavailableMessage(hostname string) string { + return fmt.Sprintf("We couldn't request a TLS certificate for %s just now and will keep trying. HTTPS for this hostname stays as it is in the meantime.", hostname) +} + +func certificateRequestRefusedMessage(hostname, detail string) string { + return fmt.Sprintf("A TLS certificate cannot be issued for %s: %s", hostname, detail) +} + +func certificateRequestNotOwnedMessage(hostname string) string { + return fmt.Sprintf("A TLS certificate cannot be requested for %s because another resource holds the certificate request this hostname would use.", hostname) +} + +func certificateMaterialRefusedMessage(hostname string) string { + return fmt.Sprintf("The TLS certificate issued for %s did not pass our checks and was not applied. HTTPS for this hostname stays as it is in the meantime.", hostname) +} + +func certificateBeingReplacedMessage(hostname string) string { + return fmt.Sprintf("The TLS certificate request for %s is being replaced. HTTPS for this hostname stays as it is in the meantime.", hostname) +} + +func certificateRenewalOverdueMessage(hostname string, notAfter time.Time) string { + return fmt.Sprintf("The TLS certificate for %s expires on %s and has not been renewed.", hostname, notAfter.UTC().Format(time.DateOnly)) +} + +func certificateNotIssuedMessage(hostname, detail string) string { + if detail == "" { + return fmt.Sprintf("The TLS certificate for %s has not been issued after more than an hour.", hostname) + } + return fmt.Sprintf("The TLS certificate for %s has not been issued: %s", hostname, detail) +} + +// certificateServiceIssue is what one listener's pass could not do: the +// counter reason and the message for the customer. +type certificateServiceIssue struct { + reason string + message string +} + +// ensureListenerTLSCertificates is the certificate-service counterpart of +// ensureListenerCertificates for wildcard listeners: one TLSCertificate each in +// the project control plane, issued over DNS-01, and its issued Secret +// mirrored downstream under the name the listener references. Nothing here +// fails the gateway reconcile: a listener whose step failed keeps whatever +// Secret it has, the failure is returned as a message for its status, and that +// listener alone retries with backoff. +func (r *GatewayReconciler) ensureListenerTLSCertificates( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + downstreamStrategy downstreamclient.ResourceStrategy, + claimedHostnames []string, +) (result Result, issues map[gatewayv1.SectionName]string) { + logger := log.FromContext(ctx) + now := time.Now() + issues = make(map[gatewayv1.SectionName]string) + failing := make(map[gatewayv1.SectionName]string) + desiredCerts := sets.New[string]() + + requeueSooner := func(d time.Duration) { + if d > 0 && (result.RequeueAfter == 0 || d < result.RequeueAfter) { + result.RequeueAfter = d + } + } + + for _, l := range upstreamGateway.Spec.Listeners { + hostname, ok := r.listenerUsesCertificateService(l, claimedHostnames) + if !ok || !r.listenerIssuerResolvable(upstreamGateway, l) { + continue + } + certName := tlsCertificateName(upstreamGateway.Name, l.Name) + desiredCerts.Insert(certName) + requeueSooner(certificateServiceRecheck) + + key := certificateServiceKey{gateway: upstreamGateway.UID, listener: l.Name} + if backoff, remaining, cooling := r.certificateServiceInBackoff(key, now); cooling { + issues[l.Name] = backoff.issue.message + failing[l.Name] = backoff.issue.reason + requeueSooner(remaining) + continue + } + + mirrored, issue, err := r.ensureListenerTLSCertificate(ctx, upstreamClient, upstreamGateway, downstreamGateway, downstreamStrategy, l.Name, certName, hostname, now) + if err != nil { + logger.Error(err, "certificate service step failed", "listener", l.Name, "hostname", hostname) + if issue == nil { + issue = &certificateServiceIssue{reason: certificateServiceReasonStepFailed, message: certificateServiceUnavailableMessage(hostname)} + } + } + if issue != nil { + issues[l.Name] = issue.message + failing[l.Name] = issue.reason + recordCertificateServiceFailure(upstreamGateway, l.Name, issue.reason) + requeueSooner(r.certificateServiceRequeue(key, true, now, *issue)) + continue + } + r.certificateServiceRequeue(key, false, now, certificateServiceIssue{}) + if state, ok := r.certificateServiceStates.Load(key); ok { + failing[l.Name] = state.(string) + } + if mirrored { + requeueSooner(tlsCertificateMirrorAdmitDelay) + } + } + + cleanupKey := certificateServiceKey{gateway: upstreamGateway.UID} + if _, remaining, cooling := r.certificateServiceInBackoff(cleanupKey, now); cooling { + requeueSooner(remaining) + } else if err := r.deleteStaleTLSCertificates(ctx, upstreamClient, upstreamGateway, desiredCerts); err != nil { + logger.Error(err, "failed to clean up TLSCertificates") + requeueSooner(r.certificateServiceRequeue(cleanupKey, true, now, certificateServiceIssue{})) + } else { + r.certificateServiceRequeue(cleanupKey, false, now, certificateServiceIssue{}) + } + + r.forgetRemovedListeners(upstreamGateway) + publishCertificateServiceFailing(upstreamGateway, failing) + + return result, issues +} + +// publishCertificateServiceFailing replaces the gateway's failing-listener +// series with the listeners failing now, so one that recovered or left the +// service stops reporting. +func publishCertificateServiceFailing(upstreamGateway *gatewayv1.Gateway, failing map[gatewayv1.SectionName]string) { + clearCertificateServiceFailing(upstreamGateway) + for listener, reason := range failing { + certificateServiceListenerFailing.WithLabelValues(upstreamGateway.Namespace, upstreamGateway.Name, string(listener), reason).Set(1) + } +} + +func clearCertificateServiceFailing(upstreamGateway *gatewayv1.Gateway) { + certificateServiceListenerFailing.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) +} + +// ensureListenerTLSCertificate does one listener's pass: request or confirm +// its TLSCertificate, then mirror what the service issued. An issue says what +// the customer is told; an error alone is a transient step failure. +func (r *GatewayReconciler) ensureListenerTLSCertificate( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + downstreamStrategy downstreamclient.ResourceStrategy, + listenerName gatewayv1.SectionName, + certName string, + hostname string, + now time.Time, +) (bool, *certificateServiceIssue, error) { + secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) + + cert, state, err := r.ensureTLSCertificate(ctx, upstreamClient, upstreamGateway, certName, hostname) + switch { + case errors.Is(err, errTLSCertificateNotOwned): + return false, &certificateServiceIssue{reason: certificateServiceReasonNotOwned, message: certificateRequestNotOwnedMessage(hostname)}, err + case err != nil: + if detail, refused := requestRefusal(err); refused { + return false, &certificateServiceIssue{reason: certificateServiceReasonRefused, message: certificateRequestRefusedMessage(hostname, detail)}, err + } + return false, nil, err + case state != tlsCertificateSettled: + return false, &certificateServiceIssue{reason: certificateServiceReasonStepFailed, message: certificateBeingReplacedMessage(hostname)}, nil + } + + return r.mirrorTLSCertificateSecret(ctx, downstreamStrategy, upstreamGateway, downstreamGateway, cert, secretName, hostname, now) +} + +// requestRefusal reports whether the API refused the request itself rather +// than failing to process it, and the reason it gave. +func requestRefusal(err error) (string, bool) { + if !apierrors.IsInvalid(err) && !apierrors.IsForbidden(err) && !apierrors.IsBadRequest(err) { + return "", false + } + var status apierrors.APIStatus + if errors.As(err, &status) && status.Status().Message != "" { + return status.Status().Message, true + } + return err.Error(), true +} + +// forgetRemovedListeners drops the failure series and tracker entries of +// listeners the gateway no longer has, so a removed hostname stops reporting. +func (r *GatewayReconciler) forgetRemovedListeners(upstreamGateway *gatewayv1.Gateway) { + current := sets.New[gatewayv1.SectionName]() + for _, l := range upstreamGateway.Spec.Listeners { + current.Insert(l.Name) + } + if previous, ok := r.certificateServiceListeners.Load(upstreamGateway.UID); ok { + for listener := range previous.(sets.Set[gatewayv1.SectionName]) { + if current.Has(listener) { + continue + } + certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{ + jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name, metricLabelListener: string(listener), + }) + key := certificateServiceKey{gateway: upstreamGateway.UID, listener: listener} + r.certificateServiceStates.Delete(key) + r.certificateServiceFailures.Delete(key) + } + } + r.certificateServiceListeners.Store(upstreamGateway.UID, current) +} + +// forgetGateway drops everything the certificate service step remembers about +// a gateway that is gone. +func (r *GatewayReconciler) forgetGateway(upstreamGateway *gatewayv1.Gateway) { + r.certificateServiceStates.Range(func(k, _ any) bool { + if k.(certificateServiceKey).gateway == upstreamGateway.UID { + r.certificateServiceStates.Delete(k) + } + return true + }) + r.certificateServiceFailures.Range(func(k, _ any) bool { + if k.(certificateServiceKey).gateway == upstreamGateway.UID { + r.certificateServiceFailures.Delete(k) + } + return true + }) + r.certificateServiceListeners.Delete(upstreamGateway.UID) + certificateServiceFailuresTotal.DeletePartialMatch(prometheus.Labels{jsonKeyNamespace: upstreamGateway.Namespace, jsonKeyName: upstreamGateway.Name}) + clearCertificateServiceFailing(upstreamGateway) +} + +var errTLSCertificateNotOwned = errors.New("TLSCertificate exists but is not controlled by this Gateway") + +type tlsCertificateState int + +const ( + tlsCertificateSettled tlsCertificateState = iota + tlsCertificateReplacing +) + +// ensureTLSCertificate creates the listener's TLSCertificate or confirms the +// existing one. dnsNames is immutable on the service's API, so one of ours that +// no longer matches is deleted and requested again once it is gone. One this +// gateway does not control is never touched. +func (r *GatewayReconciler) ensureTLSCertificate( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + certName, hostname string, +) (*certificatesv1alpha1.TLSCertificate, tlsCertificateState, error) { + logger := log.FromContext(ctx) + + desiredSpec := certificatesv1alpha1.TLSCertificateSpec{ + DNSNames: []certificatesv1alpha1.DNSName{certificatesv1alpha1.DNSName(hostname)}, + Issuance: certificatesv1alpha1.IssuanceModeDNS01, + } + + cert := &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: upstreamGateway.Namespace, + Name: certName, + }, + } + + err := upstreamClient.Get(ctx, client.ObjectKeyFromObject(cert), cert) + switch { + case apierrors.IsNotFound(err): + if err := controllerutil.SetControllerReference(upstreamGateway, cert, upstreamClient.Scheme()); err != nil { + return nil, tlsCertificateSettled, fmt.Errorf("failed to set controller reference on TLSCertificate %s: %w", certName, err) + } + cert.Spec = desiredSpec + if err := upstreamClient.Create(ctx, cert); err != nil { + return nil, tlsCertificateSettled, fmt.Errorf("failed to create TLSCertificate %s: %w", certName, err) + } + logger.Info("TLSCertificate requested", "tlscertificate", certName, "hostname", hostname) + return cert, tlsCertificateSettled, nil + case err != nil: + return nil, tlsCertificateSettled, fmt.Errorf("failed to get TLSCertificate %s: %w", certName, err) + } + + if !metav1.IsControlledBy(cert, upstreamGateway) { + return nil, tlsCertificateSettled, fmt.Errorf("%w: %s", errTLSCertificateNotOwned, certName) + } + + if !cert.DeletionTimestamp.IsZero() { + return cert, tlsCertificateReplacing, nil + } + + if !slices.Equal(cert.Spec.DNSNames, desiredSpec.DNSNames) { + logger.Info("TLSCertificate no longer matches its listener, requesting it again", "tlscertificate", certName, "hostname", hostname) + if err := upstreamClient.Delete(ctx, cert, client.Preconditions{UID: &cert.UID}); client.IgnoreNotFound(err) != nil { + return nil, tlsCertificateSettled, fmt.Errorf("failed to delete TLSCertificate %s: %w", certName, err) + } + return cert, tlsCertificateReplacing, nil + } + + if cert.Spec.Issuance == desiredSpec.Issuance { + return cert, tlsCertificateSettled, nil + } + + cert.Spec.Issuance = desiredSpec.Issuance + if err := upstreamClient.Update(ctx, cert); err != nil { + return nil, tlsCertificateSettled, fmt.Errorf("failed to update TLSCertificate %s: %w", certName, err) + } + logger.Info("TLSCertificate reconciled", "tlscertificate", certName, "operation", "updated") + return cert, tlsCertificateSettled, nil +} + +// mirrorTLSCertificateSecret copies the service-side Secret holding the issued +// key pair, read with the operator's own credentials from the configured +// service namespace under the name the TLSCertificate's UID derives, into the +// downstream gateway namespace under the listener's secret name, stamped with +// the upstream-owner labels the federation policy selects. The material is +// parsed, matched, checked against the hostname, its expiry and, when +// configured, the trusted roots before it may replace what is serving. +func (r *GatewayReconciler) mirrorTLSCertificateSecret( + ctx context.Context, + downstreamStrategy downstreamclient.ResourceStrategy, + upstreamGateway *gatewayv1.Gateway, + downstreamGateway *gatewayv1.Gateway, + cert *certificatesv1alpha1.TLSCertificate, + secretName string, + hostname string, + now time.Time, +) (bool, *certificateServiceIssue, error) { + logger := log.FromContext(ctx) + + if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { + return false, nil, nil + } + if r.CertificateServiceReader == nil { + return false, nil, fmt.Errorf("certificate service enabled without a client for its cluster") + } + roots, err := r.certificateServiceRoots() + if err != nil { + return false, nil, fmt.Errorf("failed to load trusted roots: %w", err) + } + + downstreamClient := downstreamStrategy.GetClient() + mirror := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: downstreamGateway.Namespace, + Name: secretName, + }, + } + if err := downstreamClient.Get(ctx, client.ObjectKeyFromObject(mirror), mirror); client.IgnoreNotFound(err) != nil { + return false, nil, fmt.Errorf("failed to get Secret %s: %w", secretName, err) + } + if mirrorHoldsIssuance(mirror, cert) { + return false, nil, nil + } + + var source corev1.Secret + sourceKey := client.ObjectKey{ + Namespace: r.Config.Gateway.CertificateService.SecretNamespace, + Name: certificatesv1alpha1.StoredSecretName(cert.UID), + } + if err := r.CertificateServiceReader.Get(ctx, sourceKey, &source); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("TLSCertificate is Ready but its service-side Secret is not readable yet", "tlscertificate", cert.Name, "secret", sourceKey) + return false, nil, nil + } + return false, nil, fmt.Errorf("failed to get service-side Secret %s for TLSCertificate %s: %w", sourceKey, cert.Name, err) + } + + if err := validateIssuedMaterial(source.Data["tls.crt"], source.Data["tls.key"], hostname, now); err != nil { + logger.Info("refusing service-side Secret that does not hold a usable certificate for the hostname", + "tlscertificate", cert.Name, "secret", sourceKey, "reason", err.Error()) + return false, &certificateServiceIssue{reason: certificateServiceReasonMaterialRefused, message: certificateMaterialRefusedMessage(hostname)}, nil + } + if roots != nil { + if err := verifyIssuedChain(source.Data["tls.crt"], hostname, roots, now); err != nil { + logger.Info("refusing service-side Secret whose chain is not trusted", + "tlscertificate", cert.Name, "secret", sourceKey, "reason", err.Error()) + return false, &certificateServiceIssue{reason: certificateServiceReasonUntrustedChain, message: certificateMaterialRefusedMessage(hostname)}, nil + } + } + + op, err := controllerutil.CreateOrUpdate(ctx, downstreamClient, mirror, func() error { + if mirror.CreationTimestamp.IsZero() { + mirror.Type = corev1.SecretTypeTLS + } + if err := downstreamStrategy.SetControllerReference(ctx, upstreamGateway, mirror); err != nil { + return fmt.Errorf("failed to set strategy reference on Secret %s: %w", secretName, err) + } + mirror.Data = map[string][]byte{ + "tls.crt": source.Data["tls.crt"], + "tls.key": source.Data["tls.key"], + } + if ca := source.Data["ca.crt"]; len(ca) > 0 { + mirror.Data["ca.crt"] = ca + } + return nil + }) + if err != nil { + return false, nil, fmt.Errorf("failed to mirror Secret %s: %w", secretName, err) + } + if op != controllerutil.OperationResultNone { + logger.Info("issued Secret mirrored downstream", "secret", secretName, "operation", op) + } + + return op != controllerutil.OperationResultNone, nil, nil +} + +// certificateServiceRoots returns the roots an issued chain must verify +// against, or nil when chain verification is off. +func (r *GatewayReconciler) certificateServiceRoots() (*x509.CertPool, error) { + if !r.Config.Gateway.CertificateService.VerifyChain { + return nil, nil + } + if r.CertificateServiceRoots != nil { + return r.CertificateServiceRoots, nil + } + return x509.SystemCertPool() +} + +// mirrorHoldsIssuance reports whether the downstream Secret already carries the +// issuance the TLSCertificate describes, so the service cluster is not read +// again for it. +func mirrorHoldsIssuance(mirror *corev1.Secret, cert *certificatesv1alpha1.TLSCertificate) bool { + if mirror.CreationTimestamp.IsZero() || cert.Status.NotAfter == nil { + return false + } + leaf, err := parseLeafCertificate(mirror.Data["tls.crt"], mirror.Data["tls.key"]) + if err != nil { + return false + } + return leaf.NotAfter.Truncate(time.Second).Equal(cert.Status.NotAfter.Truncate(time.Second)) +} + +func parseLeafCertificate(certPEM, keyPEM []byte) (*x509.Certificate, error) { + keyPair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, err + } + if keyPair.Leaf != nil { + return keyPair.Leaf, nil + } + return x509.ParseCertificate(keyPair.Certificate[0]) +} + +// validateIssuedMaterial accepts only a matching key pair whose leaf covers the +// hostname and is valid now. +func validateIssuedMaterial(certPEM, keyPEM []byte, hostname string, now time.Time) error { + if len(certPEM) == 0 || len(keyPEM) == 0 { + return fmt.Errorf("secret holds no certificate material") + } + leaf, err := parseLeafCertificate(certPEM, keyPEM) + if err != nil { + return fmt.Errorf("certificate and key do not form a key pair: %w", err) + } + if err := leaf.VerifyHostname(hostname); err != nil { + return fmt.Errorf("certificate does not cover %s: %w", hostname, err) + } + if now.Before(leaf.NotBefore) { + return fmt.Errorf("certificate is not valid until %s", leaf.NotBefore.UTC().Format(time.RFC3339)) + } + if !leaf.NotAfter.After(now.Add(listenerCertExpiryMargin)) { + return fmt.Errorf("certificate expired at %s", leaf.NotAfter.UTC().Format(time.RFC3339)) + } + return nil +} + +// verifyIssuedChain checks that the PEM chain, leaf first, builds to one of +// the trusted roots for server authentication on the hostname. +func verifyIssuedChain(certPEM []byte, hostname string, roots *x509.CertPool, now time.Time) error { + var chain []*x509.Certificate + for rest := certPEM; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate chain: %w", err) + } + chain = append(chain, c) + } + if len(chain) == 0 { + return fmt.Errorf("secret holds no certificate") + } + intermediates := x509.NewCertPool() + for _, c := range chain[1:] { + intermediates.AddCert(c) + } + _, err := chain[0].Verify(x509.VerifyOptions{ + DNSName: hostname, + Roots: roots, + Intermediates: intermediates, + CurrentTime: now, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + return err +} + +// deleteStaleTLSCertificates removes TLSCertificates this gateway controls that +// no listener wants any more. Ownership is the controller reference's UID, so a +// namesake another gateway owns is left alone. +func (r *GatewayReconciler) deleteStaleTLSCertificates( + ctx context.Context, + upstreamClient client.Client, + upstreamGateway *gatewayv1.Gateway, + desiredCerts sets.Set[string], +) error { + logger := log.FromContext(ctx) + + var certs certificatesv1alpha1.TLSCertificateList + if err := upstreamClient.List(ctx, &certs, client.InNamespace(upstreamGateway.Namespace)); err != nil { + return fmt.Errorf("failed to list TLSCertificates: %w", err) + } + + for i := range certs.Items { + cert := &certs.Items[i] + if desiredCerts.Has(cert.Name) || !metav1.IsControlledBy(cert, upstreamGateway) || !cert.DeletionTimestamp.IsZero() { + continue + } + logger.Info("deleting stale TLSCertificate", "tlscertificate", cert.Name) + if err := upstreamClient.Delete(ctx, cert, client.Preconditions{UID: &cert.UID}); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete stale TLSCertificate %s: %w", cert.Name, err) + } + } + + return nil +} + +// servingSecretHealth is listenerSecretHealth plus a check that the leaf +// actually covers the hostname, so a listener whose hostname changed cannot +// keep serving the previous name's certificate. It also returns the served +// leaf when there is one. +func servingSecretHealth( + ctx context.Context, + downstreamClient client.Client, + downstreamNamespace string, + secretName string, + hostname string, + now time.Time, +) (listenerCertStatus, *x509.Certificate) { + status := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + if !status.healthy { + return status, nil + } + var secret corev1.Secret + if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { + return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), pending: true, secretName: secretName}, nil + } + if err := validateIssuedMaterial(secret.Data["tls.crt"], secret.Data["tls.key"], hostname, now); err != nil { + return listenerCertStatus{reason: gatewayv1.ListenerReasonInvalidCertificateRef, message: certMissingMessage(hostname), secretName: secretName}, nil + } + leaf, err := parseLeafCertificate(secret.Data["tls.crt"], secret.Data["tls.key"]) + if err != nil { + return status, nil + } + return status, leaf +} + +// renewalOverdue reports whether a served certificate is well past the point +// it should have been replaced, whatever the TLSCertificate's status says. +func renewalOverdue(leaf *x509.Certificate, now time.Time) bool { + if leaf == nil { + return false + } + lifetime := leaf.NotAfter.Sub(leaf.NotBefore) + return leaf.NotAfter.Sub(now) < lifetime/renewalOverdueDivisor +} + +// tlsCertificateFailure says why a TLSCertificate is not producing a usable +// certificate, as a counter reason and the service's own explanation, or "" +// while it is healthy, still within its grace, or waiting on DNS records the +// customer has to publish. +func tlsCertificateFailure(cert *certificatesv1alpha1.TLSCertificate, now time.Time) (string, string) { + if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + return certificateServiceReasonRejected, accepted.Message + } + if !cert.DeletionTimestamp.IsZero() { + return "", "" + } + if issuing := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionIssuing); issuing != nil && + issuing.Status == metav1.ConditionFalse && issuing.Reason == "IssuanceFailed" { + return certificateServiceReasonIssuanceFailed, issuing.Message + } + ready := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) + if ready != nil && ready.Status == metav1.ConditionTrue { + return "", "" + } + if delegation := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionDNSDelegationReady); delegation != nil && delegation.Status == metav1.ConditionFalse { + return "", "" + } + since := cert.CreationTimestamp.Time + detail := "" + if ready != nil { + since = ready.LastTransitionTime.Time + detail = ready.Message + } + if since.IsZero() || now.Sub(since) < tlsCertificateIssueGrace { + return "", "" + } + return certificateServiceReasonNotReady, detail +} + +// listenerTLSCertificateHealth keeps a listener serving whenever its downstream +// Secret holds a usable certificate, whatever the TLSCertificate is doing. A +// replacement that is failing, stalled or overdue is reported as a blocked +// renewal while the listener serves, and as blocked issuance when nothing does, +// and counted once per transition. +func (r *GatewayReconciler) listenerTLSCertificateHealth( + ctx context.Context, + upstreamClient client.Client, + downstreamClient client.Client, + downstreamNamespace string, + upstreamGateway *gatewayv1.Gateway, + listenerName gatewayv1.SectionName, + hostname string, + now time.Time, +) listenerCertStatus { + logger := log.FromContext(ctx) + + certName := tlsCertificateName(upstreamGateway.Name, listenerName) + secretName := listenerCertificateSecretName(upstreamGateway.Name, listenerName) + + secretStatus, leaf := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now) + + var cert certificatesv1alpha1.TLSCertificate + if err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamGateway.Namespace, Name: certName}, &cert); err != nil { + if !apierrors.IsNotFound(err) { + logger.Error(err, "failed to get listener TLSCertificate", "tlscertificate", certName) + } + reason := "" + if secretStatus.healthy && renewalOverdue(leaf, now) { + reason = certificateServiceReasonRenewalOverdue + secretStatus.renewalBlocked = certificateRenewalOverdueMessage(hostname, leaf.NotAfter) + } + r.recordCertificateServiceState(upstreamGateway, listenerName, reason) + if secretStatus.healthy { + return secretStatus + } + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certIssuanceFailingMessage(hostname), + pending: true, + secretName: secretName, + } + } + + reason, detail := tlsCertificateFailure(&cert, now) + if reason == "" && secretStatus.healthy && renewalOverdue(leaf, now) { + reason = certificateServiceReasonRenewalOverdue + } + r.recordCertificateServiceState(upstreamGateway, listenerName, reason) + + if secretStatus.healthy { + switch reason { + case "": + case certificateServiceReasonRenewalOverdue: + secretStatus.renewalBlocked = certificateRenewalOverdueMessage(hostname, leaf.NotAfter) + case certificateServiceReasonRejected: + secretStatus.renewalBlocked = tlsCertificateRejectedMessage(hostname, detail) + default: + secretStatus.renewalBlocked = certificateNotIssuedMessage(hostname, detail) + } + return secretStatus + } + + switch reason { + case certificateServiceReasonRejected: + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: tlsCertificateRejectedMessage(hostname, detail), + secretName: secretName, + } + case certificateServiceReasonIssuanceFailed, certificateServiceReasonNotReady: + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certificateNotIssuedMessage(hostname, detail), + pending: true, + issuanceBlocked: true, + secretName: secretName, + } + } + + if !apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certIssuanceFailingMessage(hostname), + pending: true, + secretName: secretName, + } + } + + if cert.Status.NotBefore != nil && cert.Status.NotBefore.After(now) { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certNotYetValidMessage(hostname), + secretName: secretName, + } + } + if cert.Status.NotAfter != nil && !cert.Status.NotAfter.After(now.Add(listenerCertExpiryMargin)) { + return listenerCertStatus{ + reason: gatewayv1.ListenerReasonInvalidCertificateRef, + message: certExpiredMessage(hostname), + notAfter: cert.Status.NotAfter, + secretName: secretName, + } + } + + secretStatus.pending = true + return secretStatus +} + +func tlsCertificateRejectedMessage(hostname, detail string) string { + if detail == "" { + return certIssuanceFailingMessage(hostname) + } + return fmt.Sprintf("We couldn't issue a TLS certificate for %s, so HTTPS for this hostname is unavailable: %s", hostname, detail) +} + +// listGatewaysForTLSCertificateFunc enqueues the Gateway that controls a +// TLSCertificate, in the cluster the TLSCertificate lives in. +func (r *GatewayReconciler) listGatewaysForTLSCertificateFunc(clusterName multicluster.ClusterName, _ cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []mcreconcile.Request { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.Kind != KindGateway { + return nil + } + return []mcreconcile.Request{{ + ClusterName: clusterName, + Request: reconcile.Request{ + NamespacedName: client.ObjectKey{Namespace: obj.GetNamespace(), Name: owner.Name}, + }, + }} + }) +} diff --git a/internal/controller/gateway_certificate_service_test.go b/internal/controller/gateway_certificate_service_test.go new file mode 100644 index 00000000..38e20446 --- /dev/null +++ b/internal/controller/gateway_certificate_service_test.go @@ -0,0 +1,1338 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "strings" + "sync/atomic" + "testing" + "time" + + cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + "go.datum.net/network-services-operator/internal/config" + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" + gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" +) + +func newCertificateServiceTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + testScheme := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(testScheme)) + require.NoError(t, gatewayv1.Install(testScheme)) + require.NoError(t, discoveryv1.AddToScheme(testScheme)) + require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + require.NoError(t, cmv1.AddToScheme(testScheme)) + require.NoError(t, envoygatewayv1alpha1.AddToScheme(testScheme)) + require.NoError(t, certificatesv1alpha1.AddToScheme(testScheme)) + return testScheme +} + +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + pool *x509.CertPool +} + +func newTestCA(t *testing.T) testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test root"}, + NotBefore: time.Now().Add(-24 * time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + pool := x509.NewCertPool() + pool.AddCert(cert) + return testCA{cert: cert, key: key, pool: pool} +} + +func (ca testCA) issue(t *testing.T, hostname string, notBefore, notAfter time.Time) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: hostname}, + DNSNames: []string{hostname}, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, &key.PublicKey, ca.key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) +} + +func tlsListener(name gatewayv1.SectionName, hostname string) gatewayv1.Listener { + return gatewayv1.Listener{ + Name: name, + Protocol: gatewayv1.HTTPSProtocolType, + Port: DefaultHTTPSPort, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + AllowedRoutes: &gatewayv1.AllowedRoutes{ + Namespaces: &gatewayv1.RouteNamespaces{From: ptr.To(gatewayv1.NamespacesFromSame)}, + }, + TLS: &gatewayv1.ListenerTLSConfig{ + Mode: ptr.To(gatewayv1.TLSModeTerminate), + Options: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, + }, + }, + } +} + +func dnsProvenDomain(namespace string) *networkingv1alpha.Domain { + return &networkingv1alpha.Domain{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "example.com"}, + Spec: networkingv1alpha.DomainSpec{DomainName: "example.com"}, + Status: networkingv1alpha.DomainStatus{Conditions: []metav1.Condition{ + {Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue, Reason: networkingv1alpha.DomainReasonVerified}, + {Type: networkingv1alpha.DomainConditionVerifiedDNS, Status: metav1.ConditionTrue, Reason: networkingv1alpha.DomainReasonVerified}, + }}, + } +} + +func certificateServiceGatewayConfig() config.GatewayConfig { + return config.GatewayConfig{ + DownstreamGatewayClassName: "test-suite", + DownstreamHostnameAccountingNamespace: "default", + TargetDomain: "test-suite.com", + DefaultListenerTLSSecretName: "wildcard-test-suite-tls", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + gatewayv1.AnnotationKey(certificateIssuerTLSOption): autoIssuerSentinel, + }, + ClusterIssuerMap: map[string]string{autoIssuerSentinel: "datum-gateway-http"}, + CertificateService: config.CertificateServiceConfig{Enabled: true, SecretNamespace: "certificates-system", VerifyChain: true}, + } +} + +func TestEnsureDownstreamGatewayCertificateService(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + + upstreamNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}, + } + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + + const ( + gatewayName = "test-gw" + listenerName = gatewayv1.SectionName("https-hostname-0") + hostname = "*.shop.example.com" + serviceNS = "certificates-system" + upstreamCluster = "test" + ) + certName := tlsCertificateName(gatewayName, listenerName) + secretName := listenerCertificateSecretName(gatewayName, listenerName) + certUID := uuid.NewUUID() + serviceSecret := certificatesv1alpha1.StoredSecretName(certUID) + testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + ca := newTestCA(t) + + now := time.Now() + serviceCertPEM, serviceKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + otherCertPEM, otherKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + servingCertPEM, servingKeyPEM := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + + tlsSecret := func(namespace, name string, certPEM, keyPEM []byte) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": certPEM, "tls.key": keyPEM}, + } + } + + newTLSCertificate := func(owner *gatewayv1.Gateway, name string, mutate func(*certificatesv1alpha1.TLSCertificate)) *certificatesv1alpha1.TLSCertificate { + cert := &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: name, UID: certUID}, + Spec: certificatesv1alpha1.TLSCertificateSpec{ + DNSNames: []certificatesv1alpha1.DNSName{hostname}, + Issuance: certificatesv1alpha1.IssuanceModeDNS01, + }, + } + if owner != nil { + require.NoError(t, controllerutil.SetControllerReference(owner, cert, testScheme)) + } + if mutate != nil { + mutate(cert) + } + return cert + } + + readyStatus := func(cert *certificatesv1alpha1.TLSCertificate) { + cert.Status.NotBefore = &metav1.Time{Time: now.Add(-time.Hour)} + cert.Status.NotAfter = &metav1.Time{Time: now.Add(60 * 24 * time.Hour)} + apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue, Reason: "Accepted"}) + apimeta.SetStatusCondition(&cert.Status.Conditions, metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}) + } + + serving := func() []client.Object { + return []client.Object{tlsSecret(downstreamNamespaceName, secretName, servingCertPEM, servingKeyPEM)} + } + + type env struct { + upstream client.Client + downstream client.Client + result Result + reconciler *GatewayReconciler + } + + tests := []struct { + name string + upstreamObjects func(gw *gatewayv1.Gateway) []client.Object + upstreamCreate func(obj client.Object) error + downstreamObjects func() []client.Object + serviceObjects []client.Object + serviceForbidden bool + assert func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) + }{ + { + name: "requests a DNS-01 TLSCertificate upstream and withholds the listener until it issues", + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) + assert.Equal(t, []certificatesv1alpha1.DNSName{hostname}, cert.Spec.DNSNames) + assert.Equal(t, certificatesv1alpha1.IssuanceModeDNS01, cert.Spec.Issuance) + assert.True(t, metav1.IsControlledBy(&cert, upstreamGateway), "TLSCertificate should be controlled by the upstream Gateway") + + var legacy cmv1.CertificateList + require.NoError(t, e.downstream.List(ctx, &legacy)) + assert.Empty(t, legacy.Items, "no cert-manager Certificate is created for a wildcard") + + var routes gatewayv1.HTTPRouteList + require.NoError(t, e.downstream.List(ctx, &routes, client.HasLabels{"meta.datumapis.com/http01-solver"})) + assert.Empty(t, routes.Items, "a wildcard is never answered over HTTP") + + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "listener must be withheld until a certificate is issued") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, gatewayutil.DefaultHTTPSListenerName), "shared wildcard listener is untouched") + assert.Equal(t, time.Minute, e.result.RequeueAfter, "a listener waiting on issuance is looked at again soon") + }, + }, + { + name: "moves an existing request onto DNS-01", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Spec.Issuance = certificatesv1alpha1.IssuanceModeAuto + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) + assert.Equal(t, certificatesv1alpha1.IssuanceModeDNS01, cert.Spec.Issuance) + }, + }, + { + name: "mirrors the service-side Secret when Ready and admits the listener", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: []client.Object{ + tlsSecret(serviceNS, serviceSecret, serviceCertPEM, serviceKeyPEM), + tlsSecret(serviceNS, secretName, otherCertPEM, otherKeyPEM), + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + var mirror corev1.Secret + require.NoError(t, e.downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &mirror)) + assert.Equal(t, corev1.SecretTypeTLS, mirror.Type) + assert.Equal(t, serviceCertPEM, mirror.Data["tls.crt"], "edge material comes from the service-side Secret") + assert.Equal(t, serviceKeyPEM, mirror.Data["tls.key"]) + assert.NotEqual(t, otherCertPEM, mirror.Data["tls.crt"], "only the Secret named after the TLSCertificate's UID is read") + assert.Equal(t, "cluster-"+upstreamCluster, mirror.Labels[downstreamclient.UpstreamOwnerClusterNameLabel]) + assert.Equal(t, upstreamNamespace.Name, mirror.Labels[downstreamclient.UpstreamOwnerNamespaceLabel]) + assert.Equal(t, KindGateway, mirror.Labels[downstreamclient.UpstreamOwnerKindLabel]) + + assert.Equal(t, tlsCertificateMirrorAdmitDelay, e.result.RequeueAfter, "a fresh mirror asks for a prompt re-evaluation") + + health := e.reconciler.evaluateListenerCertHealth(ctx, e.upstream, e.downstream, downstreamNamespaceName, upstreamGateway, []string{hostname}) + status, gated := health[listenerName] + require.True(t, gated) + assert.True(t, status.healthy, "listener is admitted on the next pass: %s", status.message) + assert.Empty(t, status.renewalBlocked) + assert.Equal(t, secretName, status.secretName) + require.NotNil(t, status.notAfter) + assert.WithinDuration(t, now.Add(60*24*time.Hour), status.notAfter.Time, time.Minute) + }, + }, + { + name: "a rejected request withholds the listener and tells the customer why", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, + Reason: "DeniedDomain", Message: "names under datum.net are denied", + }) + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + resolved := listenerCondition(t, upstreamGateway, listenerName, string(gatewayv1.ListenerConditionResolvedRefs)) + assert.Equal(t, metav1.ConditionFalse, resolved.Status) + assert.Contains(t, resolved.Message, "names under datum.net are denied") + assert.NotContains(t, resolved.Message, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRejected), 1.0) + }, + }, + { + name: "a request the API refuses tells the customer why instead of promising to keep trying", + upstreamCreate: func(obj client.Object) error { + if _, ok := obj.(*certificatesv1alpha1.TLSCertificate); !ok { + return nil + } + return apierrors.NewInvalid(schema.GroupKind{Group: "certificates.miloapis.com", Kind: KindTLSCertificate}, certName, + field.ErrorList{field.Forbidden(field.NewPath("spec", "dnsNames"), "names under datum.net are denied")}) + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Contains(t, blocked.Message, "names under datum.net are denied") + assert.NotContains(t, blocked.Message, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRefused), 1.0) + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) + }, + }, + { + name: "never deletes a mismatched TLSCertificate another owner controls", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + other := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: "other-gw", UID: uuid.NewUUID()}} + return []client.Object{newTLSCertificate(other, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"*.previous.example.com"} + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &cert)) + assert.Equal(t, []certificatesv1alpha1.DNSName{"*.previous.example.com"}, cert.Spec.DNSNames, "someone else's TLSCertificate is left alone") + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter, "the clash is retried with backoff, not an error") + resolved := listenerCondition(t, upstreamGateway, listenerName, string(gatewayv1.ListenerConditionResolvedRefs)) + assert.Contains(t, resolved.Message, "another resource holds the certificate request") + }, + }, + { + name: "reads the stored Secret only from the service namespace", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: []client.Object{tlsSecret("victim-namespace", serviceSecret, serviceCertPEM, serviceKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) + }, + }, + { + name: "refuses service-side material whose key does not match", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, otherKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{}))) + }, + }, + { + name: "refuses service-side material for another hostname", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceObjects: func() []client.Object { + crt, key := ca.issue(t, "*.example.com", now.Add(-time.Hour), now.Add(60*24*time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.True(t, apierrors.IsNotFound(e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &corev1.Secret{})), "*.example.com does not cover *.shop.example.com") + }, + }, + { + name: "refuses a chain the configured roots do not trust and keeps the serving Secret", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(61 * 24 * time.Hour)} + })} + }, + downstreamObjects: serving, + serviceObjects: func() []client.Object { + crt, key := generateTLSKeyPair(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var secret corev1.Secret + require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.Equal(t, servingCertPEM, secret.Data["tls.crt"], "an untrusted chain never replaces what serves") + blocked := assertListenerRenewalBlocked(t, upstreamGateway, "did not pass our checks") + assert.NotContains(t, blocked.Message, "keep trying") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonUntrustedChain), 1.0) + }, + }, + { + name: "a rejection while the previous certificate serves is reported as a blocked renewal", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain", Message: "names under datum.net are denied", + }) + })} + }, + downstreamObjects: serving, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") + assertListenerRenewalBlocked(t, upstreamGateway, "names under datum.net are denied") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRejected), 1.0) + }, + }, + { + name: "a failed renewal order while the previous certificate serves is reported and counted", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(60 * 24 * time.Hour).Add(time.Second)} + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionFalse, Reason: "IssuanceFailed", Message: "DNS problem: NXDOMAIN looking up TXT", + }) + })} + }, + downstreamObjects: serving, + serviceObjects: []client.Object{tlsSecret(serviceNS, serviceSecret, serviceCertPEM, serviceKeyPEM)}, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + assertListenerRenewalBlocked(t, upstreamGateway, "NXDOMAIN") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonIssuanceFailed), 1.0) + }, + }, + { + name: "a request stuck short of Ready with nothing serving is reported as blocked issuance", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.Conditions = []metav1.Condition{{ + Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", + Message: "The certificate has not been issued yet.", LastTransitionTime: metav1.NewTime(now.Add(-2 * time.Hour)), + }} + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Contains(t, blocked.Message, "has not been issued yet") + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNotReady), 1.0) + assert.Equal(t, 1.0, gaugeValue(t, certificateServiceListenerFailing, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonNotReady), "a lasting failure stays visible after its one count") + }, + }, + { + name: "waiting on the customer's DNS records is progress, not a failure", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.Conditions = []metav1.Condition{ + {Type: certificatesv1alpha1.ConditionDNSDelegationReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.NewTime(now.Add(-48 * time.Hour))}, + {Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.NewTime(now.Add(-48 * time.Hour))}, + } + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + for _, ls := range upstreamGateway.Status.Listeners { + if ls.Name == listenerName { + assert.Nil(t, apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateIssuanceBlocked)) + } + } + }, + }, + { + name: "a served certificate past its renewal point with no newer issuance is overdue", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotBefore = &metav1.Time{Time: now.Add(-80 * 24 * time.Hour)} + c.Status.NotAfter = &metav1.Time{Time: now.Add(10 * 24 * time.Hour)} + })} + }, + downstreamObjects: func() []client.Object { + crt, key := ca.issue(t, hostname, now.Add(-80*24*time.Hour), now.Add(10*24*time.Hour)) + return []client.Object{tlsSecret(downstreamNamespaceName, secretName, crt, key)} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + assertListenerRenewalBlocked(t, upstreamGateway, "has not been renewed") + assert.Equal(t, 1.0, gaugeValue(t, certificateServiceListenerFailing, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRenewalOverdue)) + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonRenewalOverdue), 1.0) + }, + }, + { + name: "a forbidden service-side read while the previous certificate serves is reported as a blocked renewal", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(90 * 24 * time.Hour)} + })} + }, + downstreamObjects: serving, + serviceForbidden: true, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName), "the listener keeps serving") + assertListenerRenewalBlocked(t, upstreamGateway, "keep trying") + assert.Equal(t, certificateServiceBackoffBase, e.result.RequeueAfter) + assert.GreaterOrEqual(t, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), 1.0) + }, + }, + { + name: "a forbidden service-side read with nothing serving is reported as blocked issuance", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, readyStatus)} + }, + serviceForbidden: true, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + assert.Nil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + blocked := listenerCondition(t, upstreamGateway, listenerName, listenerConditionCertificateIssuanceBlocked) + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Equal(t, listenerReasonIssuanceFailing, blocked.Reason) + assert.Contains(t, blocked.Message, "keep trying") + }, + }, + { + name: "refuses expired service-side material and keeps the serving Secret", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + readyStatus(c) + c.Status.NotAfter = &metav1.Time{Time: now.Add(-time.Hour)} + })} + }, + downstreamObjects: serving, + serviceObjects: func() []client.Object { + crt, key := ca.issue(t, hostname, now.Add(-48*time.Hour), now.Add(-time.Hour)) + return []client.Object{tlsSecret(serviceNS, serviceSecret, crt, key)} + }(), + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + var secret corev1.Secret + require.NoError(t, e.downstream.Get(context.Background(), client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &secret)) + assert.Equal(t, servingCertPEM, secret.Data["tls.crt"], "a valid serving Secret is never replaced by material that fails validation") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, listenerName)) + }, + }, + { + name: "deletes the TLSCertificate of a listener that is gone and leaves others alone", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{ + newTLSCertificate(gw, tlsCertificateName(gatewayName, "https-hostname-9"), nil), + newTLSCertificate(nil, "someone-elses-cert", nil), + } + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + ctx := context.Background() + assert.True(t, apierrors.IsNotFound(e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, "https-hostname-9")}, &certificatesv1alpha1.TLSCertificate{}))) + assert.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: "someone-elses-cert"}, &certificatesv1alpha1.TLSCertificate{})) + assert.NoError(t, e.upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{})) + }, + }, + { + name: "requests the TLSCertificate again when its immutable spec no longer matches", + upstreamObjects: func(gw *gatewayv1.Gateway) []client.Object { + return []client.Object{newTLSCertificate(gw, certName, func(c *certificatesv1alpha1.TLSCertificate) { + c.Spec.DNSNames = []certificatesv1alpha1.DNSName{"*.previous.example.com"} + })} + }, + assert: func(t *testing.T, e env, upstreamGateway, downstreamGateway *gatewayv1.Gateway) { + err := e.upstream.Get(context.Background(), client.ObjectKey{Namespace: upstreamNamespace.Name, Name: certName}, &certificatesv1alpha1.TLSCertificate{}) + assert.True(t, apierrors.IsNotFound(err), "mismatched TLSCertificate is deleted so the next pass requests it afresh") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(listenerName, hostname)) + }) + + upstreamObjects := []client.Object{ + dnsProvenDomain(upstreamNamespace.Name), + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: gatewayv1.GatewayClassSpec{ControllerName: gatewayv1.GatewayController("test")}, + }, + } + if tt.upstreamObjects != nil { + upstreamObjects = append(upstreamObjects, tt.upstreamObjects(upstreamGateway)...) + } + var downstreamObjects []client.Object + if tt.downstreamObjects != nil { + downstreamObjects = tt.downstreamObjects() + } + for _, obj := range append(append([]client.Object{}, upstreamObjects...), downstreamObjects...) { + if obj.GetUID() == "" { + obj.SetUID(uuid.NewUUID()) + } + obj.SetCreationTimestamp(metav1.Now()) + } + + upstreamBuilder := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace). + WithObjects(upstreamObjects...). + WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). + WithStatusSubresource(upstreamObjects...) + if tt.upstreamCreate != nil { + upstreamBuilder = upstreamBuilder.WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if err := tt.upstreamCreate(obj); err != nil { + return err + } + return cl.Create(ctx, obj, opts...) + }, + }) + } + fakeUpstreamClient := upstreamBuilder.Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamObjects...). + WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}). + Build() + + serviceBuilder := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(tt.serviceObjects...) + if tt.serviceForbidden { + serviceBuilder = serviceBuilder.WithInterceptorFuncs(interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return apierrors.NewForbidden(corev1.Resource("secrets"), serviceSecret, nil) + }, + }) + } + + ctx := log.IntoContext(context.Background(), logger) + + reconciler := &GatewayReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + Config: testCfg, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + CertificateServiceReader: serviceBuilder.Build(), + CertificateServiceRoots: ca.pool, + } + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy(upstreamCluster, fakeUpstreamClient, fakeDownstreamClient) + + reconciler.prepareUpstreamGateway(upstreamGateway) + result, downstreamGateway := reconciler.ensureDownstreamGateway(ctx, upstreamCluster, fakeUpstreamClient, upstreamGateway, downstreamStrategy) + require.NoError(t, result.Err) + _, err := result.Complete(ctx) + require.NoError(t, err) + + updatedUpstream := &gatewayv1.Gateway{} + require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), updatedUpstream)) + + tt.assert(t, env{ + upstream: fakeUpstreamClient, + downstream: fakeDownstreamClient, + result: result, + reconciler: reconciler, + }, updatedUpstream, downstreamGateway) + }) + } +} + +func listenerCondition(t *testing.T, gateway *gatewayv1.Gateway, listener gatewayv1.SectionName, conditionType string) *metav1.Condition { + t.Helper() + for _, ls := range gateway.Status.Listeners { + if ls.Name != listener { + continue + } + condition := apimeta.FindStatusCondition(ls.Conditions, conditionType) + require.NotNil(t, condition, "listener %s has no %s condition", listener, conditionType) + return condition + } + t.Fatalf("no status for listener %s", listener) + return nil +} + +func assertListenerRenewalBlocked(t *testing.T, gateway *gatewayv1.Gateway, want string) *metav1.Condition { + t.Helper() + for _, ls := range gateway.Status.Listeners { + blocked := apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked) + if blocked == nil { + continue + } + resolved := apimeta.FindStatusCondition(ls.Conditions, string(gatewayv1.ListenerConditionResolvedRefs)) + require.NotNil(t, resolved) + assert.Equal(t, metav1.ConditionTrue, resolved.Status, "the listener itself stays resolved") + assert.Equal(t, metav1.ConditionTrue, blocked.Status) + assert.Equal(t, listenerReasonRenewalFailing, blocked.Reason) + assert.Contains(t, blocked.Message, want) + return blocked + } + t.Fatal("no listener reports a blocked renewal") + return nil +} + +func seriesInNamespace(t *testing.T, vec prometheus.Collector, namespace string) int { + t.Helper() + ch := make(chan prometheus.Metric, 1024) + vec.Collect(ch) + close(ch) + count := 0 + for metric := range ch { + var m dto.Metric + require.NoError(t, metric.Write(&m)) + for _, label := range m.GetLabel() { + if label.GetName() == jsonKeyNamespace && label.GetValue() == namespace { + count++ + } + } + } + return count +} + +func gaugeValue(t *testing.T, vec *prometheus.GaugeVec, labels ...string) float64 { + t.Helper() + var m dto.Metric + require.NoError(t, vec.WithLabelValues(labels...).Write(&m)) + return m.GetGauge().GetValue() +} + +func counterValue(t *testing.T, vec *prometheus.CounterVec, labels ...string) float64 { + t.Helper() + var m dto.Metric + require.NoError(t, vec.WithLabelValues(labels...).Write(&m)) + return m.GetCounter().GetValue() +} + +// certificateServiceHarness drives full gateway reconciles against fake +// clients that persist across passes, so a test can change the operator's +// config between them as a rollout would. +type certificateServiceHarness struct { + t *testing.T + ctx context.Context + cfg config.NetworkServicesOperator + upstream client.Client + downstream client.Client + service client.Reader + roots *x509.CertPool + gateway client.ObjectKey + namespace string + reconciler *GatewayReconciler +} + +func (h *certificateServiceHarness) reconcile() (*gatewayv1.Gateway, *gatewayv1.Gateway, Result) { + h.t.Helper() + if h.reconciler == nil || h.reconciler.Config.Gateway.CertificateService.Enabled != h.cfg.Gateway.CertificateService.Enabled { + h.reconciler = &GatewayReconciler{ + mgr: &fakeMockManager{cl: h.upstream}, + Config: h.cfg, + DownstreamCluster: &fakeCluster{cl: h.downstream}, + CertificateServiceReader: h.service, + CertificateServiceRoots: h.roots, + } + } + var current gatewayv1.Gateway + require.NoError(h.t, h.upstream.Get(h.ctx, h.gateway, ¤t)) + h.reconciler.prepareUpstreamGateway(¤t) + strategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", h.upstream, h.downstream) + result, downstreamGateway := h.reconciler.ensureDownstreamGateway(h.ctx, "test", h.upstream, ¤t, strategy) + require.NoError(h.t, result.Err) + _, err := result.Complete(h.ctx) + require.NoError(h.t, err) + require.NoError(h.t, h.upstream.Get(h.ctx, h.gateway, ¤t)) + + var claims corev1.ConfigMapList + require.NoError(h.t, h.downstream.List(h.ctx, &claims, client.InNamespace(h.cfg.Gateway.DownstreamHostnameAccountingNamespace))) + for i := range claims.Items { + stampCreated(h.t, h.ctx, h.downstream, &claims.Items[i]) + } + var gateways gatewayv1.GatewayList + require.NoError(h.t, h.downstream.List(h.ctx, &gateways, client.InNamespace(h.namespace))) + for i := range gateways.Items { + stampCreated(h.t, h.ctx, h.downstream, &gateways.Items[i]) + } + var certs cmv1.CertificateList + require.NoError(h.t, h.downstream.List(h.ctx, &certs, client.InNamespace(h.namespace))) + for i := range certs.Items { + stampCreated(h.t, h.ctx, h.downstream, &certs.Items[i]) + } + var secrets corev1.SecretList + require.NoError(h.t, h.downstream.List(h.ctx, &secrets, client.InNamespace(h.namespace))) + for i := range secrets.Items { + stampCreated(h.t, h.ctx, h.downstream, &secrets.Items[i]) + } + var tlsCerts certificatesv1alpha1.TLSCertificateList + require.NoError(h.t, h.upstream.List(h.ctx, &tlsCerts)) + for i := range tlsCerts.Items { + stampCreated(h.t, h.ctx, h.upstream, &tlsCerts.Items[i]) + } + return ¤t, downstreamGateway, result +} + +func TestCertificateServiceLeavesExactHostnamesOnCertManager(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + const ( + gatewayName = "mixed-gw" + serviceNS = "certificates-system" + exact = "app.example.com" + wildcard = "*.shop.example.com" + ) + exactListener := gatewayv1.SectionName("https-hostname-0") + wildcardListenerName := gatewayv1.SectionName("https-hostname-1") + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "mixed", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + ca := newTestCA(t) + now := time.Now() + + cfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + upstreamGateway := newGateway(cfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(exactListener, exact), tlsListener(wildcardListenerName, wildcard)) + }) + domain := dnsProvenDomain(upstreamNamespace.Name) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + for _, obj := range []client.Object{domain, gatewayClass} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + var tlsCertificateCreates, tlsCertificateDeletes atomic.Int32 + var requestedNames []string + upstream := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass). + WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if cert, ok := obj.(*certificatesv1alpha1.TLSCertificate); ok { + tlsCertificateCreates.Add(1) + cert.UID = uuid.NewUUID() + for _, name := range cert.Spec.DNSNames { + requestedNames = append(requestedNames, string(name)) + } + } + return cl.Create(ctx, obj, opts...) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*certificatesv1alpha1.TLSCertificate); ok { + tlsCertificateDeletes.Add(1) + } + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + certificateWrites := map[string]int{} + countCertificate := func(obj client.Object, verb string) { + if _, ok := obj.(*cmv1.Certificate); ok { + certificateWrites[verb+" "+obj.GetName()]++ + } + } + downstream := fake.NewClientBuilder().WithScheme(testScheme). + WithStatusSubresource(&gatewayv1.Gateway{}, &cmv1.Certificate{}). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + countCertificate(obj, "create") + return cl.Create(ctx, obj, opts...) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + countCertificate(obj, "delete") + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + + issuedCrt, issuedKey := ca.issue(t, wildcard, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + service := fake.NewClientBuilder().WithScheme(testScheme).Build() + + h := &certificateServiceHarness{ + t: t, ctx: ctx, cfg: cfg, upstream: upstream, downstream: downstream, service: service, roots: ca.pool, + gateway: client.ObjectKeyFromObject(upstreamGateway), namespace: downstreamNamespaceName, + } + + exactCertificate := func() *cmv1.Certificate { + var cert cmv1.Certificate + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateName(gatewayName, exactListener)}, &cert)) + return &cert + } + + h.reconcile() + + assert.Equal(t, []string{wildcard}, requestedNames, "only the wildcard is handed to the certificate service") + assert.Equal(t, []string{exact}, exactCertificate().Spec.DNSNames, "the exact hostname gets its cert-manager Certificate exactly as before") + assert.True(t, apierrors.IsNotFound(downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateName(gatewayName, wildcardListenerName)}, &cmv1.Certificate{})), + "cert-manager never orders for the wildcard while the service is on") + exactName := listenerCertificateName(gatewayName, exactListener) + require.Equal(t, 1, certificateWrites["create "+exactName]) + + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, wildcardListenerName)}, &cert)) + cert.Status = certificatesv1alpha1.TLSCertificateStatus{ + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, + } + require.NoError(t, upstream.Status().Update(ctx, &cert)) + require.NoError(t, service.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: certificatesv1alpha1.StoredSecretName(cert.UID)}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, + })) + h.reconcile() + _, downstreamGateway, _ := h.reconcile() + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, wildcardListenerName), "the wildcard serves the service-issued certificate") + + var mirrored corev1.Secret + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName(gatewayName, wildcardListenerName)}, &mirrored)) + require.Equal(t, issuedCrt, mirrored.Data["tls.crt"]) + + h.cfg.Gateway.CertificateService.Enabled = false + h.reconcile() + assert.Zero(t, seriesInNamespace(t, certificateServiceListenerFailing, upstreamNamespace.Name), "nothing reports as failing once the service is off") + h.cfg.Gateway.CertificateService.Enabled = true + h.reconcile() + + assert.Equal(t, int32(1), tlsCertificateCreates.Load(), "turning the service off and on again requests nothing new") + assert.Zero(t, tlsCertificateDeletes.Load(), "turning the service off withdraws no request") + assert.Equal(t, []string{wildcard}, requestedNames, "an exact hostname is never handed to the service, whatever the flag did") + assert.Equal(t, 1, certificateWrites["create "+exactName], "the exact hostname's Certificate is never replaced") + assert.Zero(t, certificateWrites["delete "+exactName]) + assert.Equal(t, []string{exact}, exactCertificate().Spec.DNSNames) + require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, wildcardListenerName)}, &cert)) + assert.Equal(t, []certificatesv1alpha1.DNSName{wildcard}, cert.Spec.DNSNames) + + var stillMirrored corev1.Secret + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName(gatewayName, wildcardListenerName)}, &stillMirrored)) + assert.Equal(t, issuedCrt, stillMirrored.Data["tls.crt"], "the issued certificate is still the one served") +} + +func TestCertificateServiceOneFailingListenerDoesNotDelayOthers(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + const ( + gatewayName = "two-gw" + serviceNS = "certificates-system" + failing = "*.broken.example.com" + healthy = "*.shop.example.com" + ) + failingListener := gatewayv1.SectionName("https-hostname-0") + healthyListener := gatewayv1.SectionName("https-hostname-1") + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "two", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + ca := newTestCA(t) + now := time.Now() + + cfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + upstreamGateway := newGateway(cfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(failingListener, failing), tlsListener(healthyListener, healthy)) + }) + domain := dnsProvenDomain(upstreamNamespace.Name) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + for _, obj := range []client.Object{domain, gatewayClass} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + failingName := tlsCertificateName(gatewayName, failingListener) + upstream := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass). + WithStatusSubresource(upstreamGateway, &certificatesv1alpha1.TLSCertificate{}). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetName() == failingName { + return apierrors.NewServiceUnavailable("webhook unavailable") + } + obj.SetUID(uuid.NewUUID()) + return cl.Create(ctx, obj, opts...) + }, + }).Build() + downstream := fake.NewClientBuilder().WithScheme(testScheme).WithStatusSubresource(&gatewayv1.Gateway{}).Build() + issuedCrt, issuedKey := ca.issue(t, healthy, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + service := fake.NewClientBuilder().WithScheme(testScheme).Build() + + h := &certificateServiceHarness{ + t: t, ctx: ctx, cfg: cfg, upstream: upstream, downstream: downstream, service: service, roots: ca.pool, + gateway: client.ObjectKeyFromObject(upstreamGateway), namespace: downstreamNamespaceName, + } + + first, _, _ := h.reconcile() + assert.Contains(t, listenerCondition(t, first, failingListener, listenerConditionCertificateIssuanceBlocked).Message, "keep trying") + + var cert certificatesv1alpha1.TLSCertificate + require.NoError(t, upstream.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace.Name, Name: tlsCertificateName(gatewayName, healthyListener)}, &cert), "the healthy wildcard is requested in the same pass") + cert.Status = certificatesv1alpha1.TLSCertificateStatus{ + NotBefore: &metav1.Time{Time: now.Add(-time.Hour)}, + NotAfter: &metav1.Time{Time: now.Add(60 * 24 * time.Hour)}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued", LastTransitionTime: metav1.Now()}}, + } + require.NoError(t, upstream.Status().Update(ctx, &cert)) + require.NoError(t, service.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS, Name: certificatesv1alpha1.StoredSecretName(cert.UID)}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}, + })) + + second, _, result := h.reconcile() + _, _, cooling := h.reconciler.certificateServiceInBackoff(certificateServiceKey{gateway: upstreamGateway.UID, listener: failingListener}, time.Now()) + assert.True(t, cooling, "the failing listener is still backing off") + + var mirrored corev1.Secret + require.NoError(t, downstream.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName(gatewayName, healthyListener)}, &mirrored), "the healthy wildcard is mirrored while the other backs off") + assert.Equal(t, issuedCrt, mirrored.Data["tls.crt"]) + assert.Contains(t, listenerCondition(t, second, failingListener, listenerConditionCertificateIssuanceBlocked).Message, "keep trying", "the failing listener keeps its message") + assert.LessOrEqual(t, result.RequeueAfter, certificateServiceBackoffMax) +} + +func TestTLSCertificateName(t *testing.T) { + long := tlsCertificateName("a-gateway-with-a-deliberately-very-long-name-for-this-test", "https-hostname-with-a-long-name-0") + assert.LessOrEqual(t, len(long), 63) + assert.NotEqual(t, tlsCertificateName("a-b", "c"), tlsCertificateName("a", "b-c"), "gateway and listener never blur into each other") + dotted := tlsCertificateName("my.dotted.gateway.name.that.is.long.enough.to.be.cut", "https-0") + assert.Regexp(t, `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, dotted) + assert.LessOrEqual(t, len(dotted), 63) + assert.Equal(t, tlsCertificateName("gw", "https-0"), tlsCertificateName("gw", "https-0")) + assert.True(t, strings.HasPrefix(tlsCertificateName("gw", "https-0"), "gw-https-0-")) +} + +func TestTLSCertificateNameGolden(t *testing.T) { + for _, tt := range []struct{ gateway, listener, want string }{ + {"gw", "https-0", "gw-https-0-bb35a504d1"}, + {"a-b", "c", "a-b-c-4e84717d75"}, + {"a", "b-c", "a-b-c-b88f83c840"}, + {"my.dotted.gateway.name.that.is.long.enough.to.be.cut", "https-0", "my-dotted-gateway-name-that-is-long-enough-to-be-cut-c30f01e877"}, + {"a-gateway-with-a-deliberately-very-long-name-for-this-test", "https-hostname-with-a-long-name-0", "a-gateway-with-a-deliberately-very-long-name-for-thi-8c34270044"}, + } { + assert.Equal(t, tt.want, tlsCertificateName(tt.gateway, gatewayv1.SectionName(tt.listener))) + } +} + +func TestIsSingleLabelWildcard(t *testing.T) { + assert.True(t, isSingleLabelWildcard("*.example.com")) + assert.True(t, isSingleLabelWildcard("*.shop.example.com")) + assert.False(t, isSingleLabelWildcard("app.example.com")) + assert.False(t, isSingleLabelWildcard("*.*.example.com")) + assert.False(t, isSingleLabelWildcard("*.")) + assert.False(t, isSingleLabelWildcard("a.*.example.com")) +} + +func TestCertificateServiceRequeueBackoff(t *testing.T) { + r := &GatewayReconciler{} + key := certificateServiceKey{gateway: types.UID("gw"), listener: "https-0"} + other := certificateServiceKey{gateway: types.UID("gw"), listener: "https-1"} + now := time.Now() + trying := certificateServiceIssue{reason: certificateServiceReasonStepFailed, message: "trying"} + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(key, true, now, trying)) + assert.Equal(t, 10*time.Second, r.certificateServiceRequeue(key, true, now, trying)) + assert.Equal(t, 20*time.Second, r.certificateServiceRequeue(key, true, now, trying)) + for range 10 { + r.certificateServiceRequeue(key, true, now, trying) + } + assert.Equal(t, certificateServiceBackoffMax, r.certificateServiceRequeue(key, true, now, trying)) + + backoff, remaining, cooling := r.certificateServiceInBackoff(key, now.Add(time.Minute)) + assert.True(t, cooling, "an event inside the window repeats the message instead of the calls") + assert.Equal(t, trying, backoff.issue) + assert.Equal(t, certificateServiceBackoffMax-time.Minute, remaining) + _, _, cooling = r.certificateServiceInBackoff(key, now.Add(certificateServiceBackoffMax)) + assert.False(t, cooling) + _, _, cooling = r.certificateServiceInBackoff(other, now) + assert.False(t, cooling, "another listener on the same gateway is not held back") + + assert.Zero(t, r.certificateServiceRequeue(key, false, now, certificateServiceIssue{})) + assert.Equal(t, 5*time.Second, r.certificateServiceRequeue(key, true, now, trying), "a success resets the backoff") +} + +func TestValidateIssuedMaterialWildcard(t *testing.T) { + now := time.Now() + wildcardCrt, wildcardKey := generateTLSKeyPair(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + apexCrt, apexKey := generateTLSKeyPair(t, "example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + + assert.NoError(t, validateIssuedMaterial(wildcardCrt, wildcardKey, "*.example.com", now)) + assert.NoError(t, validateIssuedMaterial(wildcardCrt, wildcardKey, "app.example.com", now)) + assert.Error(t, validateIssuedMaterial(wildcardCrt, wildcardKey, "example.com", now), "a wildcard does not cover the apex") + assert.Error(t, validateIssuedMaterial(apexCrt, apexKey, "*.example.com", now), "an apex certificate does not cover a wildcard listener") +} + +func TestVerifyIssuedChain(t *testing.T) { + now := time.Now() + ca := newTestCA(t) + other := newTestCA(t) + crt, _ := ca.issue(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + selfSigned, _ := generateTLSKeyPair(t, "*.example.com", now.Add(-time.Hour), now.Add(24*time.Hour)) + + assert.NoError(t, verifyIssuedChain(crt, "*.example.com", ca.pool, now)) + assert.Error(t, verifyIssuedChain(crt, "*.example.com", other.pool, now), "a chain from another CA is not trusted") + assert.Error(t, verifyIssuedChain(selfSigned, "*.example.com", ca.pool, now), "a self-signed leaf is not trusted") + assert.Error(t, verifyIssuedChain(crt, "*.other.com", ca.pool, now)) + assert.Error(t, verifyIssuedChain(nil, "*.example.com", ca.pool, now)) +} + +func TestTLSCertificateFailure(t *testing.T) { + now := time.Now() + old := metav1.NewTime(now.Add(-2 * tlsCertificateIssueGrace)) + fresh := metav1.NewTime(now.Add(-time.Minute)) + for _, tt := range []struct { + name string + created metav1.Time + conditions []metav1.Condition + want string + }{ + {name: "ready", created: old, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue}}}, + {name: "rejected", created: fresh, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse}}, want: certificateServiceReasonRejected}, + {name: "renewal order failed while ready", created: old, conditions: []metav1.Condition{ + {Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue}, + {Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionFalse, Reason: "IssuanceFailed"}, + }, want: certificateServiceReasonIssuanceFailed}, + {name: "fresh request with no status yet", created: fresh}, + {name: "request the service never engaged", created: old, want: certificateServiceReasonNotReady}, + {name: "pending past the grace", created: old, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: old}}, want: certificateServiceReasonNotReady}, + {name: "pending within the grace", created: old, conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: fresh}}}, + {name: "waiting on the customer's records", created: old, conditions: []metav1.Condition{ + {Type: certificatesv1alpha1.ConditionDNSDelegationReady, Status: metav1.ConditionFalse, LastTransitionTime: old}, + {Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: old}, + }}, + } { + t.Run(tt.name, func(t *testing.T) { + cert := &certificatesv1alpha1.TLSCertificate{ObjectMeta: metav1.ObjectMeta{CreationTimestamp: tt.created}} + cert.Status.Conditions = tt.conditions + got, _ := tlsCertificateFailure(cert, now) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestRenewalOverdue(t *testing.T) { + now := time.Now() + leaf := func(age, left time.Duration) *x509.Certificate { + return &x509.Certificate{NotBefore: now.Add(-age), NotAfter: now.Add(left)} + } + assert.False(t, renewalOverdue(leaf(10*24*time.Hour, 80*24*time.Hour), now)) + assert.False(t, renewalOverdue(leaf(65*24*time.Hour, 25*24*time.Hour), now), "inside the window the service renews in") + assert.True(t, renewalOverdue(leaf(70*24*time.Hour, 20*24*time.Hour), now)) + assert.False(t, renewalOverdue(nil, now)) +} + +func TestCertificateServiceCRDAbsentDoesNotBlockGateway(t *testing.T) { + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + withoutCertificates := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(withoutCertificates)) + require.NoError(t, gatewayv1.Install(withoutCertificates)) + require.NoError(t, discoveryv1.AddToScheme(withoutCertificates)) + require.NoError(t, networkingv1alpha.AddToScheme(withoutCertificates)) + downstreamScheme := newCertificateServiceTestScheme(t) + + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: uuid.NewUUID()}} + const hostname = "*.shop.example.com" + testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, "test-gw", func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener("https-hostname-0", hostname)) + }) + domain := dnsProvenDomain(upstreamNamespace.Name) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + route := newHTTPRoute(upstreamNamespace.Name, "app", func(r *gatewayv1.HTTPRoute) { + r.Spec.ParentRefs = []gatewayv1.ParentReference{{Name: gatewayv1.ObjectName(upstreamGateway.Name)}} + r.Spec.Hostnames = []gatewayv1.Hostname{hostname} + }) + for _, obj := range []client.Object{domain, gatewayClass, route} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + + fakeUpstreamClient := fake.NewClientBuilder().WithScheme(withoutCertificates). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, route). + WithStatusSubresource(upstreamGateway, route).Build() + fakeDownstreamClient := fake.NewClientBuilder().WithScheme(downstreamScheme).WithStatusSubresource(&gatewayv1.Gateway{}).Build() + + reconciler := &GatewayReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + Config: testCfg, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + CertificateServiceReader: fake.NewClientBuilder().WithScheme(downstreamScheme).Build(), + } + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy("test", fakeUpstreamClient, fakeDownstreamClient) + + reconciler.prepareUpstreamGateway(upstreamGateway) + result, downstreamGateway := reconciler.ensureDownstreamGateway(ctx, "test", fakeUpstreamClient, upstreamGateway, downstreamStrategy) + require.NoError(t, result.Err, "a missing TLSCertificate API must not fail the gateway") + _, err := result.Complete(ctx) + require.NoError(t, err) + assert.Equal(t, certificateServiceBackoffBase, result.RequeueAfter) + + require.NotNil(t, downstreamGateway) + assert.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKeyFromObject(downstreamGateway), &gatewayv1.Gateway{}), "the downstream gateway is still programmed") + assert.NotNil(t, gatewayutil.GetListenerByName(downstreamGateway.Spec.Listeners, gatewayutil.DefaultHTTPListenerName)) + + var updated gatewayv1.Gateway + require.NoError(t, fakeUpstreamClient.Get(ctx, client.ObjectKeyFromObject(upstreamGateway), &updated)) + assert.NotEmpty(t, updated.Status.Listeners, "gateway status is still written") + assert.Contains(t, listenerCondition(t, &updated, "https-hostname-0", listenerConditionCertificateIssuanceBlocked).Message, "keep trying") + + var downstreamRoutes gatewayv1.HTTPRouteList + require.NoError(t, fakeDownstreamClient.List(ctx, &downstreamRoutes, client.InNamespace(downstreamGateway.Namespace))) + assert.NotEmpty(t, downstreamRoutes.Items, "downstream routes are still reconciled") +} + +func TestCertificateServiceRenewalBlockedClearsOnRecovery(t *testing.T) { + testScheme := newCertificateServiceTestScheme(t) + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "recovery", UID: uuid.NewUUID()}} + downstreamNamespaceName := "ns-" + string(upstreamNamespace.UID) + const hostname = "*.shop.example.com" + const gatewayName = "recovery-gw" + const listenerName = gatewayv1.SectionName("https-hostname-0") + const serviceNS = "certificates-system" + secretName := listenerCertificateSecretName(gatewayName, listenerName) + certName := tlsCertificateName(gatewayName, listenerName) + ca := newTestCA(t) + + testCfg := config.NetworkServicesOperator{Gateway: certificateServiceGatewayConfig()} + upstreamGateway := newGateway(testCfg, upstreamNamespace.Name, gatewayName, func(g *gatewayv1.Gateway) { + g.UID = uuid.NewUUID() + g.Spec.Listeners = append(g.Spec.Listeners, tlsListener(listenerName, hostname)) + }) + domain := dnsProvenDomain(upstreamNamespace.Name) + gatewayClass := &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: gatewayv1.GatewayClassSpec{ControllerName: "test"}} + + now := time.Now() + servingCrt, servingKey := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(60*24*time.Hour)) + issuedCrt, issuedKey := ca.issue(t, hostname, now.Add(-time.Hour), now.Add(90*24*time.Hour)) + + cert := &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: upstreamNamespace.Name, Name: certName}, + Spec: certificatesv1alpha1.TLSCertificateSpec{DNSNames: []certificatesv1alpha1.DNSName{hostname}, Issuance: certificatesv1alpha1.IssuanceModeDNS01}, + Status: certificatesv1alpha1.TLSCertificateStatus{ + NotAfter: &metav1.Time{Time: now.Add(90 * 24 * time.Hour)}, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Issued"}}, + }, + } + require.NoError(t, controllerutil.SetControllerReference(upstreamGateway, cert, testScheme)) + serving := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: secretName}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": servingCrt, "tls.key": servingKey}} + issued := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: serviceNS}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": issuedCrt, "tls.key": issuedKey}} + for _, obj := range []client.Object{domain, gatewayClass, cert, serving, issued} { + obj.SetUID(uuid.NewUUID()) + obj.SetCreationTimestamp(metav1.Now()) + } + issued.Name = certificatesv1alpha1.StoredSecretName(cert.UID) + + fakeUpstreamClient := fake.NewClientBuilder().WithScheme(testScheme). + WithObjects(upstreamGateway, upstreamNamespace, domain, gatewayClass, cert). + WithStatusSubresource(upstreamGateway, cert).Build() + fakeDownstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(serving).WithStatusSubresource(&gatewayv1.Gateway{}).Build() + + forbidden := true + serviceClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(issued).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if forbidden { + return apierrors.NewForbidden(corev1.Resource("secrets"), key.Name, nil) + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + + h := &certificateServiceHarness{ + t: t, ctx: ctx, cfg: testCfg, upstream: fakeUpstreamClient, downstream: fakeDownstreamClient, service: serviceClient, roots: ca.pool, + gateway: client.ObjectKeyFromObject(upstreamGateway), namespace: downstreamNamespaceName, + } + + before := counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed) + first, _, _ := h.reconcile() + assertListenerRenewalBlocked(t, first, "keep trying") + assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed)) + + forbidden = false + h.reconciler.certificateServiceFailures.Delete(certificateServiceKey{gateway: upstreamGateway.UID, listener: listenerName}) + second, _, _ := h.reconcile() + for _, ls := range second.Status.Listeners { + if ls.Name == listenerName { + assert.Nil(t, apimeta.FindStatusCondition(ls.Conditions, listenerConditionCertificateRenewalBlocked), "a recovered renewal no longer reads as blocked") + } + } + var mirror corev1.Secret + require.NoError(t, fakeDownstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespaceName, Name: secretName}, &mirror)) + assert.Equal(t, issuedCrt, mirror.Data["tls.crt"], "the issued certificate replaces the serving one once readable") + assert.Equal(t, before+1, counterValue(t, certificateServiceFailuresTotal, upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed), "a success adds nothing") + + removed := second.DeepCopy() + removed.Spec.Listeners = removed.Spec.Listeners[:len(removed.Spec.Listeners)-1] + h.reconciler.forgetRemovedListeners(removed) + var m dto.Metric + require.NoError(t, certificateServiceFailuresTotal.WithLabelValues(upstreamNamespace.Name, gatewayName, string(listenerName), certificateServiceReasonStepFailed).Write(&m)) + assert.Zero(t, m.GetCounter().GetValue(), "a removed listener's series is dropped") +} + +func TestCertificateServiceStateCountedPerTransition(t *testing.T) { + r := &GatewayReconciler{} + gw := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: "count", Name: "gw", UID: uuid.NewUUID()}} + rejected := func() float64 { + return counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRejected) + } + overdue := func() float64 { + return counterValue(t, certificateServiceFailuresTotal, "count", "gw", "https-0", certificateServiceReasonRenewalOverdue) + } + before, beforeOverdue := rejected(), overdue() + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + assert.Equal(t, before+1, rejected()) + r.recordCertificateServiceState(gw, "https-0", "") + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRejected) + assert.Equal(t, before+2, rejected(), "a new failure after recovery counts again") + r.recordCertificateServiceState(gw, "https-0", certificateServiceReasonRenewalOverdue) + assert.Equal(t, beforeOverdue+1, overdue(), "a different failure counts under its own reason") +} + +// stampCreated gives an object the fake client created a creation timestamp, +// which the real API server sets and the controller reads to tell a fresh +// object from an existing one. +func stampCreated(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) { + t.Helper() + if created := obj.GetCreationTimestamp(); !created.IsZero() { + return + } + obj.SetCreationTimestamp(metav1.Now()) + require.NoError(t, cl.Update(ctx, obj)) +} diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 176b67e5..3da7e0d4 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -5,11 +5,13 @@ package controller import ( "context" "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "slices" "strconv" "strings" + "sync" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -45,6 +47,7 @@ import ( mcsource "sigs.k8s.io/multicluster-runtime/pkg/source" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" "go.datum.net/network-services-operator/internal/config" downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" @@ -92,6 +95,20 @@ type GatewayReconciler struct { Config config.NetworkServicesOperator DownstreamCluster cluster.Cluster + + // CertificateServiceReader reads the certificate service's cluster, where + // each issued key pair is stored. Required when + // Config.Gateway.CertificateService.Enabled. + CertificateServiceReader client.Reader + + // CertificateServiceRoots are the roots an issued chain must verify + // against when Config.Gateway.CertificateService.VerifyChain is set. Nil + // means the system roots. + CertificateServiceRoots *x509.CertPool + + certificateServiceFailures sync.Map + certificateServiceStates sync.Map + certificateServiceListeners sync.Map } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -306,6 +323,7 @@ func (r *GatewayReconciler) ensureDownstreamGateway( // hard-fail (LastFailureTime). See #260. listenerCertHealth := r.evaluateListenerCertHealth( ctx, + upstreamClient, downstreamClient, downstreamGateway.Namespace, upstreamGateway, @@ -361,6 +379,37 @@ func (r *GatewayReconciler) ensureDownstreamGateway( } } + var certificateServiceRequeue time.Duration + if r.Config.Gateway.CertificateService.Enabled { + // The service is a dependency of issuance, not of routing: whatever it + // does, DNS, status and routes below still reconcile, and a listener + // that could not be served tells the customer why. + serviceResult, issues := r.ensureListenerTLSCertificates( + ctx, + upstreamClient, + upstreamGateway, + downstreamGateway, + downstreamStrategy, + claimedHostnames, + ) + for name, message := range issues { + status, gated := listenerCertHealth[name] + if !gated { + continue + } + if status.healthy { + status.renewalBlocked = message + } else { + status.message = message + status.issuanceBlocked = true + } + listenerCertHealth[name] = status + } + certificateServiceRequeue = serviceResult.RequeueAfter + } else { + clearCertificateServiceFailing(upstreamGateway) + } + certResult := r.ensureListenerCertificates( ctx, upstreamGateway, @@ -436,6 +485,9 @@ func (r *GatewayReconciler) ensureDownstreamGateway( break } } + if certificateServiceRequeue > 0 && (result.RequeueAfter == 0 || certificateServiceRequeue < result.RequeueAfter) { + result.RequeueAfter = certificateServiceRequeue + } addresses := make([]gatewayv1.GatewayStatusAddress, 0, len(targetDomainHostnames)) @@ -479,8 +531,20 @@ type listenerCertStatus struct { // Carried here so the expiry gauge can be labelled with the secret name // without recomputing it outside listenerCertHealth. secretName string + // renewalBlocked, when set on a healthy listener, says the certificate it + // serves cannot be replaced and why, so the customer hears about it before + // the expiry turns it into an outage. + renewalBlocked string + // issuanceBlocked, when set on an unhealthy listener, says the certificate + // step itself failed rather than issuance merely being underway. + issuanceBlocked bool } +const listenerConditionCertificateRenewalBlocked = "CertificateRenewalBlocked" +const listenerReasonRenewalFailing = "RenewalFailing" +const listenerConditionCertificateIssuanceBlocked = "CertificateIssuanceBlocked" +const listenerReasonIssuanceFailing = "IssuanceFailing" + // clearListenerCertMetrics removes every certificate-health gauge series for a // gateway. Used both before re-recording each reconcile and on gateway deletion // so a removed listener never leaves a series stuck at its last value. The gating @@ -499,6 +563,7 @@ func clearListenerCertMetrics(namespace, name string) { // is left out so it is never gated. func (r *GatewayReconciler) evaluateListenerCertHealth( ctx context.Context, + upstreamClient client.Client, downstreamClient client.Client, downstreamNamespace string, upstreamGateway *gatewayv1.Gateway, @@ -531,7 +596,12 @@ func (r *GatewayReconciler) evaluateListenerCertHealth( continue } - status := r.listenerCertHealth(ctx, downstreamClient, downstreamNamespace, upstreamGateway.Name, l.Name, hostname, now) + var status listenerCertStatus + if _, service := r.listenerUsesCertificateService(l, claimedHostnames); service { + status = r.listenerTLSCertificateHealth(ctx, upstreamClient, downstreamClient, downstreamNamespace, upstreamGateway, l.Name, hostname, now) + } else { + status = r.listenerCertHealth(ctx, downstreamClient, downstreamNamespace, upstreamGateway.Name, l.Name, hostname, now) + } health[l.Name] = status // Mark this listener as managed regardless of its health, so the @@ -637,9 +707,26 @@ func (r *GatewayReconciler) listenerCertHealth( } } - // Finally, load the stored certificate and key and confirm they match and - // are still valid. This catches a broken or mismatched certificate that - // would otherwise be served and break HTTPS for the listener. + if secretStatus := listenerSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, now); !secretStatus.healthy { + return secretStatus + } + + return listenerCertStatus{healthy: true, notAfter: cert.Status.NotAfter, secretName: secretName} +} + +// listenerSecretHealth loads the stored certificate and key and confirms they +// match and are still valid, catching a broken or mismatched certificate that +// would otherwise be served and break HTTPS for the listener. +func listenerSecretHealth( + ctx context.Context, + downstreamClient client.Client, + downstreamNamespace string, + secretName string, + hostname string, + now time.Time, +) listenerCertStatus { + logger := log.FromContext(ctx) + var secret corev1.Secret if err := downstreamClient.Get(ctx, client.ObjectKey{Namespace: downstreamNamespace, Name: secretName}, &secret); err != nil { if apierrors.IsNotFound(err) { @@ -675,7 +762,11 @@ func (r *GatewayReconciler) listenerCertHealth( } } - return listenerCertStatus{healthy: true, notAfter: cert.Status.NotAfter, secretName: secretName} + status := listenerCertStatus{healthy: true, secretName: secretName} + if leaf := keyPair.Leaf; leaf != nil { + status.notAfter = &metav1.Time{Time: leaf.NotAfter} + } + return status } // certIsReady reports whether a cert-manager Certificate has Ready=True. @@ -943,6 +1034,9 @@ func (r *GatewayReconciler) ensureListenerCertificates( if hasSharedSecret && (strings.HasSuffix(hostname, wildcardSuffix) || hostname == r.Config.Gateway.TargetDomain) { continue } + if _, service := r.listenerUsesCertificateService(l, claimedHostnames); service { + continue + } // Apply ClusterIssuerMap translation first — this is the admin-level // override (e.g. mapping the `auto` sentinel to a real ClusterIssuer @@ -1738,6 +1832,7 @@ func (r *GatewayReconciler) finalizeGateway( gatewayProgrammedTotal.DeleteLabelValues(upstreamGateway.Namespace, upstreamGateway.Name) // Clear this gateway's cert-health series now that it is gone. clearListenerCertMetrics(upstreamGateway.Namespace, upstreamGateway.Name) + r.forgetGateway(upstreamGateway) // Clean up DNS records created by this gateway if r.Config.Gateway.EnableDNSIntegration { @@ -2120,6 +2215,29 @@ func (r *GatewayReconciler) ensureDownstreamGatewayHTTPRoutes( apimeta.SetStatusCondition(&status.Conditions, programmedCondition) apimeta.SetStatusCondition(&status.Conditions, resolvedRefsCondition) + if certStatus, gated := listenerCertHealth[listener.Name]; gated && certStatus.healthy && certStatus.renewalBlocked != "" { + apimeta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: listenerConditionCertificateRenewalBlocked, + Status: metav1.ConditionTrue, + Reason: listenerReasonRenewalFailing, + Message: certStatus.renewalBlocked, + ObservedGeneration: upstreamGateway.Generation, + }) + } else { + apimeta.RemoveStatusCondition(&status.Conditions, listenerConditionCertificateRenewalBlocked) + } + if certStatus, gated := listenerCertHealth[listener.Name]; gated && !certStatus.healthy && certStatus.issuanceBlocked { + apimeta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: listenerConditionCertificateIssuanceBlocked, + Status: metav1.ConditionTrue, + Reason: listenerReasonIssuanceFailing, + Message: certStatus.message, + ObservedGeneration: upstreamGateway.Generation, + }) + } else { + apimeta.RemoveStatusCondition(&status.Conditions, listenerConditionCertificateIssuanceBlocked) + } + listenerStatus = append(listenerStatus, status) } @@ -2999,6 +3117,13 @@ func (r *GatewayReconciler) SetupWithManager(mgr mcmanager.Manager) error { ) } + if r.Config.Gateway.CertificateService.Enabled { + builder = builder.Watches( + &certificatesv1alpha1.TLSCertificate{}, + r.listGatewaysForTLSCertificateFunc, + ) + } + return builder. WithOptions(controller.TypedOptions[mcreconcile.Request]{ MaxConcurrentReconciles: r.Config.Gateway.MaxConcurrentReconciles, diff --git a/internal/controller/httpproxy_certificate_service.go b/internal/controller/httpproxy_certificate_service.go new file mode 100644 index 00000000..cfb3960c --- /dev/null +++ b/internal/controller/httpproxy_certificate_service.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" +) + +const certificateProvisioningMessage = "We're provisioning and applying a certificate to this hostname - it may take a few minutes" + +// tlsCertificateReadyCondition derives a wildcard hostname's CertificateReady +// condition from its TLSCertificate in the project control plane, the Secret +// serving it, and what the gateway reports about its listener. +func (r *HTTPProxyReconciler) tlsCertificateReadyCondition( + ctx context.Context, + upstreamClient client.Client, + downstreamClient client.Client, + downstreamNamespace string, + upstreamNamespace string, + certName string, + secretName string, + hostname string, + listenerConditions []metav1.Condition, + generation int64, +) metav1.Condition { + condition := metav1.Condition{ + Type: networkingv1alpha.HostnameConditionCertificateReady, + ObservedGeneration: generation, + } + + blocked := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateRenewalBlocked) + if blocked != nil && blocked.Status != metav1.ConditionTrue { + blocked = nil + } + if blocked == nil { + if issuance := apimeta.FindStatusCondition(listenerConditions, listenerConditionCertificateIssuanceBlocked); issuance != nil && issuance.Status == metav1.ConditionTrue { + blocked = issuance + } + } + + var cert certificatesv1alpha1.TLSCertificate + err := upstreamClient.Get(ctx, client.ObjectKey{Namespace: upstreamNamespace, Name: certName}, &cert) + switch { + case apierrors.IsNotFound(err): + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = certificateProvisioningMessage + if blocked != nil { + condition.Reason = networkingv1alpha.CertificateReadyReasonProvisioningFailed + condition.Message = blocked.Message + } + return condition + case err != nil: + condition.Status = metav1.ConditionUnknown + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = fmt.Sprintf("Failed to get certificate: %v", err) + return condition + } + + condition.Status, condition.Reason, condition.Message = tlsCertificateReadyState(&cert) + serving, _ := servingSecretHealth(ctx, downstreamClient, downstreamNamespace, secretName, hostname, time.Now()) + + switch { + case serving.healthy && (condition.Status != metav1.ConditionTrue || blocked != nil): + condition.Status = metav1.ConditionTrue + condition.Reason = networkingv1alpha.CertificateReadyReasonCertificateIssued + condition.Message = "Certificate is ready; a renewal is in progress" + if blocked != nil { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed: " + blocked.Message + } else if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + condition.Reason = networkingv1alpha.CertificateReadyReasonRenewalFailing + condition.Message = "Certificate is ready but cannot be renewed" + if accepted.Message != "" { + condition.Message += ": " + accepted.Message + } + } + case !serving.healthy && blocked != nil: + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonProvisioningFailed + condition.Message = blocked.Message + case !serving.healthy && condition.Status == metav1.ConditionTrue: + condition.Status = metav1.ConditionFalse + condition.Reason = networkingv1alpha.CertificateReadyReasonPending + condition.Message = "The certificate has been issued and is being applied to this hostname" + } + return condition +} + +// tlsCertificateReadyState maps TLSCertificate conditions onto the +// CertificateReady vocabulary: Ready wins, a rejected spec is a provisioning +// failure carrying the service's explanation, an in-flight order is a +// challenge in progress, and anything else is pending. +func tlsCertificateReadyState(cert *certificatesv1alpha1.TLSCertificate) (metav1.ConditionStatus, string, string) { + if apimeta.IsStatusConditionTrue(cert.Status.Conditions, certificatesv1alpha1.ConditionReady) { + return metav1.ConditionTrue, networkingv1alpha.CertificateReadyReasonCertificateIssued, "Certificate is ready" + } + + if accepted := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionAccepted); accepted != nil && accepted.Status == metav1.ConditionFalse { + message := accepted.Message + if message == "" { + message = "Certificate request was rejected" + } + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonProvisioningFailed, message + } + + if issuing := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionIssuing); issuing != nil && issuing.Status == metav1.ConditionTrue { + message := certificateProvisioningMessage + if issuing.Message != "" { + message = fmt.Sprintf("%s (%s)", certificateProvisioningMessage, issuing.Message) + } + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonChallengeInProgress, message + requiredDNSRecordsHint(cert) + } + + return metav1.ConditionFalse, networkingv1alpha.CertificateReadyReasonPending, certificateProvisioningMessage + requiredDNSRecordsHint(cert) +} + +// requiredDNSRecordsHint tells the customer which records issuance is waiting +// on, when the service has named any. +func requiredDNSRecordsHint(cert *certificatesv1alpha1.TLSCertificate) string { + records := make([]string, 0, len(cert.Status.RequiredDNSRecords)) + for _, record := range cert.Status.RequiredDNSRecords { + records = append(records, fmt.Sprintf("%s %s %s", record.Name, record.Type, record.Content)) + } + if len(records) == 0 { + return "" + } + return ". Publish these DNS records to continue: " + strings.Join(records, "; ") +} + +// enqueueHTTPProxyForTLSCertificate follows TLSCertificate -> Gateway -> +// HTTPProxy through controller references in the same project control plane. +func (r *HTTPProxyReconciler) enqueueHTTPProxyForTLSCertificate(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + gatewayRef := metav1.GetControllerOf(obj) + if gatewayRef == nil || gatewayRef.Kind != KindGateway { + return nil + } + + var gateway gatewayv1.Gateway + if err := cl.GetClient().Get(ctx, client.ObjectKey{Namespace: obj.GetNamespace(), Name: gatewayRef.Name}, &gateway); err != nil { + if !apierrors.IsNotFound(err) { + log.FromContext(ctx).Error(err, "failed to get Gateway owning TLSCertificate", "tlscertificate", obj.GetName()) + } + return nil + } + + proxyRef := metav1.GetControllerOf(&gateway) + if proxyRef == nil || proxyRef.Kind != KindHTTPProxy { + return nil + } + + return []mcreconcile.Request{{ + ClusterName: clusterName, + Request: ctrl.Request{NamespacedName: client.ObjectKey{Namespace: gateway.Namespace, Name: proxyRef.Name}}, + }} + }) +} diff --git a/internal/controller/httpproxy_certificate_service_test.go b/internal/controller/httpproxy_certificate_service_test.go new file mode 100644 index 00000000..d0efdec7 --- /dev/null +++ b/internal/controller/httpproxy_certificate_service_test.go @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + "go.datum.net/network-services-operator/internal/config" +) + +func TestBuildCertificateStatusesCertificateService(t *testing.T) { + t.Parallel() + + testScheme := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(testScheme)) + require.NoError(t, gatewayv1.Install(testScheme)) + require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + require.NoError(t, certificatesv1alpha1.AddToScheme(testScheme)) + + const wildcard = "*.shop.example.com" + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-ns", UID: types.UID("ns-uid")}} + downstreamNamespaceName := "ns-" + string(ns.UID) + + newGatewayFor := func(hostname string) *gatewayv1.Gateway { + return &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name}, + Spec: gatewayv1.GatewaySpec{Listeners: []gatewayv1.Listener{{ + Name: "https-hostname-0", + Protocol: gatewayv1.HTTPSProtocolType, + Hostname: ptr.To(gatewayv1.Hostname(hostname)), + }}}, + } + } + httpProxy := &networkingv1alpha.HTTPProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "my-proxy", Namespace: ns.Name, Generation: 3}, + Spec: networkingv1alpha.HTTPProxySpec{Hostnames: []gatewayv1.Hostname{wildcard, "app.example.com"}}, + } + certName := tlsCertificateName("my-proxy", "https-hostname-0") + + tlsCert := func(conditions ...metav1.Condition) *certificatesv1alpha1.TLSCertificate { + return &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: certName}, + Status: certificatesv1alpha1.TLSCertificateStatus{Conditions: conditions}, + } + } + legacyReady := func() *unstructured.Unstructured { + cert := newUnstructuredForGVK(certificateGVK) + cert.SetNamespace(downstreamNamespaceName) + cert.SetName(listenerCertificateName("my-proxy", "https-hostname-0")) + _ = unstructured.SetNestedSlice(cert.Object, []any{map[string]any{"type": "Ready", "status": "True"}}, "status", "conditions") + return cert + } + + crt, key := generateTLSKeyPair(t, wildcard, time.Now().Add(-time.Hour), time.Now().Add(30*24*time.Hour)) + servingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: downstreamNamespaceName, Name: listenerCertificateSecretName("my-proxy", "https-hostname-0")}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": crt, "tls.key": key}, + } + + tests := []struct { + name string + hostname string + upstream []client.Object + mutate func(*certificatesv1alpha1.TLSCertificate) + listenerConditions []metav1.Condition + downstream []client.Object + wantStatus metav1.ConditionStatus + wantReason string + wantMessage string + }{ + { + name: "Ready maps to CertificateIssued", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + downstream: []client.Object{servingSecret}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, + }, + { + name: "Accepted=False maps to ProvisioningFailed with the service's message", + upstream: []client.Object{tlsCert(metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain", Message: "names under datum.net are denied", + })}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, + wantMessage: "names under datum.net are denied", + }, + { + name: "Issuing maps to ChallengeInProgress", + upstream: []client.Object{tlsCert( + metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue}, + metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue, Reason: "OrderInFlight"}, + )}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonChallengeInProgress, + }, + { + name: "accepted but not yet issuing is Pending", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionTrue})}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonPending, + }, + { + name: "Issuing with required DNS records names them for the customer", + upstream: []client.Object{tlsCert( + metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue, Reason: "OrderInFlight"}, + ), nil}[:1], + mutate: func(c *certificatesv1alpha1.TLSCertificate) { + c.Status.RequiredDNSRecords = []certificatesv1alpha1.RequiredDNSRecord{{Name: "_acme-challenge.shop.example.com", Type: "CNAME", Content: "abc.acme-dns.example.net", Purpose: certificatesv1alpha1.DNSRecordPurposeCertificate}} + }, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonChallengeInProgress, + wantMessage: certificateProvisioningMessage + ". Publish these DNS records to continue: _acme-challenge.shop.example.com CNAME abc.acme-dns.example.net", + }, + { + name: "not Ready yet but a serving downstream Secret keeps the hostname ready", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionIssuing, Status: metav1.ConditionTrue})}, + downstream: []client.Object{servingSecret}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, + wantMessage: "Certificate is ready; a renewal is in progress", + }, + { + name: "Accepted=False with a serving downstream Secret reports RenewalFailing but stays ready", + upstream: []client.Object{tlsCert(metav1.Condition{ + Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain", Message: "names under datum.net are denied", + })}, + downstream: []client.Object{servingSecret}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonRenewalFailing, + wantMessage: "Certificate is ready but cannot be renewed: names under datum.net are denied", + }, + { + name: "a blocked renewal the gateway reports surfaces as RenewalFailing", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + downstream: []client.Object{servingSecret}, + listenerConditions: []metav1.Condition{{ + Type: listenerConditionCertificateRenewalBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonRenewalFailing, Message: "We couldn't request a TLS certificate for *.shop.example.com just now", + }}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonRenewalFailing, + wantMessage: "Certificate is ready but cannot be renewed: We couldn't request a TLS certificate for *.shop.example.com just now", + }, + { + name: "Ready but the issued Secret could not be taken and nothing serves is a provisioning failure", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + listenerConditions: []metav1.Condition{{ + Type: listenerConditionCertificateIssuanceBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonIssuanceFailing, Message: "We couldn't request a TLS certificate for *.shop.example.com just now", + }}, + downstream: nil, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, + wantMessage: "We couldn't request a TLS certificate for *.shop.example.com just now", + }, + { + name: "Ready but not yet mirrored is pending, not issued", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionReady, Status: metav1.ConditionTrue})}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonPending, + wantMessage: "The certificate has been issued and is being applied to this hostname", + }, + { + name: "an exact hostname is answered by its cert-manager Certificate, never a TLSCertificate", + hostname: "app.example.com", + upstream: []client.Object{tlsCert(metav1.Condition{Type: certificatesv1alpha1.ConditionAccepted, Status: metav1.ConditionFalse, Reason: "DeniedDomain"})}, + downstream: []client.Object{legacyReady()}, + wantStatus: metav1.ConditionTrue, + wantReason: networkingv1alpha.CertificateReadyReasonCertificateIssued, + }, + { + name: "no TLSCertificate because the request was refused is a provisioning failure", + listenerConditions: []metav1.Condition{{ + Type: listenerConditionCertificateIssuanceBlocked, Status: metav1.ConditionTrue, Reason: listenerReasonIssuanceFailing, Message: "A TLS certificate cannot be issued for *.shop.example.com: names under datum.net are denied", + }}, + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonProvisioningFailed, + wantMessage: "A TLS certificate cannot be issued for *.shop.example.com: names under datum.net are denied", + }, + { + name: "nothing requested yet is Pending", + wantStatus: metav1.ConditionFalse, + wantReason: networkingv1alpha.CertificateReadyReasonPending, + wantMessage: certificateProvisioningMessage, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if tt.mutate != nil { + tt.mutate(tt.upstream[0].(*certificatesv1alpha1.TLSCertificate)) + } + upstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(ns).WithObjects(tt.upstream...).Build() + downstreamClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(tt.downstream...).Build() + + r := &HTTPProxyReconciler{ + Config: config.NetworkServicesOperator{Gateway: config.GatewayConfig{ + TargetDomain: "example.net", + CertificateService: config.CertificateServiceConfig{Enabled: true}, + }}, + DownstreamCluster: &clusterWithClient{c: downstreamClient, scheme: testScheme}, + } + + hostname := tt.hostname + if hostname == "" { + hostname = wildcard + } + gw := newGatewayFor(hostname) + if tt.listenerConditions != nil { + gw.Status.Listeners = []gatewayv1.ListenerStatus{{Name: "https-hostname-0", Conditions: tt.listenerConditions}} + } + got := r.buildCertificateStatuses(context.Background(), upstreamClient, "local", gw, httpProxy) + require.Len(t, got, 1) + assert.Equal(t, hostname, got[0].Hostname) + cond := apimeta.FindStatusCondition(got[0].Conditions, networkingv1alpha.HostnameConditionCertificateReady) + require.NotNil(t, cond) + assert.Equal(t, tt.wantStatus, cond.Status) + assert.Equal(t, tt.wantReason, cond.Reason) + assert.Equal(t, int64(3), cond.ObservedGeneration) + if tt.wantMessage != "" { + assert.Equal(t, tt.wantMessage, cond.Message) + } + }) + } +} diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 2000fe6e..f95b0d4e 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -42,6 +42,7 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" "go.datum.net/network-services-operator/internal/config" downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" conditionutil "go.datum.net/network-services-operator/internal/util/condition" @@ -763,6 +764,13 @@ func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { builder = builder.WatchesRawSource(downstreamCertificateClusterSource) } + if r.Config.Gateway.CertificateService.Enabled { + builder = builder.Watches( + &certificatesv1alpha1.TLSCertificate{}, + r.enqueueHTTPProxyForTLSCertificate, + ) + } + return builder. WithOptions(controller.TypedOptions[mcreconcile.Request]{ MaxConcurrentReconciles: r.Config.HTTPProxy.MaxConcurrentReconciles, @@ -1507,6 +1515,13 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( } certName := resourcename.GetValidDNS1123Name(fmt.Sprintf("%s-%s", gateway.Name, l.Name)) + + if r.Config.Gateway.CertificateService.Enabled && isSingleLabelWildcard(hostname) { + apimeta.SetStatusCondition(&hs.Conditions, r.tlsCertificateReadyCondition(ctx, upstreamClient, downstreamClient, downstreamNamespaceName, gateway.Namespace, tlsCertificateName(gateway.Name, l.Name), listenerCertificateSecretName(gateway.Name, l.Name), hostname, listenerStatusConditions(gateway, l.Name), httpProxy.Generation)) + statuses = append(statuses, hs) + continue + } + certificate := newUnstructuredForGVK(certificateGVK) certKey := client.ObjectKey{Namespace: downstreamNamespaceName, Name: certName} @@ -1569,6 +1584,17 @@ func (r *HTTPProxyReconciler) buildCertificateStatuses( return statuses } +// listenerStatusConditions returns the conditions the gateway reports for one +// of its listeners, or nil. +func listenerStatusConditions(gateway *gatewayv1.Gateway, listener gatewayv1.SectionName) []metav1.Condition { + for _, ls := range gateway.Status.Listeners { + if ls.Name == listener { + return ls.Conditions + } + } + return nil +} + // getCertificateReadyConditionReason returns the reason and message for the // CertificateReady condition based on the cert-manager Certificate's Ready condition. func getCertificateReadyConditionReason(certificate *unstructured.Unstructured) (string, string) { diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go index e15e07d6..84438d79 100644 --- a/internal/controller/metrics.go +++ b/internal/controller/metrics.go @@ -75,6 +75,31 @@ var ( []string{jsonKeyNamespace, jsonKeyName}, ) + // certificateServiceFailuresTotal counts every certificate-service step that + // could not do its work for a wildcard listener, and every transition of its + // TLSCertificate into a failing state, by reason, whether or not the listener + // still serves a certificate. A hostname can serve for weeks on its previous + // certificate while every attempt to replace it fails; this is the signal + // that says so before the expiry does. + certificateServiceFailuresTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nso_certificate_service_failures_total", + Help: "Total certificate-service failures for a Gateway listener, by reason.", + }, + []string{jsonKeyNamespace, jsonKeyName, metricLabelListener, metricLabelReason}, + ) + + // certificateServiceListenerFailing is 1 for each wildcard listener whose + // certificate cannot currently be issued or renewed, labelled with why. The + // series goes when the listener recovers, leaves the service, or is removed. + certificateServiceListenerFailing = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "nso_certificate_service_listener_failing", + Help: "1 while a Gateway listener's certificate-service issuance or renewal is failing, by reason.", + }, + []string{jsonKeyNamespace, jsonKeyName, metricLabelListener, metricLabelReason}, + ) + // gatewayListenerCertWithheld is 1 for each upstream Gateway listener that NSO // is currently withholding from the downstream because its TLS certificate is // unusable. The series for a listener is removed when the listener recovers, diff --git a/test/prometheus-rules/gateways/certificate-service-rules.yaml b/test/prometheus-rules/gateways/certificate-service-rules.yaml new file mode 100644 index 00000000..88eb890a --- /dev/null +++ b/test/prometheus-rules/gateways/certificate-service-rules.yaml @@ -0,0 +1,21 @@ +groups: +- name: nso-certificate-service + interval: 30s + rules: + # Fires when a wildcard hostname's certificate cannot be issued or renewed + # through the certificate service. The hostname may still serve its previous + # certificate, so this fires weeks before that certificate expires. + - alert: CertificateServiceIssuanceFailing + expr: | + max by (namespace, name, listener, reason) (nso_certificate_service_listener_failing) > 0 + or + sum by (namespace, name, listener, reason) (increase(nso_certificate_service_failures_total[1h])) > 0 + for: 2h + labels: + severity: warning + service: network-services + team: sre + annotations: + summary: "Certificate for Gateway listener {{ $labels.namespace }}/{{ $labels.name }}/{{ $labels.listener }} cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener {{ $labels.listener }} on Gateway {{ $labels.name }} in namespace {{ $labels.namespace }} for over two hours (reason: {{ $labels.reason }}). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" diff --git a/test/prometheus-rules/gateways/certificate-service-tests.yaml b/test/prometheus-rules/gateways/certificate-service-tests.yaml new file mode 100644 index 00000000..64d4f69b --- /dev/null +++ b/test/prometheus-rules/gateways/certificate-service-tests.yaml @@ -0,0 +1,74 @@ +rule_files: + - certificate-service-rules.yaml + +evaluation_interval: 30s + +tests: + # A renewal that keeps failing raises the alert after two hours, even though + # the failure was counted only once when it began. + - interval: 1m + input_series: + - series: 'nso_certificate_service_listener_failing{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="IssuanceFailed"}' + values: '1+0x180' + - series: 'nso_certificate_service_failures_total{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="IssuanceFailed"}' + values: '1+0x180' + alert_rule_test: + - eval_time: 150m + alertname: CertificateServiceIssuanceFailing + exp_alerts: + - exp_labels: + severity: warning + service: network-services + team: sre + namespace: customer-ns + name: my-gateway + listener: https-hostname-0 + reason: IssuanceFailed + exp_annotations: + summary: "Certificate for Gateway listener customer-ns/my-gateway/https-hostname-0 cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener https-hostname-0 on Gateway my-gateway in namespace customer-ns for over two hours (reason: IssuanceFailed). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" + + # A step that keeps failing and retrying raises the alert from the counter alone. + - interval: 1m + input_series: + - series: 'nso_certificate_service_failures_total{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="StepFailed"}' + values: '0+1x180' + alert_rule_test: + - eval_time: 150m + alertname: CertificateServiceIssuanceFailing + exp_alerts: + - exp_labels: + severity: warning + service: network-services + team: sre + namespace: customer-ns + name: my-gateway + listener: https-hostname-0 + reason: StepFailed + exp_annotations: + summary: "Certificate for Gateway listener customer-ns/my-gateway/https-hostname-0 cannot be issued or renewed" + description: "The certificate service has been failing to issue or renew the certificate for listener https-hostname-0 on Gateway my-gateway in namespace customer-ns for over two hours (reason: StepFailed). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing" + + # A failure that cleared within the hour does not alert. + - interval: 1m + input_series: + - series: 'nso_certificate_service_listener_failing{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="NotReady"}' + values: '1+0x30 _x150' + - series: 'nso_certificate_service_failures_total{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="NotReady"}' + values: '1+0x180' + alert_rule_test: + - eval_time: 150m + alertname: CertificateServiceIssuanceFailing + exp_alerts: [] + + # A failure younger than two hours does not alert yet. + - interval: 1m + input_series: + - series: 'nso_certificate_service_listener_failing{namespace="customer-ns", name="my-gateway", listener="https-hostname-0", reason="Rejected"}' + values: '1+0x90' + alert_rule_test: + - eval_time: 90m + alertname: CertificateServiceIssuanceFailing + exp_alerts: []