feat: Issue wildcard certificates through the certificate service

api/v1alpha/httpproxy_types.go

@@ -551,6 +551,10 @@ const (
 
 	// CertificateReadyReasonChallengeInProgress indicates an ACME challenge is in progress.
 	CertificateReadyReasonChallengeInProgress = "ChallengeInProgress"
+
+	// CertificateReadyReasonRenewalFailing indicates the hostname still serves a
+	// valid certificate but its replacement cannot be issued; the message says why.
+	CertificateReadyReasonRenewalFailing = "RenewalFailing"
 )
 
 // Reasons for HostnameConditionAvailable.

config/rbac/role.yaml

@@ -61,6 +61,24 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - certificates.miloapis.com
+  resources:
+  - tlscertificates
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - certificates.miloapis.com
+  resources:
+  - tlscertificates/status
+  verbs:
+  - get
 - apiGroups:
   - coordination.k8s.io
   resources:

config/telemetry/alerts/gateways.yaml

@@ -173,3 +173,24 @@ spec:
         summary: "{{ $value }} edge listener(s) have every TLS certificate broken and cannot be protected"
         description: "The extension server left {{ $value }} edge listener(s) untouched because every certificate on them is broken. It never removes a listener entirely, so the edge will reject the configuration update for those listeners (EnvoyListenerUpdateRejected confirms it). This means the controller did not withhold the listener before it reached the edge. Check extension server logs for 'listeners_left_intact' and why the controller did not withhold the listener."
         runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#tlsbackstoplistenerallcertsbroken"
+
+  - name: nso-certificate-service
+    interval: 30s
+    rules:
+    # Fires when a wildcard hostname's certificate cannot be issued or renewed
+    # through the certificate service. The hostname may still serve its previous
+    # certificate, so this fires weeks before that certificate expires.
+    - alert: CertificateServiceIssuanceFailing
+      expr: |
+        max by (namespace, name, listener, reason) (nso_certificate_service_listener_failing) > 0
+        or
+        sum by (namespace, name, listener, reason) (increase(nso_certificate_service_failures_total[1h])) > 0
+      for: 2h
+      labels:
+        severity: warning
+        service: network-services
+        team: sre
+      annotations:
+        summary: "Certificate for Gateway listener {{ $labels.namespace }}/{{ $labels.name }}/{{ $labels.listener }} cannot be issued or renewed"
+        description: "The certificate service has been failing to issue or renew the certificate for listener {{ $labels.listener }} on Gateway {{ $labels.name }} in namespace {{ $labels.namespace }} for over two hours (reason: {{ $labels.reason }}). The listener's CertificateRenewalBlocked or CertificateIssuanceBlocked condition says why."
+        runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/gateway-tls-certificates.md#certificateserviceissuancefailing"

docs/runbooks/gateway-tls-certificates.md

@@ -87,6 +87,47 @@ renewal depends on it.
 customer-driven gating event — no platform fix. If renewal is failing for a
 platform reason, fix the issuer / ACME path so cert-manager can renew.
 
+## CertificateServiceIssuanceFailing
+
+**Meaning.** A wildcard hostname's certificate has not been issued or renewed by
+the certificate service for over two hours. Only wildcard hostnames use the
+service; exact hostnames stay on cert-manager and are covered by the alerts
+above.
+
+**Impact.** If the listener still serves a certificate, none yet: it carries
+`CertificateRenewalBlocked` and keeps serving until that certificate expires. If
+it serves nothing, it carries `CertificateIssuanceBlocked` and the wildcard is
+unavailable.
+
+**Diagnose.** The `reason` label says where it failed:
+
+| Reason | Where |
+|---|---|
+| `Rejected`, `Refused` | The service refused the request; the condition message carries its reason |
+| `IssuanceFailed`, `NotReady` | The service accepted it but the ACME order failed or never completed |
+| `RenewalOverdue` | The served certificate is past its renewal point and nothing newer arrived |
+| `MaterialRefused`, `UntrustedChain` | The operator refused what the service issued |
+| `StepFailed`, `NotOwned` | The operator could not reach the service, or the request name is taken |
+
+Read the TLSCertificate in the project, named after the gateway and listener:
+
+```sh
+kubectl -n <namespace> get tlscertificates -o yaml
+```
+
+The issued key pair never enters the project. It is on the service cluster, in
+the `secretNamespace` the operator is configured with, in a Secret named `tc-`
+plus the first 32 hex characters of the SHA-256 of the TLSCertificate's UID:
+
+```sh
+uid=$(kubectl -n <namespace> get tlscertificate <name> -o jsonpath='{.metadata.uid}')
+kubectl -n certificates-system get secret "tc-$(printf %s "$uid" | sha256sum | cut -c1-32)"
+```
+
+**Remediate.** A refusal or a missing DNS delegation record is for the customer.
+An ACME failure, a refused or untrusted chain, or a step failure is a platform
+fault in the certificate service or the operator's access to it.
+
 ## TLSBackstopPruningChains
 
 **Meaning.** The extension server is actively dropping broken certificates from

internal/agent/catalog.go

@@ -540,6 +540,18 @@ var dnsAndCertCatalog = []ReasonInfo{
 			"status message.",
 		Skill: SkillCertificateNotIssued,
 	},
+	{
+		Reason:        networkingv1alpha.CertificateReadyReasonRenewalFailing,
+		ConditionType: networkingv1alpha.HostnameConditionCertificateReady,
+		Actionability: ActionabilityUser,
+		Scope:         ScopeOneHostname,
+		Explanation: "HTTPS still works on this hostname's current certificate, but the replacement " +
+			"cannot be issued. When the current certificate expires HTTPS will fail.",
+		Remediation: "The status message says what blocked the renewal. If it names the hostname " +
+			"itself, check that it still resolves publicly to this load balancer; otherwise " +
+			"it is a platform issue and support can see the same message.",
+		Skill: SkillCertificateNotIssued,
+	},
 }
 
 // domainCatalog covers the Domain behind a custom hostname. This is where

internal/certificates/v1alpha1/groupversion_info.go

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// Package v1alpha1 carries the certificates.miloapis.com/v1alpha1 types the
+// operator consumes from the Milo certificate service. It mirrors the
+// service's API until its module is importable.
+//
+// +kubebuilder:object:generate=true
+// +kubebuilder:skip
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	GroupVersion = schema.GroupVersion{Group: "certificates.miloapis.com", Version: "v1alpha1"}
+
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(GroupVersion, &TLSCertificate{}, &TLSCertificateList{})
+	metav1.AddToGroupVersion(scheme, GroupVersion)
+	return nil
+}

internal/certificates/v1alpha1/stored_secret.go

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package v1alpha1
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// StoredSecretName returns the name of the kubernetes.io/tls Secret holding the
+// issued key pair for the TLSCertificate with the given UID. The Secret lives in
+// the certificate service's namespace on the service cluster, and its name
+// depends on nothing but the UID.
+func StoredSecretName(uid types.UID) string {
+	sum := sha256.Sum256([]byte(uid))
+	return "tc-" + hex.EncodeToString(sum[:16])
+}

internal/certificates/v1alpha1/stored_secret_test.go

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package v1alpha1
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestStoredSecretName(t *testing.T) {
+	uid := types.UID("6f1c2a4e-0b7d-4c1e-9a43-2d5f8e7b1c90")
+	got := StoredSecretName(uid)
+	if got != "tc-15441e1abfba82916b1ba950ace11517" {
+		t.Fatalf("unexpected name %q", got)
+	}
+	if again := StoredSecretName(uid); again != got {
+		t.Fatalf("name is not deterministic: %q then %q", got, again)
+	}
+	if other := StoredSecretName("7a2d3b5f-1c8e-4d2f-8b54-3e6a9f8c2d01"); other == got {
+		t.Fatalf("different UIDs share the name %q", got)
+	}
+}

internal/certificates/v1alpha1/tlscertificate_types.go

@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// IssuanceMode selects how the certificate authority validates control of the
+// requested names.
+type IssuanceMode string
+
+const (
+	IssuanceModeAuto   IssuanceMode = "Auto"
+	IssuanceModeHTTP01 IssuanceMode = "HTTP01"
+	IssuanceModeDNS01  IssuanceMode = "DNS01"
+)
+
+// ChallengeType is a resolved issuance mode: the ACME challenge type used to
+// validate the names.
+type ChallengeType string
+
+const (
+	ChallengeTypeHTTP01 ChallengeType = "HTTP01"
+	ChallengeTypeDNS01  ChallengeType = "DNS01"
+)
+
+// DNSRecordPurpose explains why a DNS record must be published.
+type DNSRecordPurpose string
+
+const (
+	DNSRecordPurposeRouting     DNSRecordPurpose = "Routing"
+	DNSRecordPurposeCertificate DNSRecordPurpose = "Certificate"
+)
+
+// ChallengeState is the lifecycle state of an ACME challenge.
+type ChallengeState string
+
+const (
+	ChallengeStatePending ChallengeState = "Pending"
+	ChallengeStateValid   ChallengeState = "Valid"
+	ChallengeStateInvalid ChallengeState = "Invalid"
+)
+
+// Condition types reported on a TLSCertificate.
+const (
+	ConditionAccepted           = "Accepted"
+	ConditionDNSDelegationReady = "DNSDelegationReady"
+	ConditionIssuing            = "Issuing"
+	ConditionReady              = "Ready"
+)
+
+// TLSCertificateSpec defines the desired state of TLSCertificate. The service
+// does not verify that the project controls the names; the caller creates a
+// TLSCertificate only for names whose ownership it has already verified.
+type TLSCertificateSpec struct {
+	DNSNames []DNSName    `json:"dnsNames"`
+	Issuance IssuanceMode `json:"issuance,omitempty"`
+}
+
+// DNSName is a lowercase RFC 1123 hostname, optionally prefixed with "*.".
+type DNSName string
+
+// RequiredDNSRecord is a DNS record the name's owner must publish before
+// issuance can complete.
+type RequiredDNSRecord struct {
+	Name    string           `json:"name"`
+	Type    string           `json:"type"`
+	Content string           `json:"content"`
+	Purpose DNSRecordPurpose `json:"purpose"`
+}
+
+// ACMEChallenge is a live ACME challenge for one name. For HTTP01, the
+// consumer serves GET /.well-known/acme-challenge/<token> with the body <key>
+// on dnsName.
+type ACMEChallenge struct {
+	DNSName string         `json:"dnsName"`
+	Type    ChallengeType  `json:"type"`
+	Token   string         `json:"token"`
+	Key     string         `json:"key"`
+	State   ChallengeState `json:"state"`
+}
+
+// TLSCertificateStatus defines the observed state of TLSCertificate.
+type TLSCertificateStatus struct {
+	Issuance           ChallengeType       `json:"issuance,omitempty"`
+	DelegationTarget   string              `json:"delegationTarget,omitempty"`
+	NotBefore          *metav1.Time        `json:"notBefore,omitempty"`
+	NotAfter           *metav1.Time        `json:"notAfter,omitempty"`
+	RenewalTime        *metav1.Time        `json:"renewalTime,omitempty"`
+	RequiredDNSRecords []RequiredDNSRecord `json:"requiredDNSRecords,omitempty"`
+	Challenges         []ACMEChallenge     `json:"challenges,omitempty"`
+	Conditions         []metav1.Condition  `json:"conditions,omitempty"`
+	ObservedGeneration int64               `json:"observedGeneration,omitempty"`
+}
+
+// TLSCertificate requests a publicly trusted TLS certificate for a set of
+// hostnames. The issued key pair stays on the service cluster and is never
+// written to the project. Its name is at most 63 characters.
+//
+// +kubebuilder:object:root=true
+type TLSCertificate struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   TLSCertificateSpec   `json:"spec"`
+	Status TLSCertificateStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+type TLSCertificateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []TLSCertificate `json:"items"`
+}