From 6fdaf9c80040341ca2932342ade81e6a4dc634d3 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Tue, 27 Oct 2026 22:31:43 +0600 Subject: [PATCH] Add NATS guides and examples Signed-off-by: Tamal Saha --- docs/examples/nats/auth/nats-config-auth.yaml | 11 + .../nats/auth/nats-external-operator.yaml | 14 ++ .../nats/auth/nats-operator-auth.yaml | 19 ++ .../nats/backup/backupconfiguration.yaml | 39 +++ docs/examples/nats/backup/backupstorage.yaml | 17 ++ docs/examples/nats/backup/restoresession.yaml | 22 ++ docs/examples/nats/cluster/core-nats.yaml | 11 + docs/examples/nats/cluster/custom-config.yaml | 14 ++ docs/examples/nats/cluster/nats-cluster.yaml | 40 ++++ .../nats/cluster/nats-custom-config.yaml | 11 + .../nats/monitoring/builtin-prom-nats.yaml | 11 + .../nats/monitoring/prom-operator-nats.yaml | 16 ++ .../nats/multi-dc/nats-distributed.yaml | 29 +++ .../nats/multi-dc/nats-gateway-a.yaml | 15 ++ .../nats/multi-dc/nats-hub-leaf-accept.yaml | 15 ++ docs/examples/nats/multi-dc/nats-leaf.yaml | 16 ++ .../nats/multi-dc/placement-policy.yaml | 19 ++ .../opsrequests/horizontal-scaling-dc.yaml | 17 ++ .../nats/opsrequests/horizontal-scaling.yaml | 13 + .../nats/opsrequests/reconfigure-tls-add.yaml | 21 ++ .../opsrequests/reconfigure-tls-rotate.yaml | 13 + .../nats/opsrequests/reconfigure.yaml | 15 ++ docs/examples/nats/opsrequests/restart.yaml | 11 + .../nats/opsrequests/rotate-auth.yaml | 11 + .../nats/opsrequests/update-version.yaml | 13 + .../nats/opsrequests/vertical-scaling.yaml | 20 ++ .../nats/opsrequests/volume-expansion.yaml | 14 ++ docs/examples/nats/quickstart/quickstart.yaml | 17 ++ docs/examples/nats/tls/issuer.yaml | 8 + docs/examples/nats/tls/nats-tls.yaml | 23 ++ docs/guides/nats/_index.md | 10 + docs/guides/nats/auth-modes.md | 131 +++++++++++ docs/guides/nats/backup.md | 139 +++++++++++ docs/guides/nats/clustering.md | 152 ++++++++++++ docs/guides/nats/monitoring.md | 88 +++++++ docs/guides/nats/multi-dc.md | 222 ++++++++++++++++++ docs/guides/nats/ops-requests.md | 189 +++++++++++++++ docs/guides/nats/overview.md | 64 +++++ docs/guides/nats/quickstart.md | 122 ++++++++++ docs/guides/nats/tls.md | 105 +++++++++ 40 files changed, 1737 insertions(+) create mode 100644 docs/examples/nats/auth/nats-config-auth.yaml create mode 100644 docs/examples/nats/auth/nats-external-operator.yaml create mode 100644 docs/examples/nats/auth/nats-operator-auth.yaml create mode 100644 docs/examples/nats/backup/backupconfiguration.yaml create mode 100644 docs/examples/nats/backup/backupstorage.yaml create mode 100644 docs/examples/nats/backup/restoresession.yaml create mode 100644 docs/examples/nats/cluster/core-nats.yaml create mode 100644 docs/examples/nats/cluster/custom-config.yaml create mode 100644 docs/examples/nats/cluster/nats-cluster.yaml create mode 100644 docs/examples/nats/cluster/nats-custom-config.yaml create mode 100644 docs/examples/nats/monitoring/builtin-prom-nats.yaml create mode 100644 docs/examples/nats/monitoring/prom-operator-nats.yaml create mode 100644 docs/examples/nats/multi-dc/nats-distributed.yaml create mode 100644 docs/examples/nats/multi-dc/nats-gateway-a.yaml create mode 100644 docs/examples/nats/multi-dc/nats-hub-leaf-accept.yaml create mode 100644 docs/examples/nats/multi-dc/nats-leaf.yaml create mode 100644 docs/examples/nats/multi-dc/placement-policy.yaml create mode 100644 docs/examples/nats/opsrequests/horizontal-scaling-dc.yaml create mode 100644 docs/examples/nats/opsrequests/horizontal-scaling.yaml create mode 100644 docs/examples/nats/opsrequests/reconfigure-tls-add.yaml create mode 100644 docs/examples/nats/opsrequests/reconfigure-tls-rotate.yaml create mode 100644 docs/examples/nats/opsrequests/reconfigure.yaml create mode 100644 docs/examples/nats/opsrequests/restart.yaml create mode 100644 docs/examples/nats/opsrequests/rotate-auth.yaml create mode 100644 docs/examples/nats/opsrequests/update-version.yaml create mode 100644 docs/examples/nats/opsrequests/vertical-scaling.yaml create mode 100644 docs/examples/nats/opsrequests/volume-expansion.yaml create mode 100644 docs/examples/nats/quickstart/quickstart.yaml create mode 100644 docs/examples/nats/tls/issuer.yaml create mode 100644 docs/examples/nats/tls/nats-tls.yaml create mode 100644 docs/guides/nats/_index.md create mode 100644 docs/guides/nats/auth-modes.md create mode 100644 docs/guides/nats/backup.md create mode 100644 docs/guides/nats/clustering.md create mode 100644 docs/guides/nats/monitoring.md create mode 100644 docs/guides/nats/multi-dc.md create mode 100644 docs/guides/nats/ops-requests.md create mode 100644 docs/guides/nats/overview.md create mode 100644 docs/guides/nats/quickstart.md create mode 100644 docs/guides/nats/tls.md diff --git a/docs/examples/nats/auth/nats-config-auth.yaml b/docs/examples/nats/auth/nats-config-auth.yaml new file mode 100644 index 000000000..a05d319a7 --- /dev/null +++ b/docs/examples/nats/auth/nats-config-auth.yaml @@ -0,0 +1,11 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-config-auth + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + auth: + mode: Config + deletionPolicy: WipeOut diff --git a/docs/examples/nats/auth/nats-external-operator.yaml b/docs/examples/nats/auth/nats-external-operator.yaml new file mode 100644 index 000000000..18fad8e3a --- /dev/null +++ b/docs/examples/nats/auth/nats-external-operator.yaml @@ -0,0 +1,14 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-external-operator + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + auth: + mode: Operator + operator: + externalSecret: + name: my-nats-operator + deletionPolicy: WipeOut diff --git a/docs/examples/nats/auth/nats-operator-auth.yaml b/docs/examples/nats/auth/nats-operator-auth.yaml new file mode 100644 index 000000000..cc4e742ae --- /dev/null +++ b/docs/examples/nats/auth/nats-operator-auth.yaml @@ -0,0 +1,19 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-operator-auth + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + auth: + mode: Operator + operator: + resolverStorage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + allowDelete: false + deletionPolicy: WipeOut diff --git a/docs/examples/nats/backup/backupconfiguration.yaml b/docs/examples/nats/backup/backupconfiguration.yaml new file mode 100644 index 000000000..fe1c42392 --- /dev/null +++ b/docs/examples/nats/backup/backupconfiguration.yaml @@ -0,0 +1,39 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: nats-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: NATS + namespace: demo + name: nats-cluster + backends: + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/30 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: s3-nats-repo + backend: s3-backend + directory: /nats + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: nats-addon + tasks: + - name: logical-backup + params: + includeConsumers: "true" + - name: manifest-backup diff --git a/docs/examples/nats/backup/backupstorage.yaml b/docs/examples/nats/backup/backupstorage.yaml new file mode 100644 index 000000000..cb231d512 --- /dev/null +++ b/docs/examples/nats/backup/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + endpoint: http://minio.storage.svc:9000 + bucket: nats-backup + region: us-east-1 + secretName: s3-secret + usagePolicy: + allowedNamespaces: + from: All + deletionPolicy: WipeOut diff --git a/docs/examples/nats/backup/restoresession.yaml b/docs/examples/nats/backup/restoresession.yaml new file mode 100644 index 000000000..9c4f31548 --- /dev/null +++ b/docs/examples/nats/backup/restoresession.yaml @@ -0,0 +1,22 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: nats-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: NATS + namespace: demo + name: nats-restored + dataSource: + repository: s3-nats-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: nats-addon + tasks: + - name: manifest-restore + - name: logical-backup-restore diff --git a/docs/examples/nats/cluster/core-nats.yaml b/docs/examples/nats/cluster/core-nats.yaml new file mode 100644 index 000000000..a4a7b7bf4 --- /dev/null +++ b/docs/examples/nats/cluster/core-nats.yaml @@ -0,0 +1,11 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-core + namespace: demo +spec: + version: "2.15.1" + replicas: 2 + jetStream: + enabled: false + deletionPolicy: WipeOut diff --git a/docs/examples/nats/cluster/custom-config.yaml b/docs/examples/nats/cluster/custom-config.yaml new file mode 100644 index 000000000..9c786be47 --- /dev/null +++ b/docs/examples/nats/cluster/custom-config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nats-custom-config + namespace: demo +stringData: + nats.conf: | + ping_interval: "30s" + max_connections: 5000 + accounts.conf: | + ORDERS { + jetstream: enabled + users: [{user: orders, password: orders-secret}] + } diff --git a/docs/examples/nats/cluster/nats-cluster.yaml b/docs/examples/nats/cluster/nats-cluster.yaml new file mode 100644 index 000000000..8ab6e1865 --- /dev/null +++ b/docs/examples/nats/cluster/nats-cluster.yaml @@ -0,0 +1,40 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-cluster + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + jetStream: + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: standard + memoryStoreSize: 256Mi + serverConfig: + maxConnections: 10000 + maxPayload: 1048576 + lameDuck: + gracePeriod: 10s + duration: 30s + podTemplate: + spec: + containers: + - name: nats + resources: + requests: + cpu: "500m" + memory: 1Gi + limits: + cpu: "1" + memory: 1Gi + serviceTemplates: + - alias: primary + spec: + type: LoadBalancer + deletionPolicy: WipeOut diff --git a/docs/examples/nats/cluster/nats-custom-config.yaml b/docs/examples/nats/cluster/nats-custom-config.yaml new file mode 100644 index 000000000..0cf798189 --- /dev/null +++ b/docs/examples/nats/cluster/nats-custom-config.yaml @@ -0,0 +1,11 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-custom + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + configuration: + secretName: nats-custom-config + deletionPolicy: WipeOut diff --git a/docs/examples/nats/monitoring/builtin-prom-nats.yaml b/docs/examples/nats/monitoring/builtin-prom-nats.yaml new file mode 100644 index 000000000..c037c7564 --- /dev/null +++ b/docs/examples/nats/monitoring/builtin-prom-nats.yaml @@ -0,0 +1,11 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-builtin-prom + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + monitor: + agent: prometheus.io/builtin + deletionPolicy: WipeOut diff --git a/docs/examples/nats/monitoring/prom-operator-nats.yaml b/docs/examples/nats/monitoring/prom-operator-nats.yaml new file mode 100644 index 000000000..8d27dafac --- /dev/null +++ b/docs/examples/nats/monitoring/prom-operator-nats.yaml @@ -0,0 +1,16 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-monitored + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s + deletionPolicy: WipeOut diff --git a/docs/examples/nats/multi-dc/nats-distributed.yaml b/docs/examples/nats/multi-dc/nats-distributed.yaml new file mode 100644 index 000000000..2aad61e27 --- /dev/null +++ b/docs/examples/nats/multi-dc/nats-distributed.yaml @@ -0,0 +1,29 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-dc + namespace: demo +spec: + version: "2.15.1" + replicas: 6 + distributed: true + jetStream: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + arbiter: + replicas: 1 + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + podTemplate: + spec: + podPlacementPolicy: + name: nats-3dc + deletionPolicy: WipeOut diff --git a/docs/examples/nats/multi-dc/nats-gateway-a.yaml b/docs/examples/nats/multi-dc/nats-gateway-a.yaml new file mode 100644 index 000000000..0d8238517 --- /dev/null +++ b/docs/examples/nats/multi-dc/nats-gateway-a.yaml @@ -0,0 +1,15 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: dc-a + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + gateway: + name: dc-a + remotes: + - name: dc-b + urls: + - nats://dc-b-pods.demo.svc.slice.local:7222 + deletionPolicy: WipeOut diff --git a/docs/examples/nats/multi-dc/nats-hub-leaf-accept.yaml b/docs/examples/nats/multi-dc/nats-hub-leaf-accept.yaml new file mode 100644 index 000000000..ee277b24a --- /dev/null +++ b/docs/examples/nats/multi-dc/nats-hub-leaf-accept.yaml @@ -0,0 +1,15 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-hub + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + leafNodes: + accept: true + serviceTemplates: + - alias: leafnodes + spec: + type: LoadBalancer + deletionPolicy: WipeOut diff --git a/docs/examples/nats/multi-dc/nats-leaf.yaml b/docs/examples/nats/multi-dc/nats-leaf.yaml new file mode 100644 index 000000000..e449c5627 --- /dev/null +++ b/docs/examples/nats/multi-dc/nats-leaf.yaml @@ -0,0 +1,16 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-edge + namespace: demo +spec: + version: "2.15.1" + replicas: 1 + leafNodes: + jetStreamDomain: edge + remotes: + - natsRef: + name: nats-hub + namespace: demo + account: APP + deletionPolicy: WipeOut diff --git a/docs/examples/nats/multi-dc/placement-policy.yaml b/docs/examples/nats/multi-dc/placement-policy.yaml new file mode 100644 index 000000000..663030a1d --- /dev/null +++ b/docs/examples/nats/multi-dc/placement-policy.yaml @@ -0,0 +1,19 @@ +apiVersion: apps.k8s.appscode.com/v1 +kind: PlacementPolicy +metadata: + name: nats-3dc +spec: + clusterSpreadConstraint: + slice: + projectNamespace: kubeslice-demo + sliceName: demo-slice + distributionRules: + - clusterName: dc-a + replicaIndices: [0, 1, 2] + role: Member + - clusterName: dc-b + replicaIndices: [3, 4, 5] + role: Member + - clusterName: dc-c + replicaIndices: [] + role: Arbiter diff --git a/docs/examples/nats/opsrequests/horizontal-scaling-dc.yaml b/docs/examples/nats/opsrequests/horizontal-scaling-dc.yaml new file mode 100644 index 000000000..b16bae434 --- /dev/null +++ b/docs/examples/nats/opsrequests/horizontal-scaling-dc.yaml @@ -0,0 +1,17 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-hscale-dc + namespace: demo +spec: + type: HorizontalScaling + databaseRef: + name: nats-dc + horizontalScaling: + dataCenters: + - clusterName: dc-a + replicas: 5 + - clusterName: dc-b + replicas: 5 + timeout: 1h + apply: IfReady diff --git a/docs/examples/nats/opsrequests/horizontal-scaling.yaml b/docs/examples/nats/opsrequests/horizontal-scaling.yaml new file mode 100644 index 000000000..a4ba9fa6d --- /dev/null +++ b/docs/examples/nats/opsrequests/horizontal-scaling.yaml @@ -0,0 +1,13 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-hscale + namespace: demo +spec: + type: HorizontalScaling + databaseRef: + name: nats-cluster + horizontalScaling: + replicas: 5 + timeout: 30m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/reconfigure-tls-add.yaml b/docs/examples/nats/opsrequests/reconfigure-tls-add.yaml new file mode 100644 index 000000000..281f74b6d --- /dev/null +++ b/docs/examples/nats/opsrequests/reconfigure-tls-add.yaml @@ -0,0 +1,21 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-add-tls + namespace: demo +spec: + type: ReconfigureTLS + databaseRef: + name: nats-cluster + tls: + issuerRef: + apiGroup: cert-manager.io + kind: Issuer + name: nats-ca-issuer + certificates: + - alias: server + subject: + organizations: + - kubedb + timeout: 30m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/reconfigure-tls-rotate.yaml b/docs/examples/nats/opsrequests/reconfigure-tls-rotate.yaml new file mode 100644 index 000000000..4dc6705b5 --- /dev/null +++ b/docs/examples/nats/opsrequests/reconfigure-tls-rotate.yaml @@ -0,0 +1,13 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-rotate-tls + namespace: demo +spec: + type: ReconfigureTLS + databaseRef: + name: nats-tls + tls: + rotateCertificates: true + timeout: 10m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/reconfigure.yaml b/docs/examples/nats/opsrequests/reconfigure.yaml new file mode 100644 index 000000000..736feda9a --- /dev/null +++ b/docs/examples/nats/opsrequests/reconfigure.yaml @@ -0,0 +1,15 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-reconfigure + namespace: demo +spec: + type: Reconfigure + databaseRef: + name: nats-cluster + configuration: + serverConfig: + maxConnections: 20000 + restart: auto + timeout: 10m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/restart.yaml b/docs/examples/nats/opsrequests/restart.yaml new file mode 100644 index 000000000..3c6b14181 --- /dev/null +++ b/docs/examples/nats/opsrequests/restart.yaml @@ -0,0 +1,11 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-restart + namespace: demo +spec: + type: Restart + databaseRef: + name: nats-cluster + timeout: 30m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/rotate-auth.yaml b/docs/examples/nats/opsrequests/rotate-auth.yaml new file mode 100644 index 000000000..ba0e2f8b4 --- /dev/null +++ b/docs/examples/nats/opsrequests/rotate-auth.yaml @@ -0,0 +1,11 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-rotate-auth + namespace: demo +spec: + type: RotateAuth + databaseRef: + name: nats-cluster + timeout: 10m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/update-version.yaml b/docs/examples/nats/opsrequests/update-version.yaml new file mode 100644 index 000000000..0d32e4dc2 --- /dev/null +++ b/docs/examples/nats/opsrequests/update-version.yaml @@ -0,0 +1,13 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-update-version + namespace: demo +spec: + type: UpdateVersion + databaseRef: + name: nats-cluster + updateVersion: + targetVersion: "2.15.1" + timeout: 30m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/vertical-scaling.yaml b/docs/examples/nats/opsrequests/vertical-scaling.yaml new file mode 100644 index 000000000..4e539639e --- /dev/null +++ b/docs/examples/nats/opsrequests/vertical-scaling.yaml @@ -0,0 +1,20 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-vscale + namespace: demo +spec: + type: VerticalScaling + databaseRef: + name: nats-cluster + verticalScaling: + nats: + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 2Gi + timeout: 30m + apply: IfReady diff --git a/docs/examples/nats/opsrequests/volume-expansion.yaml b/docs/examples/nats/opsrequests/volume-expansion.yaml new file mode 100644 index 000000000..8ca38f901 --- /dev/null +++ b/docs/examples/nats/opsrequests/volume-expansion.yaml @@ -0,0 +1,14 @@ +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-volume-expansion + namespace: demo +spec: + type: VolumeExpansion + databaseRef: + name: nats-cluster + volumeExpansion: + mode: Online + nats: 20Gi + timeout: 30m + apply: IfReady diff --git a/docs/examples/nats/quickstart/quickstart.yaml b/docs/examples/nats/quickstart/quickstart.yaml new file mode 100644 index 000000000..d9b5bada4 --- /dev/null +++ b/docs/examples/nats/quickstart/quickstart.yaml @@ -0,0 +1,17 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-quickstart + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + jetStream: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + deletionPolicy: WipeOut diff --git a/docs/examples/nats/tls/issuer.yaml b/docs/examples/nats/tls/issuer.yaml new file mode 100644 index 000000000..fb00cae13 --- /dev/null +++ b/docs/examples/nats/tls/issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: nats-ca-issuer + namespace: demo +spec: + ca: + secretName: nats-ca diff --git a/docs/examples/nats/tls/nats-tls.yaml b/docs/examples/nats/tls/nats-tls.yaml new file mode 100644 index 000000000..dac23aa68 --- /dev/null +++ b/docs/examples/nats/tls/nats-tls.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-tls + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + tls: + issuerRef: + apiGroup: cert-manager.io + kind: Issuer + name: nats-ca-issuer + certificates: + - alias: server + subject: + organizations: + - kubedb + - alias: client + subject: + organizations: + - kubedb + deletionPolicy: WipeOut diff --git a/docs/guides/nats/_index.md b/docs/guides/nats/_index.md new file mode 100644 index 000000000..0eaba1117 --- /dev/null +++ b/docs/guides/nats/_index.md @@ -0,0 +1,10 @@ +--- +title: NATS +menu: + docs_{{ .version }}: + identifier: nats-guides + name: NATS + parent: guides + weight: 10 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/nats/auth-modes.md b/docs/guides/nats/auth-modes.md new file mode 100644 index 000000000..5d3f21e68 --- /dev/null +++ b/docs/guides/nats/auth-modes.md @@ -0,0 +1,131 @@ +--- +title: NATS Authentication Modes +menu: + docs_{{ .version }}: + identifier: nats-auth + name: Auth Modes + parent: nats-guides + weight: 50 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# NATS Authentication Modes + +NATS has no built-in admin user, so KubeDB defines two accounts and generates the credentials. + +| Account | User | Purpose | +|---|---|---| +| `SYS` (system account) | `kubedb-sys` | operator only: evacuate, peer-remove, meta-leader step-down, `$SYS` events. Stored in `-sys-auth`. You never need it. | +| `APP` | `admin` | applications and the KubeDB health checker. Stored in `-auth`. | + +`spec.auth.mode` selects how these are expressed. It is **immutable**: a running cluster cannot move between modes. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/auth](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/auth) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Before You Begin + +- You need a Kubernetes cluster with `kubectl` configured. If you do not have one, create it with [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). +- Install KubeDB following the steps [here](/docs/setup/README.md), with `--set global.featureGates.NATS=true` so the NATS CRDs are installed. +- This tutorial uses the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +## Config mode (default) + +Accounts and users are rendered into the server configuration. Passwords are stored as bcrypt hashes in the config; the plaintext stays in the Secrets. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-config-auth + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + auth: + mode: Config + deletionPolicy: WipeOut +``` + +- Use your own admin password by creating a Secret with `username: admin` and a `password`, and setting `spec.authSecret.name`. The default name is `-auth`. +- Add accounts and users with `accounts.conf` in [`spec.configuration`](/docs/guides/nats/clustering.md#custom-configuration). +- `spec.disableSecurity: true` lets clients connect without credentials. They land in the `APP` account through `no_auth_user`. The system account keeps its password. This is immutable and is not allowed in Operator mode. + +## Operator mode (decentralized JWT) + +The trust chain is *operator, accounts, users*, all signed JWTs. KubeDB generates the whole chain when `spec.auth.operator` has no `externalSecret`. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-operator-auth + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + auth: + mode: Operator + operator: + resolverStorage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + allowDelete: false + deletionPolicy: WipeOut +``` + +KubeDB creates these Secrets: + +| Secret | Contents | +|---|---| +| `-operator` | operator seed, signing-key seed, `operator.jwt`. Never mounted into pods. | +| `-sys-creds` | `sys.creds` for the `SYS` user; also what you need to `nsc push` new accounts. | +| `-auth` | `user.creds` for the `admin` user of `APP`, plus `username` and `password` keys so the AppBinding keeps its usual shape. | +| `-account-` | one per managed account: account seed, signing key, `account.jwt`. | + +- `resolverStorage` is the PVC of the full account resolver, mounted at `/resolver` (default `1Gi`). +- `allowDelete` is the resolver's `allow_delete` (default `false`). +- Users and accounts live outside the config. Manage them with `nsc` or `nats auth` using `-sys-creds`. The `accounts` and `authorization` keys are rejected in `spec.configuration`. +- Connect with the credentials file: + + ```bash + $ kubectl get secret -n demo nats-operator-auth -o jsonpath='{.data.user\.creds}' | base64 -d > user.creds + $ nats --server nats://127.0.0.1:4222 --creds user.creds server check connection + ``` + +### Bring your own operator + +If you created the operator with `nsc`, put `operator.jwt`, `sys-account.jwt` and `sys.creds` in a Secret and reference it. KubeDB then generates nothing and only renders the server configuration. + +```yaml +spec: + auth: + mode: Operator + operator: + externalSecret: + name: my-nats-operator +``` + +Rotating the operator key is your responsibility in this case. KubeDB does not rotate the operator key in either variant. + +> **Keep the key Secrets.** In Operator mode the seeds exist nowhere else. Losing them makes the account JWTs in the resolver unusable for further changes. They are included in [backups](/docs/guides/nats/backup.md). + +## Rotating credentials + +A `RotateAuth` ops request rotates credentials. In Config mode the old and new `admin` hash are both valid until the new one is active, then the old one is removed. Only the removed user's connections are closed. In Operator mode it issues a new user key and JWT, revokes the old key and pushes the updated account JWT, with no config reload and no restart. See [ops requests](/docs/guides/nats/ops-requests.md#rotateauth). + +## Cleanup + +```bash +$ kubectl delete -n demo nats/nats-config-auth nats/nats-operator-auth +``` diff --git a/docs/guides/nats/backup.md b/docs/guides/nats/backup.md new file mode 100644 index 000000000..2e7228920 --- /dev/null +++ b/docs/guides/nats/backup.md @@ -0,0 +1,139 @@ +--- +title: Backup and Restore NATS +menu: + docs_{{ .version }}: + identifier: nats-backup + name: Backup + parent: nats-guides + weight: 90 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Backup and Restore NATS + +Core NATS holds no data, so backup means **JetStream streams**. NATS uses [KubeStash](https://kubestash.com) with the `nats-addon`. KubeDB does not add new backup kinds. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/backup](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/backup) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## What is backed up + +- For each stream in the application account, the server's snapshot API writes the stream config and state plus a compressed archive in chunks. Consumer state is included by default. +- Publishers are not blocked while a snapshot runs, but `purge` and `delete` on that stream wait or fail until it ends. +- KV and object-store buckets are streams underneath, so they are covered. +- Accounts and users are part of the `NATS` spec and Secrets, which the `manifest-backup` task covers. +- **PVC volume snapshots are not a supported backup.** One node's disk is only one replica of a Raft group. + +## Prerequisites + +- Install KubeStash and the KubeDB addons (`nats-addon` is in the `kubedb-kubestash-catalog` chart). +- A `BackupStorage`, a `RetentionPolicy` and an encryption Secret in `demo`. + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + endpoint: http://minio.storage.svc:9000 + bucket: nats-backup + region: us-east-1 + secretName: s3-secret + usagePolicy: + allowedNamespaces: + from: All + deletionPolicy: WipeOut +``` + +## Backup + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: nats-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: NATS + namespace: demo + name: nats-cluster + backends: + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/30 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: s3-nats-repo + backend: s3-backend + directory: /nats + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: nats-addon + tasks: + - name: logical-backup + params: + includeConsumers: "true" + - name: manifest-backup +``` + +`logical-backup` parameters: `streams` (comma-separated, default all), `excludeStreams`, `includeConsumers` (default `true`), `chunkSize` and `windowSize` (lower them on a slow link), and `jsDomain` if the server has one. + +### Operator mode + +For `auth.mode: Operator` the key Secrets (`-operator`, `-sys-creds`, `-auth`, every `-account-`) are part of the backup and exist nowhere else. The `manifest-backup` task is therefore **mandatory**: the backup is refused without it. When the operator is externally managed (`operator.externalSecret`), those keys are yours to keep and KubeStash only warns. The repository then holds private keys, so treat its encryption password and storage access as key-grade secrets. + +## Restore + +A stream can only be restored if it does not exist, so restore into a fresh `NATS` object (or one whose streams are deleted). + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: nats-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: NATS + namespace: demo + name: nats-restored + dataSource: + repository: s3-nats-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: nats-addon + tasks: + - name: manifest-restore + - name: logical-backup-restore +``` + +Streams are recreated with their sequence numbers and consumer positions. In Operator mode the key Secrets are recreated **before** the streams and the config, and the restore fails if they are missing, because a new operator key would reject every existing account JWT. The `logical-backup-restore` parameter `overwriteKeys` (default `true`) replaces the target's key Secrets with the snapshot's, `waitTimeout` (seconds, default `300`) waits for the target to be ready, and `jsDomain` is as above. + +## Cleanup + +```bash +$ kubectl delete -n demo backupconfiguration/nats-backup restoresession/nats-restore +``` diff --git a/docs/guides/nats/clustering.md b/docs/guides/nats/clustering.md new file mode 100644 index 000000000..6bf8b77c9 --- /dev/null +++ b/docs/guides/nats/clustering.md @@ -0,0 +1,152 @@ +--- +title: NATS Clustering and Configuration +menu: + docs_{{ .version }}: + identifier: nats-clustering + name: Clustering + parent: nats-guides + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Clustering and Configuration + +A `NATS` object is one cluster. The servers form a full mesh over routes on port 6222. A server needs only one seed route because the servers gossip, but KubeDB renders routes to every current pod, so a seed that is down at boot does not matter. Routes are authenticated with a generated credential stored in `-cluster-auth`. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/cluster](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/cluster) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Before You Begin + +- You need a Kubernetes cluster with `kubectl` configured. If you do not have one, create it with [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). +- Install KubeDB following the steps [here](/docs/setup/README.md), with `--set global.featureGates.NATS=true` so the NATS CRDs are installed. +- This tutorial uses the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +## Three-server cluster with JetStream + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-cluster + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + jetStream: + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: standard + memoryStoreSize: 256Mi + serverConfig: + maxConnections: 10000 + maxPayload: 1048576 + lameDuck: + gracePeriod: 10s + duration: 30s + podTemplate: + spec: + containers: + - name: nats + resources: + requests: + cpu: "500m" + memory: 1Gi + limits: + cpu: "1" + memory: 1Gi + serviceTemplates: + - alias: primary + spec: + type: LoadBalancer + deletionPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/cluster/nats-cluster.yaml +nats.kubedb.com/nats-cluster created +``` + +- `jetStream.memoryStoreSize` caps the JetStream memory store. If unset, memory streams are disabled. It must be at most 50% of the container memory limit. +- `jetStream.storage` sets the file store PVC. KubeDB writes `max_file_store` to the config explicitly so that [volume expansion](/docs/guides/nats/ops-requests.md#volumeexpansion) can raise it without a restart. +- `serverConfig` holds typed limits: `maxConnections`, `maxPayload`, `maxPending`, `maxSubscriptions`, `maxControlLine`, `writeDeadline`, `authTimeout`, `debug`, `trace`. `maxPayload` must not exceed `maxPending`. +- `lameDuck` sets how long a pod keeps serving while it asks clients to move before it stops. `duration` must be at least `30s` (the server minimum) and `gracePeriod` must be smaller than `duration`. Changing it rolls the pods, since it also changes `terminationGracePeriodSeconds`. + +## Core NATS without JetStream + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-core + namespace: demo +spec: + version: "2.15.1" + replicas: 2 + jetStream: + enabled: false + deletionPolicy: WipeOut +``` + +No PVC is created and any replica count `>= 1` is valid. `jetStream.enabled` cannot be changed after creation. + +## Custom configuration + +Ordinary server settings go in a Secret referenced by `spec.configuration.secretName`. Two keys are supported: + +- `nats.conf`: extra top-level server settings. +- `accounts.conf`: extra accounts, in `Config` auth mode only. The `SYS` and `APP` account names are reserved. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: nats-custom-config + namespace: demo +stringData: + nats.conf: | + ping_interval: "30s" + max_connections: 5000 + accounts.conf: | + ORDERS { + jetstream: enabled + users: [{user: orders, password: orders-secret}] + } +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/cluster/custom-config.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/cluster/nats-custom-config.yaml +``` + +KubeDB owns `server_name`, `listen`, `jetstream`, `cluster`, `accounts` and `authorization`; the webhook rejects them in `nats.conf`. Most settings reload in place when changed through a [Reconfigure](/docs/guides/nats/ops-requests.md#reconfigure) ops request. + +## Resources KubeDB creates + +```bash +$ kubectl get svc -n demo -l app.kubernetes.io/instance=nats-cluster +NAME TYPE PORT(S) +nats-cluster LoadBalancer 4222/TCP +nats-cluster-pods ClusterIP 6222/TCP,8222/TCP +nats-cluster-stats ClusterIP 7777/TCP +``` + +`spec.serviceTemplates` accepts the aliases `primary`, `stats` and `leafnodes`. + +## Cleanup + +```bash +$ kubectl delete -n demo nats/nats-cluster nats/nats-core nats/nats-custom +``` diff --git a/docs/guides/nats/monitoring.md b/docs/guides/nats/monitoring.md new file mode 100644 index 000000000..2e8b7a67c --- /dev/null +++ b/docs/guides/nats/monitoring.md @@ -0,0 +1,88 @@ +--- +title: Monitoring NATS +menu: + docs_{{ .version }}: + identifier: nats-monitoring + name: Monitoring + parent: nats-guides + weight: 60 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring NATS + +KubeDB adds a `prometheus-nats-exporter` sidecar when `spec.monitor` is set. It reads the local monitor port (`8222`) with `-connz -routez -subz -varz` and, when JetStream is on, `-jsz=all`, and serves `/metrics` on port `7777` of the `-stats` Service. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/monitoring](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/monitoring) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Before You Begin + +- You need a Kubernetes cluster with `kubectl` configured. If you do not have one, create it with [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). +- Install KubeDB following the steps [here](/docs/setup/README.md), with `--set global.featureGates.NATS=true` so the NATS CRDs are installed. +- This tutorial uses the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +## Prometheus operator + +Install the [Prometheus operator](https://github.com/prometheus-operator/prometheus-operator), then: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-monitored + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s + deletionPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/monitoring/prom-operator-nats.yaml +$ kubectl get servicemonitor -n demo +$ kubectl get svc -n demo nats-monitored-stats +``` + +The `serviceMonitor.labels` must match your Prometheus `serviceMonitorSelector`. + +## Builtin Prometheus + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/monitoring/builtin-prom-nats.yaml +``` + +The stats Service carries the `prometheus.io/scrape`, `prometheus.io/path` and `prometheus.io/port` annotations for your Prometheus configuration. See [using builtin Prometheus](/docs/guides/rabbitmq/monitoring/using-builtin-prometheus.md) for a scrape configuration that applies to any KubeDB database. + +## Alerts worth setting + +- `slow_consumers > 0` +- the JetStream meta group has no leader +- a peer reported `outdated` for more than a few minutes +- JetStream storage above 80% of `max_file_store` +- missing routes for a peer (`/routez` grouped by remote name) +- pods in lame-duck mode for longer than the configured duration + +Stream replica lag per peer is not an exporter metric. KubeDB reads `/jsz` in its health checker and reports it as the conditions `JetStreamQuorum` and `UnderReplicatedStreams` on the `NATS` status. Phase `Critical` means some pods are not Ready but the meta group still has quorum. + +Port `8222` has no authentication and exposes connection and subscription details. KubeDB creates a NetworkPolicy that allows ingress to it only from the operator and Prometheus pods. + +## Cleanup + +```bash +$ kubectl delete -n demo nats/nats-monitored nats/nats-builtin-prom +``` diff --git a/docs/guides/nats/multi-dc.md b/docs/guides/nats/multi-dc.md new file mode 100644 index 000000000..8c5e258a3 --- /dev/null +++ b/docs/guides/nats/multi-dc.md @@ -0,0 +1,222 @@ +--- +title: NATS Across Data Centers +menu: + docs_{{ .version }}: + identifier: nats-multi-dc + name: Multi-DC + parent: nats-guides + weight: 70 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# NATS Across Data Centers + +**A NATS cluster never spans data centers.** Each DC runs its own NATS cluster with a full route mesh. Clusters in different DCs are joined by **gateways** into a *super-cluster*, and edge sites that can only dial out attach as **leaf nodes**. A gateway keeps one connection per remote cluster and forwards only the messages that have remote interest, which is why this scales across a WAN better than stretching one route mesh. + +Every DC is active. There is no primary DC and no failover step: clients talk to their local DC, and the gateways carry messages to the DCs that have subscribers. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/multi-dc](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/multi-dc) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Two ways to build a super-cluster + +| | Distributed mode | Hand-built gateways | +|---|---|---| +| You create | one `NATS` object with `spec.distributed: true` and a `PlacementPolicy` | one `NATS` object per DC, each with `spec.gateway` | +| KubeDB creates | a PetSet, governing Service and client Service per DC; gateway remotes; the Arbiter DC | only what a single `NATS` normally gets | +| Needs | OCM hub and spokes, KubeSlice | network reachability between the gateway ports | + +## Gateways between separate NATS objects + +Give each cluster a gateway whose name equals its cluster name (the object name). The server refuses to start otherwise. Gateways gossip, so a few remotes are enough to seed discovery. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: dc-a + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + gateway: + name: dc-a + remotes: + - name: dc-b + urls: + - nats://dc-b-pods.demo.svc.slice.local:7222 + deletionPolicy: WipeOut +``` + +A remote may instead use `natsRef` to point at another `NATS` object through its AppBinding, so KubeDB fills in the URLs. + +- Gateway configuration is not reloadable. Changing a configured remote is a rolling restart of that cluster. +- **Adding a DC does not restart the existing DCs.** The new DC lists any one existing DC, and the existing clusters learn about it by gossip. They keep their original list until their next planned roll. KubeDB never sets `rejectUnknown`; set `gateway.rejectUnknown: true` only if you want a closed super-cluster and accept the restart. +- The gateway uses the `server` certificate when `spec.tls` is set (`gatewayTLS`, default on). Gateway TLS material reloads in place. +- Gateways use a generated shared credential, `-gateway-auth`. In Operator mode, gateways between clusters of the same operator need no JWT work. + +## Distributed mode: one cluster per DC from one object + +Distributed mode needs a kubeops.dev `PlacementPolicy` with a `clusterSpreadConstraint`. It requires KubeSlice (`slice` is mandatory) and must **not** set `failoverPolicy`, because NATS has no primary DC. + +```yaml +apiVersion: apps.k8s.appscode.com/v1 +kind: PlacementPolicy +metadata: + name: nats-3dc +spec: + clusterSpreadConstraint: + slice: + projectNamespace: kubeslice-demo + sliceName: demo-slice + distributionRules: + - clusterName: dc-a + replicaIndices: [0, 1, 2] + role: Member + - clusterName: dc-b + replicaIndices: [3, 4, 5] + role: Member + - clusterName: dc-c + replicaIndices: [] + role: Arbiter +``` + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-dc + namespace: demo +spec: + version: "2.15.1" + replicas: 6 + distributed: true + jetStream: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + arbiter: + replicas: 1 + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + podTemplate: + spec: + podPlacementPolicy: + name: nats-3dc + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/multi-dc/placement-policy.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/multi-dc/nats-distributed.yaml +``` + +For each `Member` rule KubeDB creates: + +- a PetSet `-` with its own derived placement policy, so each DC is an ordinary cluster. `cluster.name` and `gateway.name` equal the DC name, and routes list only that DC's pods. +- a governing Service `-pods-` and a client Service `-`. Pods are exported over the KubeSlice network on ports 6222, 7222, 7422 and 8222. +- gateway remotes: every DC lists the other DCs. Cross-DC names use `*.svc.slice.local`. + +Do not set `spec.gateway.name` when `distributed` is true; each DC's gateway is named after the DC. + +### Clients stay local + +Clients learn only the servers of their own cluster, so a client cannot discover another DC on its own. Give the application an explicit list: + +``` +nats://nats-dc-dc-a.demo.svc:4222,nats://nats-dc-dc-b.demo.svc:4222 +``` + +The AppBinding of a distributed `NATS` lists every DC Service in `spec.clientConfig`, so KubeStash and other tools see them all. + +## The Arbiter DC and why you need three DCs + +JetStream has **one meta group over every server in the super-cluster**. Gateways always merge the groups. A whole-DC loss is survivable only if a majority of meta peers remains, which needs at least three DCs. + +- A gateway-joined `NATS` with JetStream needs an **odd number of DCs counting any Arbiter DC, and at least 3**. The webhook enforces this. +- With **two data DCs, add an Arbiter DC**: an `Arbiter` rule with *empty* `replicaIndices`. KubeDB runs `spec.arbiter.replicas` (default 1) servers there in their own PetSet `--arbiter`, with a small PVC (default `1Gi`). They are tagged `!jetstream`, so they vote in the meta group but hold no stream data, serve no clients and open no leaf port. Two DCs of three servers plus one arbiter make 7 meta peers; losing a data DC leaves 4 of 7, so stream and consumer management continues. +- An arbiter can still be elected meta-leader. KubeDB steps the leader down to a data DC after every ops request and when its health checker sees the leader on an arbiter. This is a preference, not a guarantee. +- **No third site?** Connect the two DCs with leaf nodes and a distinct JetStream domain per DC (each DC keeps its own meta group, data crosses by mirror or source) rather than gateways. The webhook requires this for two data DCs with JetStream and no Arbiter. + +50 ms between DCs is fine: Raft heartbeats are 1 s and elections 4 to 9 s. KubeDB reads the gateway round-trip time per remote and raises a warning condition above 100 ms. + +### Placing streams + +Create each stream inside one DC so its commits stay at LAN latency: + +```bash +$ nats stream add ORDERS --subjects 'orders.*' --replicas 3 --cluster dc-a +``` + +Copy a stream to another DC with a **mirror** or **source**. These are NATS-native asynchronous replication, declared as stream configuration rather than KubeDB API: + +```bash +$ nats stream add ORDERS_DR --mirror ORDERS --replicas 3 --cluster dc-b +``` + +To promote the copy after losing `dc-a`, edit the mirror stream and remove its `mirror` setting so it accepts writes. Mirrors are asynchronous, so the replication lag is the data you can lose. + +## Leaf nodes + +A leaf node attaches an edge site to a hub without being part of its mesh. The leaf dials out, so the hub needs no inbound route to the edge. + +Hub side, in any DC's cluster: + +```yaml +spec: + leafNodes: + accept: true + serviceTemplates: + - alias: leafnodes + spec: + type: LoadBalancer +``` + +Port 7422 opens on every pod and on the client Service. Its type comes from the `leafnodes` alias (default `ClusterIP`). Choose `LoadBalancer` so an edge site can reach it. + +Leaf side, another `NATS` object, usually one replica: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-edge + namespace: demo +spec: + version: "2.15.1" + replicas: 1 + leafNodes: + jetStreamDomain: edge + remotes: + - natsRef: + name: nats-hub + namespace: demo + account: APP + deletionPolicy: WipeOut +``` + +- A remote has `urls` or `natsRef`, an optional `account`, an optional `credentialsSecret` (a Secret with a `creds` key, Operator mode) and an optional JetStream `domain`. +- In Config mode the leaf binds the default account unless `account` is set, and then the hub must define the same account. In Operator mode the leaf presents its credentials and the hub attaches it to the matching account. A leaf on the default account has no subject boundary, so the webhook warns about it. +- **A leaf that runs its own JetStream needs `jetStreamDomain`.** The webhook requires it whenever JetStream is on and `leafNodes.remotes` is set, otherwise the leaf would extend the hub's JetStream. +- Changing a remote's URLs, account or credentials path reloads live as remove-plus-add. Rotating the contents of the same `.creds` file needs nothing, because the next reconnect reads it. The hub's own `leafnodes` block (listen, authorization users) does not reload, so changing hub leaf credentials is a rolling restart. +- A leaf's local JetStream is backed up like any `NATS`; the hub's copy is independent. + +## Day-2 operations in multi-DC + +Ops requests run inside one DC at a time and one pod at a time, never two DCs concurrently. The Arbiter DC goes first and the DC holding the meta leader goes last. See [ops requests](/docs/guides/nats/ops-requests.md#multi-dc-clusters). + +## Cleanup + +```bash +$ kubectl delete -n demo nats/nats-dc nats/nats-hub nats/nats-edge nats/dc-a +$ kubectl delete placementpolicy nats-3dc +``` diff --git a/docs/guides/nats/ops-requests.md b/docs/guides/nats/ops-requests.md new file mode 100644 index 000000000..d1d126c90 --- /dev/null +++ b/docs/guides/nats/ops-requests.md @@ -0,0 +1,189 @@ +--- +title: NATS Ops Requests +menu: + docs_{{ .version }}: + identifier: nats-ops-requests + name: Ops Requests + parent: nats-guides + weight: 80 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# NATS Ops Requests + +Day-2 operations use `NATSOpsRequest`. Fields that an ops request owns (version, replicas, resources, storage size, configuration, TLS, auth secret) are rejected when edited directly on a provisioned `NATS` object. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/opsrequests](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/opsrequests) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +Supported types: `UpdateVersion`, `HorizontalScaling`, `VerticalScaling`, `VolumeExpansion`, `Reconfigure`, `ReconfigureTLS`, `RotateAuth`, `Restart` and `StorageMigration`. + +## How a roll works + +Anything that restarts pods (`UpdateVersion`, `Restart`, memory changes, `Reconfigure` with a restart, enabling or disabling TLS, `StorageMigration`) uses one procedure: + +1. Before the first pod, strict `/healthz` on every pod must be `ok` and the JetStream meta group must have a leader. Otherwise the request waits, then fails after its `timeout`. +2. Non-leader pods roll first, highest ordinal first. The meta leader rolls last, after KubeDB steps leadership down to a pod that is already current. +3. For each pod KubeDB deletes it; the pre-stop hook puts the server in lame-duck mode (listeners close, Raft leaderships transfer, clients are moved away over the lame-duck duration). It then waits for the pod to be Ready, for strict `/healthz` (all streams and consumers caught up) and for its routes to reappear on its peers. +4. The next pod is touched only after that. A pod that does not catch up within `timeout` fails the request and leaves the cluster paused for you to inspect. + +Clients keep working only if they use a library that reconnects, and connect through the Service or a server list. A stream with `num_replicas: 1` is unavailable while its pod restarts; KubeDB emits a warning event naming such streams. + +## UpdateVersion + +```yaml +apiVersion: ops.kubedb.com/v1alpha1 +kind: NATSOpsRequest +metadata: + name: nats-update-version + namespace: demo +spec: + type: UpdateVersion + databaseRef: + name: nats-cluster + updateVersion: + targetVersion: "2.15.1" + timeout: 30m + apply: IfReady +``` + +The target `NATSVersion` must exist and be allowed by the current version's update constraints. All three images (server, reloader, exporter) are updated. Mixed versions during the roll are supported by NATS. There is no downgrade and a failed pod is not rolled back, so take a [backup](/docs/guides/nats/backup.md) first; it is the supported recovery path. + +## HorizontalScaling + +```yaml +spec: + type: HorizontalScaling + databaseRef: + name: nats-cluster + horizontalScaling: + replicas: 5 +``` + +With JetStream the count must be 1 or an odd number `>= 3`. + +- **Scale up** adds pods that join the mesh and the meta group on their own. No data moves: existing streams stay where they are, and rebalancing is a user action (`nats stream edit`). +- **Scale down** removes one ordinal at a time, highest first. It is refused if any stream or consumer has more replicas than the target size. KubeDB evacuates the server from the meta group and its streams, lowers the replica count, removes the peer once the pod is gone, and deletes the PVC of the removed ordinal. +- Going from 1 to 3 replicas with JetStream restarts the original pod with a cluster configuration first. + +## VerticalScaling + +```yaml +spec: + type: VerticalScaling + databaseRef: + name: nats-cluster + verticalScaling: + nats: + resources: + requests: {cpu: "1", memory: 2Gi} + limits: {cpu: "2", memory: 2Gi} +``` + +`nats`, `reloader` and `exporter` can be sized separately. `memoryStoreSize` must stay at or below 50% of the new memory limit. A memory change always restarts the pods, because `GOMEMLIMIT` is read at process start; a CPU-only change can be applied in place with `mode: InPlace`. + +## VolumeExpansion + +```yaml +spec: + type: VolumeExpansion + databaseRef: + name: nats-cluster + volumeExpansion: + mode: Online + nats: 20Gi +``` + +`mode` is `Online` or `Offline`. After the PVCs grow, KubeDB raises `max_file_store` to 90% of the new size and reloads the config. The server accepts increases without a restart, and rejects decreases. With a storage driver that resizes online, no pod restarts. + +## Reconfigure + +```yaml +spec: + type: Reconfigure + databaseRef: + name: nats-cluster + configuration: + serverConfig: + maxConnections: 20000 + restart: auto +``` + +You can also give `configSecret`, `applyConfig` (inline `nats.conf` or `accounts.conf` content) or `removeCustomConfig`, and set `lameDuck`. KubeDB validates the new config with `nats-server -t` first, then the reloader signals the servers. + +| Reloads live | Needs a roll | +|---|---| +| users, accounts, authorization, resolver settings | `listen`, `http_port`, cluster `host`/`port` | +| `max_connections`, `max_payload`, `max_control_line`, `ping_interval`, `write_deadline`, `auth_timeout` | `max_subscriptions`, `max_pending` | +| `debug`, `trace`, logging, `server_tags` | `lame_duck_duration`, `lame_duck_grace_period`, `system_account` | +| client TLS, cluster routes and TLS | `no_auth_user`, `allow_non_tls` | +| `max_file_store` increases | `store_dir`, JetStream domain, decreases | +| leaf remotes added, removed or re-pointed | `gateway` (except TLS material), hub-side `leafnodes` block | + +`restart` is `auto` (roll only if a right-hand key changed), `false` (fail with the server's own error if the change is not reloadable) or `true` (always roll). After a reload KubeDB checks `/varz` on every pod to confirm the new config was loaded, and fails the request, naming the pods that differ, if one rejected it. + +## ReconfigureTLS + +```yaml +spec: + type: ReconfigureTLS + databaseRef: + name: nats-tls + tls: + rotateCertificates: true +``` + +`tls.issuerRef` and `tls.certificates` add or change TLS and `tls.remove: true` removes it. See the [TLS guide](/docs/guides/nats/tls.md#day-2-changes) for what each change does to running clients. + +## RotateAuth + +```yaml +spec: + type: RotateAuth + databaseRef: + name: nats-cluster +``` + +Without `authentication.secretRef` KubeDB generates a new password. With it, KubeDB uses the credentials from your Secret. The old and new credentials are both valid during the change, so applications have a window to read the new Secret. The cluster-route and gateway credentials cannot hold two values at once. The route credential is replaced in one config write and reloaded. The gateway block does not reload, so rotating it is a rolling restart per DC. See [auth modes](/docs/guides/nats/auth-modes.md#rotating-credentials). + +## Restart + +```yaml +spec: + type: Restart + databaseRef: + name: nats-cluster +``` + +A rolling restart of every pod using the procedure above. + +## Multi-DC clusters + +For a distributed `NATS`, a request runs inside one DC at a time, the Arbiter DC first and the DC with the meta leader last. After every request KubeDB steps the meta leader down to a data DC. `HorizontalScaling` takes per-DC targets and is refused if a stream placed in that DC would fall below its replica count: + +```yaml +spec: + type: HorizontalScaling + databaseRef: + name: nats-dc + horizontalScaling: + dataCenters: + - clusterName: dc-a + replicas: 5 + - clusterName: dc-b + replicas: 5 +``` + +`UpdateVersion` goes DC by DC. Adding a DC rolls only the new DC. Removing a DC is refused until its streams are moved or mirrored elsewhere. + +## Autoscaling + +A `NATSAutoscaler` object scales compute and, for JetStream, storage, using the same ops requests under the hood. + +## Cleanup + +```bash +$ kubectl delete natsopsrequest -n demo --all +``` diff --git a/docs/guides/nats/overview.md b/docs/guides/nats/overview.md new file mode 100644 index 000000000..5c026f674 --- /dev/null +++ b/docs/guides/nats/overview.md @@ -0,0 +1,64 @@ +--- +title: NATS Overview +menu: + docs_{{ .version }}: + identifier: nats-overview + name: Overview + parent: nats-guides + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# NATS with KubeDB + +[NATS](https://nats.io) is a messaging system with core publish/subscribe and request/reply, and **JetStream**, a persistence layer with streams, consumers, key-value and object stores. KubeDB runs NATS as a managed database: you declare a `NATS` object and KubeDB creates the PetSet, Services, credentials, certificates and monitoring, and runs day-2 operations without dropping clients. + +## What KubeDB creates + +A `NATS` object is always **one NATS cluster**. All pods are identical `nats-server` processes, so there is a single PetSet and no per-role topology. Each pod runs these containers: + +| Container | Purpose | +|---|---| +| `nats` | the server | +| `reloader` | signals the server to reload its configuration when the config Secret changes | +| `exporter` | `prometheus-nats-exporter`, only when monitoring is enabled | + +| Port | Purpose | Exposed on | +|---|---|---| +| 4222 | clients | the client Service `` | +| 6222 | cluster routes | the headless Service `-pods` only | +| 8222 | HTTP monitor, probes | the headless Service; limited by a NetworkPolicy | +| 7777 | Prometheus exporter | the stats Service `-stats` | +| 7222 | gateway (multi-DC) | the headless Service only | +| 7422 | leaf nodes | the headless Service and, when `leafNodes.accept` is true, the client Service | + +## Custom resources + +| Kind | API group | Purpose | +|---|---|---| +| `NATS` | `kubedb.com/v1alpha2` | the cluster: replicas, JetStream storage, auth, TLS, configuration, monitoring | +| `NATSVersion` | `catalog.kubedb.com/v1alpha1` | server, reloader and exporter images, upgrade constraints, lame-duck defaults | +| `NATSOpsRequest` | `ops.kubedb.com/v1alpha1` | day-2 operations | +| `NATSAutoscaler` | `autoscaling.kubedb.com/v1alpha1` | compute and storage autoscaling | + +The resource plural of `NATS` is `natses` (a plural equal to the singular is not allowed). The short name is `nats`, so `kubectl get nats` works. + +## JetStream and replica counts + +- With JetStream enabled (the default) use `replicas: 1` or an odd number `>= 3`, because streams and the JetStream meta group are Raft groups. +- With `jetStream.enabled: false` the cluster holds no state, needs no PVC, and any `replicas >= 1` is valid. +- `jetStream.storageType: Ephemeral` uses an `emptyDir` and is allowed only with one replica. + +## Guides + +- [Quickstart](/docs/guides/nats/quickstart.md) +- [Clustering and configuration](/docs/guides/nats/clustering.md) +- [TLS](/docs/guides/nats/tls.md) +- [Authentication modes](/docs/guides/nats/auth-modes.md) +- [Monitoring](/docs/guides/nats/monitoring.md) +- [Multi-DC: gateways, leaf nodes and distributed mode](/docs/guides/nats/multi-dc.md) +- [Ops requests](/docs/guides/nats/ops-requests.md) +- [Backup and restore](/docs/guides/nats/backup.md) diff --git a/docs/guides/nats/quickstart.md b/docs/guides/nats/quickstart.md new file mode 100644 index 000000000..2cded2b3c --- /dev/null +++ b/docs/guides/nats/quickstart.md @@ -0,0 +1,122 @@ +--- +title: NATS Quickstart +menu: + docs_{{ .version }}: + identifier: nats-quickstart + name: Quickstart + parent: nats-guides + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# NATS QuickStart + +This tutorial shows how to run a three-server NATS cluster with JetStream using KubeDB and connect to it. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/quickstart](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/quickstart) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Before You Begin + +- You need a Kubernetes cluster with `kubectl` configured. If you do not have one, create it with [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). +- Install KubeDB following the steps [here](/docs/setup/README.md), with `--set global.featureGates.NATS=true` so the NATS CRDs are installed. +- This tutorial uses the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +## Find Available NATSVersion + +KubeDB installs a `NATSVersion` object for every supported release: + +```bash +$ kubectl get natsversion +NAME VERSION DB_IMAGE DEPRECATED AGE +2.14.8 2.14.8 ghcr.io/appscode-images/nats:2.14.8 2m +2.15.1 2.15.1 ghcr.io/appscode-images/nats:2.15.1 2m +``` + +## Create a NATS cluster + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-quickstart + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + jetStream: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + deletionPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/quickstart/quickstart.yaml +nats.kubedb.com/nats-quickstart created +``` + +Here, + +- `spec.version` is the name of the `NATSVersion` object that holds the images. +- `spec.replicas` is the number of `nats-server` pods. With JetStream on, use 1 or an odd number `>= 3`. +- `spec.jetStream.storage` is the PVC that holds the JetStream file store. The default is `10Gi`. Do not set limits here, PVCs are not resized automatically; use a [VolumeExpansion](/docs/guides/nats/ops-requests.md#volumeexpansion) ops request. +- `spec.deletionPolicy` controls what is kept when the object is deleted: `DoNotTerminate` blocks the delete, `Halt` keeps PVCs and Secrets, `Delete` keeps Secrets, `WipeOut` removes everything. + +Wait for the object to become `Ready`: + +```bash +$ kubectl get nats -n demo nats-quickstart +NAME VERSION STATUS AGE +nats-quickstart 2.15.1 Ready 2m + +$ kubectl get petset,svc,pvc,appbinding -n demo -l app.kubernetes.io/instance=nats-quickstart +``` + +KubeDB creates a PetSet, a client Service `nats-quickstart` (port 4222), a headless Service `nats-quickstart-pods` (routes, monitor), PVCs, and an AppBinding. + +## Connect + +KubeDB generates the `admin` user for the application account and stores it in `nats-quickstart-auth`: + +```bash +$ kubectl get secret -n demo nats-quickstart-auth -o jsonpath='{.data.username}' | base64 -d +admin +$ kubectl get secret -n demo nats-quickstart-auth -o jsonpath='{.data.password}' | base64 -d +``` + +Port-forward and use the [`nats` CLI](https://github.com/nats-io/natscli): + +```bash +$ kubectl port-forward -n demo svc/nats-quickstart 4222 +$ nats --server nats://127.0.0.1:4222 --user admin --password "$PASS" server check connection +$ nats --server nats://127.0.0.1:4222 --user admin --password "$PASS" stream add ORDERS --subjects 'orders.*' --replicas 3 --storage file --defaults +$ nats --server nats://127.0.0.1:4222 --user admin --password "$PASS" pub orders.new hello +$ nats --server nats://127.0.0.1:4222 --user admin --password "$PASS" stream info ORDERS +``` + +Because the stream has `--replicas 3` and the cluster has three servers, every message is stored on all three pods. + +## Cleanup + +```bash +$ kubectl patch -n demo nats/nats-quickstart -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +$ kubectl delete -n demo nats/nats-quickstart +$ kubectl delete ns demo +``` + +## Next Steps + +- Learn about [clustering and configuration](/docs/guides/nats/clustering.md). +- Secure it with [TLS](/docs/guides/nats/tls.md). diff --git a/docs/guides/nats/tls.md b/docs/guides/nats/tls.md new file mode 100644 index 000000000..56233c882 --- /dev/null +++ b/docs/guides/nats/tls.md @@ -0,0 +1,105 @@ +--- +title: Run NATS with TLS +menu: + docs_{{ .version }}: + identifier: nats-tls + name: TLS/SSL + parent: nats-guides + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Run NATS with TLS + +KubeDB uses [cert-manager](https://cert-manager.io) to issue certificates. Two certificate aliases are supported: + +- `server`: presented by every server on the client, route, gateway, leafnode and monitor listeners. +- `client`: used by KubeDB (health checker, ops-manager) and by applications to connect. + +> Note: The yaml files used in this tutorial are stored in [docs/examples/nats/tls](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/nats/tls) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Before You Begin + +- You need a Kubernetes cluster with `kubectl` configured. If you do not have one, create it with [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). +- Install KubeDB following the steps [here](/docs/setup/README.md), with `--set global.featureGates.NATS=true` so the NATS CRDs are installed. +- This tutorial uses the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +Install cert-manager [v1.0.0 or later](https://cert-manager.io/docs/installation/). + +## Create an Issuer + +```bash +$ openssl req -newkey rsa:2048 -keyout ca.key -nodes -x509 -days 365 -out ca.crt -subj "/CN=nats/O=kubedb" +$ kubectl create secret tls nats-ca -n demo --cert=ca.crt --key=ca.key +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/tls/issuer.yaml +issuer.cert-manager.io/nats-ca-issuer created +``` + +## Create a NATS cluster with TLS + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: NATS +metadata: + name: nats-tls + namespace: demo +spec: + version: "2.15.1" + replicas: 3 + tls: + issuerRef: + apiGroup: cert-manager.io + kind: Issuer + name: nats-ca-issuer + certificates: + - alias: server + subject: + organizations: + - kubedb + - alias: client + subject: + organizations: + - kubedb + deletionPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/nats/tls/nats-tls.yaml +``` + +When `spec.tls` is set, `issuerRef` is required. TLS is on by default for every listener. Switch one off with `spec.clientTLS`, `clusterTLS`, `gatewayTLS`, `leafNodeTLS` or `monitorTLS` set to `false`. + +```bash +$ kubectl get secrets -n demo | grep nats-tls +nats-tls-server-cert kubernetes.io/tls 3 +nats-tls-client-cert kubernetes.io/tls 3 +``` + +Connect with the CA and client certificate: + +```bash +$ kubectl get secret -n demo nats-tls-client-cert -o jsonpath='{.data.ca\.crt}' | base64 -d > ca.crt +$ nats --server tls://127.0.0.1:4222 --tlsca ca.crt --user admin --password "$PASS" server check connection +``` + +## Day-2 changes + +Use `ReconfigureTLS` ([ops requests](/docs/guides/nats/ops-requests.md#reconfiguretls)): + +- **Rotate certificates**: zero downtime. The reloader signals the server, which re-reads the certificate files; new handshakes use the new certificate and existing connections keep their session. +- **Change issuer or CA**: the new CA is added to the CA bundle next to the old one, certificates are rotated, then the old CA is dropped. No restart. +- **Add or remove TLS on the client port**: two zero-downtime rolls. The first accepts both TLS and plaintext clients (`allow_non_tls`). KubeDB then waits for plaintext clients to disappear, or for the request `timeout`. The second roll removes the plaintext allowance. Move your applications to TLS between the two. + +## Cleanup + +```bash +$ kubectl delete -n demo nats/nats-tls issuer/nats-ca-issuer secret/nats-ca +```