Fix billing adapters and add a reproducible GKE/Stripe example

.github/workflows/ci.yml

@@ -26,3 +26,16 @@ jobs:
       - uses: azure/setup-helm@v4
       - run: helm lint deploy/helm/vbilling
       - run: helm template vbilling deploy/helm/vbilling --set adapters='{stripe}' --set stripe.existingSecret=vbilling-stripe > /dev/null
+
+  gke-stripe:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check for accidentally committed demo credentials
+        run: python3 scripts/check-demo-secrets.py --tracked
+      - name: Test the secret check with synthetic fixtures
+        run: python3 scripts/check-demo-secrets.py --self-test
+      - name: Test demo helpers without cloud credentials
+        run: python3 -m unittest discover -s examples/gke-stripe/tests -v
+      - name: Check local launcher syntax
+        run: bash -n examples/gke-stripe/scripts/run-local.sh

.gitignore

@@ -20,3 +20,24 @@ stripe-webhook-secret
 metronome-token
 lago-api-key
 webhook-secret
+
+# Local demo credentials and generated runtime state (never commit)
+.env.*
+!.env.example
+*.kubeconfig
+*.kubeconfig.*
+kubeconfig
+api-token
+private-tenants.json
+/examples/gke-gpu/join.txt
+/examples/gke-gpu/startup.sh
+/examples/gke-gpu/agent.yaml
+/examples/gke-stripe/.runtime/
+
+# Demo helper bytecode and CLI authentication files
+__pycache__/
+*.py[cod]
+vcluster-cli.json
+platform-cli.json
+private-join.txt
+private-startup.sh

README.md

@@ -19,6 +19,8 @@ vBilling is the **metering pipe**, not the billing engine. It turns what runs in
 
 > vBilling lives in `vClusterLabs-Experiments`: it is an experimental project, not a supported vCluster Labs product.
 
+**[GPU metering with Stripe](examples/gke-stripe/README.md)** provides a step-by-step setup for shared L4 GPUs, optional private A100 nodes, usage delivery to Stripe test mode, and resource cleanup. Credentials and runtime data stay outside the repository.
+
 ## What's new in v0.2
 
 - **Stripe and Metronome adapters.** Use Stripe Billing Meters on their own, or Metronome for rating with Stripe for payment, invoicing and tax. vBilling creates the Stripe customer and links it in Metronome.
@@ -290,7 +292,7 @@ Override any mapping with `ENFORCEMENT_RULES`, for example `alerts.spend_thresho
 - **`annotate`**: also annotate the tenant cluster, label its namespace `vbilling.vcluster.com/billing-state`, and emit a Kubernetes event.
 - **`enforce`**: also label suspended namespaces `vbilling.vcluster.com/suspended=true`. With `enforcement.admissionPolicy.enabled=true` (Kubernetes 1.30+), a ValidatingAdmissionPolicy then denies **new** pods in those namespaces. Running workloads keep running, the tenant sees the reason as a sync error inside their tenant cluster, and a paid invoice lifts it automatically. The policy acts in the control plane cluster, so it holds tenant clusters on shared or dedicated nodes; tenant clusters with their own nodes (Private Nodes, Auto Nodes, Standalone) run their pods only inside the tenant cluster, so act on their billing state with your own automation.
 
-Payment failures on invoices that Stripe collects arrive through Stripe's webhooks, so in Metronome + Stripe mode, send Stripe's webhooks to vBilling as well. Billing states are enabled once `STRIPE_WEBHOOK_SECRET` or `METRONOME_WEBHOOK_SECRET` is set.
+Payment failures on invoices that Stripe collects arrive through Stripe's webhooks, so in Metronome + Stripe mode, send Stripe's webhooks to vBilling as well. Billing states are always available: Stripe and Metronome webhooks set them once `STRIPE_WEBHOOK_SECRET` or `METRONOME_WEBHOOK_SECRET` is set, and `PUT /api/v1/billing-states/{tenant}` sets them from anything else, such as Lago or a custom billing platform.
 
 ## Custom usage: tokens, Slurm, storage, network
 

cmd/vbilling/main.go

@@ -198,11 +198,11 @@ func main() {
 	log.Println("vBilling stopped")
 }
 
-// newEnforcer enables billing-state enforcement when a webhook secret is set.
+// newEnforcer sets up billing states. They are always available: Stripe and
+// Metronome webhooks set them once their secrets are configured, and
+// PUT /api/v1/billing-states/{tenant} sets them from anywhere else, such as
+// Lago or a custom billing platform.
 func newEnforcer(cfg *config.Config, kube kubernetes.Interface, ctrl *controller.Controller, dests []destinations.Destination, tel *telemetry.Registry) *enforcement.Enforcer {
-	if cfg.StripeWebhookSecret == "" && cfg.MetronomeWebhookSecret == "" {
-		return nil
-	}
 	rules, err := enforcement.ParseRules(cfg.EnforcementRules)
 	if err != nil {
 		log.Fatalf("ENFORCEMENT_RULES: %v", err)
@@ -225,7 +225,8 @@ func newEnforcer(cfg *config.Config, kube kubernetes.Interface, ctrl *controller
 	if err != nil {
 		log.Fatalf("open billing state: %v", err)
 	}
-	log.Printf("Billing-state enforcement enabled (mode=%s)", cfg.EnforcementMode)
+	log.Printf("Billing states: mode=%s, Stripe webhooks %s, Metronome webhooks %s", cfg.EnforcementMode,
+		onOff(cfg.StripeWebhookSecret != ""), onOff(cfg.MetronomeWebhookSecret != ""))
 	return enforcement.New(kube, enforcement.Options{
 		Mode:                   cfg.EnforcementMode,
 		StripeWebhookSecret:    cfg.StripeWebhookSecret,
@@ -269,3 +270,10 @@ func getKubeConfig() (*rest.Config, error) {
 	log.Printf("Using kubeconfig (%s)", os.Getenv("KUBECONFIG"))
 	return cfg, nil
 }
+
+func onOff(b bool) string {
+	if b {
+		return "on"
+	}
+	return "off"
+}

docs/custom-adapter.html

@@ -378,7 +378,7 @@ <h3 id="responses">Responses, retries and dead letters</h3>
       </div>
       <div class="callout callout-warn">
         <div class="callout-icon">!</div>
-        <div><strong>Deduplicate on <code>id</code>.</strong> A batch can arrive twice: after a timeout, a restart before the cursor was saved, a dead-letter replay or a cursor rewind. A usage event keeps its ID for the same window, so upsert on it. Tenant and catalog IDs carry a timestamp: keep the latest. Return 2xx for types you ignore, because vBilling holds all delivery to the endpoint until it accepts the catalog event.</div>
+        <div><strong>Deduplicate on <code>id</code>.</strong> A batch can arrive twice: after a timeout, a restart before the cursor was saved, a dead-letter replay or a cursor rewind. A usage event keeps its ID for the same window, so upsert on it. Tenant and catalog IDs carry a timestamp: keep the latest. Return 2xx for types you ignore. A catalog event your endpoint rejects with 400, 413 or 422 is logged and skipped; any other failure holds delivery to the endpoint until the catalog event is accepted.</div>
       </div>
       <p>Dead letters stay on the ledger volume until a replay clears them. Add <code>-H "Authorization: Bearer $API_TOKEN"</code> when the API is protected:</p>
       <div class="code-block"><code><span class="cmd">curl</span> -s localhost:8080/api/v1/destinations/webhook/dead-letters \
@@ -436,7 +436,7 @@ <h2 id="go-adapter">Go adapter</h2>
           <tbody>
             <tr><td><code>EventFilter</code></td><td><code>Accepts(metric string) bool</code></td><td>Skip metrics the platform should not see. <code>destinations.BillableOnly</code> drops the informational ones. Skipped events are counted, never retried.</td></tr>
             <tr><td><code>Reconciler</code></td><td><code>RecordedTotals(ctx context.Context, q TotalsQuery) ([]Total, error)</code></td><td>Report what the platform recorded per tenant and metric, so <code>GET /api/v1/reconcile</code> can compare it with the ledger.</td></tr>
-            <tr><td><code>CustomerResolver</code></td><td><code>TenantForCustomer(ctx context.Context, customerID string) (string, bool)</code></td><td>Map a platform customer ID back to a tenant for inbound billing webhooks. Only the Stripe and Metronome adapters' webhooks (<code>/webhooks/stripe</code>, <code>/webhooks/metronome</code>) consult it today.</td></tr>
+            <tr><td><code>CustomerResolver</code></td><td><code>TenantForCustomer(ctx context.Context, customerID string) (string, bool)</code></td><td>Map a platform customer ID back to a tenant for inbound billing webhooks. Only the Stripe and Metronome adapters' webhooks (<code>/webhooks/stripe</code>, <code>/webhooks/metronome</code>) consult it today; any other billing platform sets billing states with <code>PUT /api/v1/billing-states/{tenant}</code>.</td></tr>
           </tbody>
         </table>
       </div>

docs/example-gke.html

@@ -277,6 +277,8 @@
       <h1>Example: GKE with shared and private GPU nodes</h1>
       <p class="subtitle">Both vCluster tenancy models on GPUs, metered by vBilling and invoiced by Stripe test mode, from an empty Google Cloud project. The manifests, values files and commands are the ones from a recorded run, with every key, token and account ID left out.</p>
 
+      <p><strong>For a setup without Prometheus, see <a href="https://github.com/vClusterLabs-Experiments/vbilling/tree/main/examples/gke-stripe">GPU metering with Stripe</a>.</strong> It uses two L4 sharing slots, local vBilling, and paused Stripe test subscriptions. This example uses four slots, Prometheus/DCGM, and automatic subscriptions.</p>
+
       <h2 id="what">What you build</h2>
       <ul>
         <li><strong>GKE</strong> as the control plane cluster, with a spot L4 node pool time-shared four ways.</li>
@@ -330,27 +332,36 @@ <h2 id="prometheus">2. Prometheus</h2>
 <span class="cmd">INGEST_IP=</span>$(kubectl -n monitoring get svc prometheus-ingest -o jsonpath='{.status.loadBalancer.ingress[0].ip}')</code></div>
 
       <h2 id="tenants">3. Tenant clusters</h2>
-      <div class="code-block"><code><span class="cmd">vcluster</span> platform login https://&lt;your-platform-host&gt;   <span class="comment">  # browser login, no key on the command line</span>
+      <div class="code-block"><code><span class="cmd">vcluster</span> platform login https://your-platform.example   <span class="comment">  # browser login, no key on the command line</span>
 <span class="cmd">vcluster</span> create shared-l4 -n shared-l4 -f 02-tenant-shared.yaml --connect=false
 <span class="cmd">vcluster</span> create private-a100 -n private-a100 -f 03-tenant-private.yaml --connect=false</code></div>
       <p>When the CLI is logged in, <code>vcluster create</code> registers each tenant cluster with the Platform, which is what licenses Private Nodes. A tenant cluster created before logging in is registered with <code>vcluster platform add vcluster private-a100 -n private-a100 --project default</code>. The private tenant cluster's <code>exportKubeConfig.additionalSecrets</code> writes the read-only Secret <code>vbilling-kubeconfig</code> that vBilling reads (see <a href="private-nodes.html#private">Private Nodes</a>).</p>
-      <p>Tell vBilling who the tenants are (the emails use the reserved <code>.example</code> domain):</p>
+      <p>Optionally set stable customer IDs and display names. Without these annotations, vBilling derives an identity per tenant cluster. The emails below use the reserved <code>.example</code> domain:</p>
       <div class="code-block"><code><span class="cmd">kubectl</span> annotate ns shared-l4 vbilling.vcluster.com/tenant=acme-ai "vbilling.vcluster.com/display-name=Acme AI" \
   vbilling.vcluster.com/email=billing@acme-ai.example vbilling.vcluster.com/plan=gpu-cloud
 <span class="cmd">kubectl</span> annotate ns private-a100 vbilling.vcluster.com/tenant=initech "vbilling.vcluster.com/display-name=Initech" \
   vbilling.vcluster.com/email=billing@initech.example vbilling.vcluster.com/plan=gpu-cloud</code></div>
 
       <h2 id="node">4. A private GPU node</h2>
       <p>Create a join token, put the join command it prints into the startup script, and boot the machine. The VM must reach the tenant cluster's internal load balancer, so keep it in the same VPC and region.</p>
-      <div class="code-block"><code><span class="cmd">vcluster</span> connect private-a100 -n private-a100 -- vcluster token create --expires 2h | grep node/join &gt; join.txt
-<span class="cmd">awk</span> -v cmd="$(cat join.txt)" '$0 == "JOIN_COMMAND" { print cmd; next } { print }' 04-node-startup.sh &gt; startup.sh
-<span class="cmd">gcloud</span> compute instances create gpu-node-1 --zone $ZONE --machine-type a2-highgpu-1g \
+      <div class="code-block"><code>set -euo pipefail
+set +x  # do not trace commands while handling credentials
+umask 077
+JOIN_DIR=$(mktemp -d "/tmp/vbilling-private-node.XXXXXX")
+chmod 700 "$JOIN_DIR"
+vcluster connect private-a100 -n private-a100 -- vcluster token create --expires 2h | grep node/join &gt; "$JOIN_DIR/join.txt"
+# awk reads the token from the file; the token is never a process argument.
+awk 'FNR == NR { cmd = $0; next } $0 == "JOIN_COMMAND" { print cmd; next } { print }' \
+  "$JOIN_DIR/join.txt" 04-node-startup.sh &gt; "$JOIN_DIR/startup.sh"
+gcloud compute instances create gpu-node-1 --zone $ZONE --machine-type a2-highgpu-1g \
   --provisioning-model SPOT --instance-termination-action STOP --maintenance-policy TERMINATE \
   --image-family ubuntu-2204-lts --image-project ubuntu-os-cloud --boot-disk-size 150GB \
-  --no-service-account --no-scopes --metadata-from-file startup-script=startup.sh</code></div>
+  --no-service-account --no-scopes --metadata-from-file "startup-script=$JOIN_DIR/startup.sh"</code></div>
       <p>Once <code>gpu-node-1</code> is Ready in the tenant cluster, remove the token from the instance and from disk, and label the node so it gets an SKU:</p>
       <div class="code-block"><code><span class="cmd">gcloud</span> compute instances remove-metadata gpu-node-1 --zone $ZONE --keys startup-script
-<span class="cmd">rm</span> join.txt startup.sh
+<span class="cmd">rm</span> -f "$JOIN_DIR/join.txt" "$JOIN_DIR/startup.sh"
+<span class="cmd">rmdir</span> "$JOIN_DIR"
+unset JOIN_DIR
 <span class="cmd">vcluster</span> connect private-a100 -n private-a100 -- kubectl label node gpu-node-1 \
   node.kubernetes.io/instance-type=a2-highgpu-1g vbilling.vcluster.com/capacity-type=spot</code></div>
 
@@ -372,14 +383,25 @@ <h2 id="vbilling">6. vBilling</h2>
 <span class="cmd">helm</span> install vbilling ../../deploy/helm/vbilling -n vbilling -f 07-vbilling-values.yaml
 <span class="cmd">vcluster</span> connect shared-l4 -n shared-l4 -- kubectl apply -f workloads/shared-inference.yaml
 <span class="cmd">vcluster</span> connect private-a100 -n private-a100 -- kubectl apply -f workloads/private-training.yaml</code></div>
-      <p>vBilling creates the Stripe customers and meters on the first windows. Give the meters prices tagged with the plan, then restart vBilling so it subscribes the tenants:</p>
-      <div class="code-block"><code><span class="cmd">export</span> STRIPE_API_KEY="$(cat ~/.stripe-test-key)"
-<span class="cmd">curl</span> -s https://api.stripe.com/v1/billing/meters -u "$STRIPE_API_KEY:" | grep -E '"(id|event_name)"'
-<span class="cmd">curl</span> -s https://api.stripe.com/v1/prices -u "$STRIPE_API_KEY:" \
+      <p>vBilling creates the Stripe customers and meters on the first windows. The operator creates the Product and Price. This example enables automatic subscription creation; prices must be tagged with the plan. The following creates a Product and Price together, so run it once for each intended meter and reuse the existing Price on retries. Then restart vBilling so it subscribes the tenants. For paused test subscriptions and repeatable setup helpers, use <a href="https://github.com/vClusterLabs-Experiments/vbilling/tree/main/examples/gke-stripe">the GPU example</a>.</p>
+      <div class="code-block"><code>set +x  # keep the key out of shell trace output
+stripe_test() {
+  local key
+  key=$(cat "$HOME/.stripe-test-key") || return 1
+  if [[ ! "$key" =~ ^sk_test_[A-Za-z0-9]+$ ]]; then
+    printf '%s\n' 'Expected a Stripe test secret key.' &gt;&amp;2
+    return 1
+  fi
+  # printf is a shell builtin: only curl's stdin receives the credential.
+  printf 'user = "%s:"\n' "$key" | curl --fail-with-body --silent --show-error --config - "$@"
+}
+stripe_test https://api.stripe.com/v1/billing/meters | grep -E '"(id|event_name)"'
+stripe_test https://api.stripe.com/v1/prices \
   -d currency=usd -d unit_amount_decimal=220 \
   -d "recurring[interval]=month" -d "recurring[usage_type]=metered" -d "recurring[meter]=&lt;meter id&gt;" \
   -d "product_data[name]=A100 40GB GPU-hour" -d "metadata[vbilling_plan]=gpu-cloud"
-<span class="cmd">kubectl</span> -n vbilling rollout restart statefulset/vbilling</code></div>
+unset -f stripe_test
+kubectl -n vbilling rollout restart statefulset/vbilling</code></div>
 
       <h2 id="check">7. Check it</h2>
       <div class="code-block"><code><span class="cmd">kubectl</span> -n vbilling port-forward svc/vbilling 8080:8080 &amp;

docs/lago.html

@@ -362,7 +362,7 @@ <h2 id="offboarding">Offboarding</h2>
       <h2 id="limits">Good to know</h2>
       <ul>
         <li><code>GET /api/v1/reconcile</code> covers Stripe and Metronome. For Lago, compare the ledger (<code>GET /api/v1/usage</code>) with Lago's current usage for the subscription.</li>
-        <li>Billing states and enforcement come from Stripe and Metronome webhooks, and are only enabled when one of their webhook secrets is set. A Lago-only setup has no billing states.</li>
+        <li>Lago sends no billing signals to vBilling. Set billing states from your own automation with <code>PUT /api/v1/billing-states/{tenant}</code>, for example <code>{"state": "suspended", "reason": "invoice overdue"}</code>.</li>
         <li>Lago also receives <code>vcluster_gpu_utilization</code>, an informational metric that v0.1 created in Lago and existing plans may still reference. Leave it unpriced.</li>
       </ul>