From 6b7c14e6325048d737af47eb5aa1b8b493d70a90 Mon Sep 17 00:00:00 2001 From: Saiyam Pathak Date: Mon, 5 Oct 2026 23:42:27 +0530 Subject: [PATCH 1/4] Meter every tenancy model and deliver usage through a durable ledger vBilling v0.2 core: - Hash-chained ledger with atomic window commits, per-destination cursors, dead letters with replay, and ingest deduplication. - Delivery to Stripe Billing Meters, Metronome, Lago and a signed CloudEvents webhook, with deterministic event IDs. - Metering for shared nodes, dedicated nodes, and tenant clusters with their own nodes (vCluster Private Nodes, Auto Nodes, Standalone), read through a read-only tenant kubeconfig. MIG, time-slicing, GKE partitions and DRA; deletion watches; fair metering of downtime. - Prometheus: DCGM utilization including MIG instances, PromQL-defined billable metrics, auth headers, and a vCluster Platform preset. - Reconciliation, billing-state enforcement, ingest API and dashboard. Fixes from a run on GKE with L4 and A100 GPUs and a Stripe sandbox: - Find Stripe customers by listing them where search is unavailable. - Count physical GPUs once under MIG and fall back to engine activity. - Record GPUs that are installed but not allocatable as downtime. - Add the {{window_seconds}} template and trim secrets read from files. --- cmd/vbilling/main.go | 263 +++- .../helm/vbilling/templates/deployment.yaml | 82 -- go.mod | 3 + go.sum | 4 + internal/api/dashboard/index.html | 411 ++++++ internal/api/server.go | 838 +++++++++++ internal/api/server_test.go | 338 +++++ internal/config/config.go | 292 +++- internal/config/config_test.go | 66 + internal/controller/controller.go | 572 +++++--- internal/controller/controller_test.go | 331 +++++ internal/destinations/destination.go | 166 ++- internal/destinations/destination_test.go | 48 +- internal/destinations/httpx.go | 44 + internal/destinations/lago/adapter.go | 139 +- internal/destinations/lago/adapter_test.go | 73 + internal/destinations/lago/bootstrap.go | 84 +- internal/destinations/lago/client.go | 29 +- internal/destinations/metronome/adapter.go | 421 ++++++ .../destinations/metronome/adapter_test.go | 468 ++++++ internal/destinations/metronome/client.go | 304 ++++ .../destinations/metronome/schema_test.go | 175 +++ .../testdata/openapi-request-schemas.json | 1 + internal/destinations/metronome/webhook.go | 56 + internal/destinations/noop/adapter.go | 19 +- internal/destinations/sentcache.go | 41 + internal/destinations/stripe/adapter.go | 593 ++++++++ internal/destinations/stripe/adapter_test.go | 414 ++++++ internal/destinations/stripe/client.go | 484 ++++++ internal/destinations/stripe/fake_test.go | 378 +++++ internal/destinations/stripe/webhook.go | 71 + internal/destinations/webhook/adapter.go | 161 ++ internal/destinations/webhook/adapter_test.go | 81 + internal/discovery/discovery.go | 390 ++--- internal/discovery/discovery_test.go | 96 ++ internal/enforcement/enforcer.go | 378 +++++ internal/enforcement/enforcer_test.go | 183 +++ internal/kv/kv.go | 98 ++ internal/metrics/collector.go | 1299 ++++++++--------- internal/metrics/collector_test.go | 585 ++++++++ internal/metrics/dra.go | 185 +++ internal/metrics/dra_test.go | 81 + internal/metrics/graveyard.go | 193 +++ internal/metrics/nodes.go | 334 +++++ internal/metrics/pods.go | 150 ++ internal/metrics/prometheus.go | 269 ++++ internal/metrics/prometheus_test.go | 88 ++ internal/metrics/tenant.go | 171 +++ internal/metrics/tenant_test.go | 259 ++++ internal/pipeline/dispatcher.go | 330 +++++ internal/pipeline/dispatcher_test.go | 316 ++++ internal/pipeline/tenants.go | 86 ++ internal/spool/bench_test.go | 45 + internal/spool/spool.go | 1145 +++++++++++++++ internal/spool/spool_test.go | 409 ++++++ internal/telemetry/telemetry.go | 131 ++ internal/tenantapi/tenantapi.go | 274 ++++ internal/tenantapi/tenantapi_test.go | 139 ++ internal/usage/catalog.go | 178 +++ internal/usage/schema_test.go | 66 + internal/usage/usage.go | 212 +++ internal/usage/usage_test.go | 163 +++ 62 files changed, 14264 insertions(+), 1439 deletions(-) delete mode 100644 deploy/helm/vbilling/templates/deployment.yaml create mode 100644 internal/api/dashboard/index.html create mode 100644 internal/api/server.go create mode 100644 internal/api/server_test.go create mode 100644 internal/config/config_test.go create mode 100644 internal/controller/controller_test.go create mode 100644 internal/destinations/httpx.go create mode 100644 internal/destinations/lago/adapter_test.go create mode 100644 internal/destinations/metronome/adapter.go create mode 100644 internal/destinations/metronome/adapter_test.go create mode 100644 internal/destinations/metronome/client.go create mode 100644 internal/destinations/metronome/schema_test.go create mode 100644 internal/destinations/metronome/testdata/openapi-request-schemas.json create mode 100644 internal/destinations/metronome/webhook.go create mode 100644 internal/destinations/sentcache.go create mode 100644 internal/destinations/stripe/adapter.go create mode 100644 internal/destinations/stripe/adapter_test.go create mode 100644 internal/destinations/stripe/client.go create mode 100644 internal/destinations/stripe/fake_test.go create mode 100644 internal/destinations/stripe/webhook.go create mode 100644 internal/destinations/webhook/adapter.go create mode 100644 internal/destinations/webhook/adapter_test.go create mode 100644 internal/discovery/discovery_test.go create mode 100644 internal/enforcement/enforcer.go create mode 100644 internal/enforcement/enforcer_test.go create mode 100644 internal/kv/kv.go create mode 100644 internal/metrics/collector_test.go create mode 100644 internal/metrics/dra.go create mode 100644 internal/metrics/dra_test.go create mode 100644 internal/metrics/graveyard.go create mode 100644 internal/metrics/nodes.go create mode 100644 internal/metrics/pods.go create mode 100644 internal/metrics/prometheus.go create mode 100644 internal/metrics/prometheus_test.go create mode 100644 internal/metrics/tenant.go create mode 100644 internal/metrics/tenant_test.go create mode 100644 internal/pipeline/dispatcher.go create mode 100644 internal/pipeline/dispatcher_test.go create mode 100644 internal/pipeline/tenants.go create mode 100644 internal/spool/bench_test.go create mode 100644 internal/spool/spool.go create mode 100644 internal/spool/spool_test.go create mode 100644 internal/telemetry/telemetry.go create mode 100644 internal/tenantapi/tenantapi.go create mode 100644 internal/tenantapi/tenantapi_test.go create mode 100644 internal/usage/catalog.go create mode 100644 internal/usage/schema_test.go create mode 100644 internal/usage/usage.go create mode 100644 internal/usage/usage_test.go diff --git a/cmd/vbilling/main.go b/cmd/vbilling/main.go index 18b8618..bb1b1cc 100644 --- a/cmd/vbilling/main.go +++ b/cmd/vbilling/main.go @@ -2,111 +2,270 @@ package main import ( "context" + "errors" "log" + "net/http" "os" "os/signal" + "path/filepath" + "sync" "syscall" + "time" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + metricsclient "k8s.io/metrics/pkg/client/clientset/versioned" + + "github.com/vclusterlabs-experiments/vbilling/internal/api" "github.com/vclusterlabs-experiments/vbilling/internal/config" "github.com/vclusterlabs-experiments/vbilling/internal/controller" "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/metronome" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/enforcement" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" "github.com/vclusterlabs-experiments/vbilling/internal/metrics" + "github.com/vclusterlabs-experiments/vbilling/internal/pipeline" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/telemetry" + "github.com/vclusterlabs-experiments/vbilling/internal/tenantapi" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" - // Register built-in adapters via blank import so their init() funcs run. + // Register the remaining built-in adapters. _ "github.com/vclusterlabs-experiments/vbilling/internal/destinations/lago" _ "github.com/vclusterlabs-experiments/vbilling/internal/destinations/noop" - - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - metricsclient "k8s.io/metrics/pkg/client/clientset/versioned" + _ "github.com/vclusterlabs-experiments/vbilling/internal/destinations/webhook" ) +var version = "0.2.0" + func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) - log.Println("vBilling - vCluster Billing Controller") - log.Println("=======================================") + log.Printf("vBilling %s: usage metering for tenant clusters", version) cfg := config.Load() - log.Printf("Adapter: %s", cfg.Adapter) - log.Printf("Plan: %s | Currency: %s", cfg.DefaultPlanCode, cfg.BillingCurrency) - log.Printf("Collection: %s | Reconcile: %s", cfg.CollectionInterval, cfg.ReconcileInterval) + if err := cfg.Validate(); err != nil { + log.Fatalf("invalid configuration: %v", err) + } + if cfg.SpotDiscountPercent > 0 { + log.Printf("WARNING: SPOT_DISCOUNT_PERCENT is deprecated and ignored. Quantities are no longer discounted; " + + "price spot and preemptible capacity in your billing backend using the capacity_type dimension.") + } + customs, err := usage.ParseCustomMetrics(cfg.CustomMetrics) + if err != nil { + log.Fatalf("CUSTOM_METRICS: %v", err) + } + for _, d := range customs { + if err := usage.RegisterCustom(d); err != nil { + log.Fatalf("CUSTOM_METRICS: %v", err) + } + } + promMetrics, err := metrics.LoadPromMetrics(cfg.PrometheusMetrics) + if err != nil { + log.Fatalf("PROMETHEUS_METRICS_FILE: %v", err) + } + log.Printf("Adapters: %v | Region: %s | Window: %s | CPU/memory basis: %s | Data: %s", + cfg.Adapters, cfg.Region, cfg.CollectionInterval, cfg.CPUMemoryBasis, cfg.DataDir) - dest, err := destinations.New(cfg.Adapter, cfg) + sp, err := spool.Open(filepath.Join(cfg.DataDir, "ledger"), spool.Options{Retention: cfg.Retention}) if err != nil { - log.Fatalf("Failed to initialize billing adapter: %v", err) + log.Fatalf("open ledger: %v", err) + } + dests, err := destinations.NewAll(cfg.Adapters, cfg) + if err != nil { + log.Fatalf("initialize adapters: %v", err) } kubeConfig, err := getKubeConfig() if err != nil { - log.Fatalf("Failed to get Kubernetes config: %v", err) + log.Fatalf("kubernetes config: %v", err) } - - kubeClient, err := kubernetes.NewForConfig(kubeConfig) + kube, err := kubernetes.NewForConfig(kubeConfig) if err != nil { - log.Fatalf("Failed to create Kubernetes client: %v", err) + log.Fatalf("kubernetes client: %v", err) } - metricsClient, err := metricsclient.NewForConfig(kubeConfig) if err != nil { - log.Fatalf("Failed to create metrics client: %v", err) + log.Fatalf("metrics client: %v", err) } - dynamicClient, err := dynamic.NewForConfig(kubeConfig) if err != nil { - log.Fatalf("Failed to create dynamic client: %v", err) + log.Fatalf("dynamic client: %v", err) } - ctx, cancel := context.WithCancel(context.Background()) + tel := telemetry.Default + tenants := pipeline.NewTenantRegistry() + disp := pipeline.New(sp, dests, pipeline.Options{ + Tenants: tenants, + Bootstrap: func(ctx context.Context, d destinations.Destination) error { return d.Bootstrap(ctx, usage.Catalog()) }, + }, tel) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - log.Printf("Bootstrapping %s adapter...", dest.Name()) - if err := dest.Bootstrap(ctx); err != nil { - log.Printf("WARNING: %s bootstrap failed: %v", dest.Name(), err) - log.Println("The controller will continue but billing may not work correctly.") - } + disc := discovery.NewDiscoverer(kube, dynamicClient, cfg.WatchNamespaces, cfg.PlatformCluster) + coll := metrics.NewCollector(kube, metricsClient, metrics.Options{ + Region: cfg.Region, + RegionFromNode: cfg.RegionFromNode, + Basis: cfg.CPUMemoryBasis, + MeterControlPlane: cfg.MeterControlPlane, + MeterByNamespace: cfg.MeterByNamespace, + GPUResources: cfg.GPUResources, + SKULabel: cfg.SKULabel, + CapacityTypeLabel: cfg.CapacityTypeLabel, + UnhealthyTaints: cfg.UnhealthyTaints, + DedicatedNodeLabel: cfg.DedicatedNodeLabel, + PrometheusURL: cfg.PrometheusURL, + PromHeaders: cfg.PrometheusHeaders, + PromTokenFile: cfg.PromTokenFile, + EgressQuery: cfg.EgressQuery, + GPUUtilQuery: cfg.GPUUtilQuery, + GPUCountQuery: cfg.GPUCountQuery, + PrometheusPreset: cfg.PrometheusPreset, + PromMetrics: promMetrics, - disc := discovery.NewDiscoverer(kubeClient, dynamicClient, cfg.WatchNamespaces) - coll := metrics.NewCollector(kubeClient, metricsClient, cfg.PrometheusURL, cfg.SpotDiscountPercent) - ctrl := controller.New(cfg, dest, disc, coll) + PrivateNodeBilling: cfg.PrivateNodeBilling, + TenantExcludeNamespaces: cfg.TenantExcludeNamespaces, + DRAGPUDrivers: cfg.DRAGPUDrivers, + }) + coll.UseDynamic(dynamicClient) + // Deleted pods and nodes stay billable until their windows are committed. + retention := cfg.MaxBackfill + 2*cfg.CollectionInterval + if cfg.WatchDeletions { + g := metrics.NewGraveyard(retention, time.Now) + go g.Watch(ctx, kube, metrics.LabelManagedBy, true, true) + coll.UseGraveyard(g) + } + if cfg.TenantAPI { + ext, err := tenantapi.LoadExternal(cfg.TenantClustersFile) + if err != nil { + log.Fatalf("TENANT_CLUSTERS_FILE: %v", err) + } + mgr := tenantapi.New(ctx, kube, tenantapi.Options{ + SecretName: cfg.TenantKubeconfigSecret, + AdminFallback: cfg.TenantAdminFallback, + External: ext, + WatchPods: cfg.PrivateNodeBilling == "usage", + Retention: retention, + }) + coll.UseTenantAPI(mgr) + disc.WithExternal(mgr.Clusters()) + log.Printf("Tenant APIs: Secret %q per tenant cluster (admin fallback %t), %d external tenant cluster(s), private nodes billed by %s", + cfg.TenantKubeconfigSecret, cfg.TenantAdminFallback, len(ext), cfg.PrivateNodeBilling) + } + ctrl := controller.New(cfg, disc, coll, sp, tenants, dests, tel) + enf := newEnforcer(cfg, kube, ctrl, dests, tel) - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + srv := &http.Server{ + Addr: cfg.ListenAddr, + Handler: api.New(cfg, sp, disp, tenants, ctrl, enf, tel, version).Handler(), + ReadHeaderTimeout: 10 * time.Second, + } + if cfg.APIToken == "" { + log.Printf("WARNING: API_TOKEN is not set; usage and admin endpoints on %s are unauthenticated (keep the Service internal)", cfg.ListenAddr) + } + var wg sync.WaitGroup + wg.Add(3) go func() { - sig := <-sigCh - log.Printf("Received signal %s, shutting down...", sig) - cancel() + defer wg.Done() + log.Printf("API listening on %s", cfg.ListenAddr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("api server: %v", err) + } + }() + go func() { + defer wg.Done() + disp.Run(ctx) + }() + go func() { + defer wg.Done() + compactLoop(ctx, sp, cfg.Adapters) }() - log.Println("Starting billing controller...") - if err := ctrl.Run(ctx); err != nil && err != context.Canceled { - log.Fatalf("Controller error: %v", err) + if err := ctrl.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + log.Printf("controller: %v", err) + } + shutdownCtx, done := context.WithTimeout(context.Background(), 10*time.Second) + defer done() + srv.Shutdown(shutdownCtx) + wg.Wait() + if err := sp.Close(); err != nil { + log.Printf("close ledger: %v", err) } - log.Println("vBilling stopped") } -func getKubeConfig() (*rest.Config, error) { - cfg, err := rest.InClusterConfig() - if err == nil { - log.Println("Using in-cluster Kubernetes config") - return cfg, nil +// newEnforcer enables billing-state enforcement when a webhook secret is set. +func newEnforcer(cfg *config.Config, kube kubernetes.Interface, ctrl *controller.Controller, dests []destinations.Destination, tel *telemetry.Registry) *enforcement.Enforcer { + if cfg.StripeWebhookSecret == "" && cfg.MetronomeWebhookSecret == "" { + return nil } + rules, err := enforcement.ParseRules(cfg.EnforcementRules) + if err != nil { + log.Fatalf("ENFORCEMENT_RULES: %v", err) + } + resolvers := map[string]destinations.CustomerResolver{} + for _, d := range dests { + switch a := d.(type) { + case *stripe.Adapter: + resolvers["stripe"] = a + case *metronome.Adapter: + resolvers["metronome"] = a + if link := a.StripeLink(); link != nil { + if _, ok := resolvers["stripe"]; !ok { + resolvers["stripe"] = link // Stripe dunning for Metronome-rated tenants + } + } + } + } + state, err := kv.Open(filepath.Join(cfg.DataDir, "billing-state.json")) + if err != nil { + log.Fatalf("open billing state: %v", err) + } + log.Printf("Billing-state enforcement enabled (mode=%s)", cfg.EnforcementMode) + return enforcement.New(kube, enforcement.Options{ + Mode: cfg.EnforcementMode, + StripeWebhookSecret: cfg.StripeWebhookSecret, + MetronomeWebhookSecret: cfg.MetronomeWebhookSecret, + Rules: rules, + Clusters: ctrl.ClustersOf, + Resolvers: resolvers, + State: state, + Telemetry: tel, + }) +} - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - home, _ := os.UserHomeDir() - kubeconfig = home + "/.kube/config" +// compactLoop applies ledger retention hourly. +func compactLoop(ctx context.Context, sp *spool.Spool, consumers []string) { + t := time.NewTicker(time.Hour) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if n, err := sp.Compact(consumers); err != nil { + log.Printf("ledger compaction: %v", err) + } else if n > 0 { + log.Printf("ledger compaction removed %d segment(s) past retention", n) + } + } } +} - cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) +func getKubeConfig() (*rest.Config, error) { + if cfg, err := rest.InClusterConfig(); err == nil { + log.Println("Using in-cluster Kubernetes config") + return cfg, nil + } + rules := clientcmd.NewDefaultClientConfigLoadingRules() + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}).ClientConfig() if err != nil { return nil, err } - - log.Printf("Using kubeconfig: %s", kubeconfig) + log.Printf("Using kubeconfig (%s)", os.Getenv("KUBECONFIG")) return cfg, nil } diff --git a/deploy/helm/vbilling/templates/deployment.yaml b/deploy/helm/vbilling/templates/deployment.yaml deleted file mode 100644 index 40b529e..0000000 --- a/deploy/helm/vbilling/templates/deployment.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "vbilling.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "vbilling.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "vbilling.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "vbilling.selectorLabels" . | nindent 8 }} - spec: - serviceAccountName: {{ include "vbilling.serviceAccountName" . }} - containers: - - name: vbilling - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: ADAPTER - value: {{ .Values.adapter | default "lago" | quote }} - {{- if eq (.Values.adapter | default "lago") "lago" }} - - name: LAGO_API_URL - value: {{ .Values.lago.apiURL | quote }} - - name: LAGO_API_KEY - {{- if .Values.lago.existingSecret }} - valueFrom: - secretKeyRef: - name: {{ .Values.lago.existingSecret }} - key: api-key - {{- else }} - value: {{ .Values.lago.apiKey | quote }} - {{- end }} - {{- end }} - - name: DEFAULT_PLAN_CODE - value: {{ .Values.billing.defaultPlanCode | quote }} - - name: BILLING_CURRENCY - value: {{ .Values.billing.currency | quote }} - - name: COLLECTION_INTERVAL - value: {{ .Values.billing.collectionInterval | quote }} - - name: RECONCILE_INTERVAL - value: {{ .Values.billing.reconcileInterval | quote }} - - name: CPU_PRICE_PER_CORE_HOUR - value: {{ .Values.pricing.cpuPerCoreHour | quote }} - - name: MEMORY_PRICE_PER_GB_HOUR - value: {{ .Values.pricing.memoryPerGBHour | quote }} - - name: STORAGE_PRICE_PER_GB_HOUR - value: {{ .Values.pricing.storagePerGBHour | quote }} - - name: INSTANCE_PRICE_PER_HOUR - value: {{ .Values.pricing.instancePerHour | quote }} - - name: GPU_DEFAULT_PRICE_PER_HOUR - value: {{ .Values.pricing.gpuDefaultPerHour | quote }} - - name: GPU_PRICES - value: {{ .Values.pricing.gpuPrices | quote }} - - name: NETWORK_EGRESS_PRICE_PER_GB - value: {{ .Values.pricing.networkEgressPerGB | quote }} - - name: LB_PRICE_PER_HOUR - value: {{ .Values.pricing.lbPerHour | quote }} - - name: SPOT_DISCOUNT_PERCENT - value: {{ .Values.pricing.spotDiscountPercent | quote }} - {{- if .Values.prometheus.url }} - - name: PROMETHEUS_URL - value: {{ .Values.prometheus.url | quote }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/go.mod b/go.mod index bc73f81..063ab10 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -19,6 +20,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/imdario/mergo v0.3.6 // indirect @@ -28,6 +30,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect diff --git a/go.sum b/go.sum index 897ec86..04de81b 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -59,6 +61,8 @@ github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= diff --git a/internal/api/dashboard/index.html b/internal/api/dashboard/index.html new file mode 100644 index 0000000..66c0c68 --- /dev/null +++ b/internal/api/dashboard/index.html @@ -0,0 +1,411 @@ + + + + + +vBilling + + + + +
+
+ + vBilling +
+
Loading...
+
+
+ +
+ This vBilling API requires a token. + + +
+ +
+
Tenants
-
-
+
GPU-hours
-
last 24 hours
+
Destinations
-
-
+
Ledger
-
-
+
+ +
+
+

GPU-hours by SKU

+

GPU-equivalent hours; MIG and time-sliced GPUs count at their fraction.

+ +
+
+

GPU-hours per hour, last 24 hours

+

Hourly totals from the ledger (UTC).

+ +
+
+ +
+

Usage by tenant

+

Unrated quantities from the vBilling ledger. Prices and invoices come from your billing backend.

+
+
+ +
+
+

Destinations

+

Each destination reads the ledger through its own cursor.

+
DestinationStatusDeliveredLagDead letters
+
+
+

Billing state

+

From Stripe and Metronome webhooks.

+
TenantStateReasonSince
+
+
+
+
+ + + diff --git a/internal/api/server.go b/internal/api/server.go new file mode 100644 index 0000000..4d114c0 --- /dev/null +++ b/internal/api/server.go @@ -0,0 +1,838 @@ +// Package api serves vBilling's HTTP surface: health and Prometheus +// metrics, tenant-visible usage line items and exports, the ingest endpoint +// for custom usage emitters, reconciliation against billing backends, +// ledger verification, delivery status, and billing webhooks. +package api + +import ( + "context" + "crypto/subtle" + "embed" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log" + "math" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/enforcement" + "github.com/vclusterlabs-experiments/vbilling/internal/pipeline" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/telemetry" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +//go:embed dashboard +var dashboardFS embed.FS + +// Controller is the part of the controller the API reads. +type Controller interface { + Ready() bool + Clusters() []discovery.TenantCluster + LastWindow() time.Time +} + +// Server is vBilling's HTTP API. +type Server struct { + cfg *config.Config + spool *spool.Spool + disp *pipeline.Dispatcher + tenants *pipeline.TenantRegistry + ctrl Controller + enf *enforcement.Enforcer // nil when no webhook secret is configured + tel *telemetry.Registry + version string + now func() time.Time +} + +func New(cfg *config.Config, sp *spool.Spool, disp *pipeline.Dispatcher, tenants *pipeline.TenantRegistry, + ctrl Controller, enf *enforcement.Enforcer, tel *telemetry.Registry, version string) *Server { + return &Server{cfg: cfg, spool: sp, disp: disp, tenants: tenants, ctrl: ctrl, enf: enf, tel: tel, version: version, now: time.Now} +} + +// Handler returns the routed handler. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "ok\n") }) + mux.HandleFunc("GET /readyz", s.readyz) + mux.HandleFunc("GET /metrics", s.metrics) + + mux.HandleFunc("GET /api/v1/status", s.auth(s.status)) + mux.HandleFunc("GET /api/v1/catalog", s.auth(s.catalog)) + mux.HandleFunc("GET /api/v1/tenants", s.auth(s.listTenants)) + mux.HandleFunc("GET /api/v1/usage", s.auth(s.usage)) + mux.HandleFunc("GET /api/v1/events", s.auth(s.exportEvents)) + mux.HandleFunc("POST /api/v1/events", s.ingest) + mux.HandleFunc("GET /api/v1/reconcile", s.auth(s.reconcile)) + mux.HandleFunc("GET /api/v1/ledger/verify", s.auth(s.verify)) + mux.HandleFunc("GET /api/v1/destinations", s.auth(s.listDestinations)) + mux.HandleFunc("GET /api/v1/destinations/{name}/dead-letters", s.auth(s.deadLetters)) + mux.HandleFunc("POST /api/v1/destinations/{name}/dead-letters/replay", s.auth(s.replayDeadLetters)) + mux.HandleFunc("POST /api/v1/destinations/{name}/cursor", s.auth(s.setCursor)) + mux.HandleFunc("GET /api/v1/billing-states", s.auth(s.billingStates)) + mux.HandleFunc("PUT /api/v1/billing-states/{tenant}", s.auth(s.setBillingState)) + + mux.HandleFunc("POST /webhooks/stripe", s.stripeWebhook) + mux.HandleFunc("POST /webhooks/metronome", s.metronomeWebhook) + + sub, _ := fs.Sub(dashboardFS, "dashboard") + mux.Handle("GET /", http.FileServer(http.FS(sub))) + return mux +} + +// --- helpers --- + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.Encode(v) +} + +func writeErr(w http.ResponseWriter, status int, format string, args ...any) { + writeJSON(w, status, map[string]string{"error": fmt.Sprintf(format, args...)}) +} + +func bearer(r *http.Request) string { + return strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") +} + +func tokenOK(got, want string) bool { + return subtle.ConstantTimeCompare([]byte(got), []byte(want)) == 1 +} + +// auth protects read and admin endpoints with API_TOKEN when it is set. +func (s *Server) auth(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if s.cfg.APIToken != "" && !tokenOK(bearer(r), s.cfg.APIToken) { + writeErr(w, http.StatusUnauthorized, "missing or invalid bearer token") + return + } + h(w, r) + } +} + +// parseRange reads from/to (RFC 3339 or YYYY-MM-DD). Defaults: the +// current UTC month to now. +func (s *Server) parseRange(r *http.Request, defFrom, defTo time.Time) (time.Time, time.Time, error) { + parse := func(v string, def time.Time) (time.Time, error) { + if v == "" { + return def, nil + } + if t, err := time.Parse(time.RFC3339, v); err == nil { + return t.UTC(), nil + } + if t, err := time.Parse("2006-01-02", v); err == nil { + return t.UTC(), nil + } + return time.Time{}, fmt.Errorf("invalid time %q (use RFC 3339 or YYYY-MM-DD)", v) + } + from, err := parse(r.URL.Query().Get("from"), defFrom) + if err != nil { + return from, from, err + } + to, err := parse(r.URL.Query().Get("to"), defTo) + if err != nil { + return from, to, err + } + if !to.After(from) { + return from, to, errors.New("to must be after from") + } + return from, to, nil +} + +func monthStart(t time.Time) time.Time { + t = t.UTC() + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) +} + +// field returns an event attribute by name for grouping and filtering. +func field(e *usage.Event, key string) string { + switch key { + case "tenant": + return e.Tenant + case "metric": + return e.Metric + case "unit": + return e.Unit + case "sku": + return e.SKU + case "region": + return e.Region + case "project": + return e.Project + case "resource_id": + return e.ResourceID + case "source": + return e.Source + case "day": + return e.WindowStart.UTC().Format("2006-01-02") + case "hour": + return e.WindowStart.UTC().Format("2006-01-02T15:00Z") + default: + return e.Dim(key) + } +} + +// filter applies tenant/metric/project/region/sku/cluster query filters. +func filter(r *http.Request) func(*usage.Event) bool { + q := r.URL.Query() + checks := map[string]string{} + for _, k := range []string{"tenant", "metric", "project", "region", "sku"} { + if v := q.Get(k); v != "" { + checks[k] = v + } + } + if v := q.Get("cluster"); v != "" { + checks[usage.DimTenantCluster] = v + } + return func(e *usage.Event) bool { + for k, v := range checks { + if field(e, k) != v { + return false + } + } + return true + } +} + +// --- health & metrics --- + +func (s *Server) readyz(w http.ResponseWriter, r *http.Request) { + if s.ctrl != nil && !s.ctrl.Ready() { + http.Error(w, "waiting for first successful discovery", http.StatusServiceUnavailable) + return + } + io.WriteString(w, "ok\n") +} + +func (s *Server) metrics(w http.ResponseWriter, r *http.Request) { + st := s.spool.Stats() + s.tel.Set("vbilling_ledger_committed_seq", "Highest committed ledger sequence number.", nil, float64(st.Committed)) + s.tel.Set("vbilling_ledger_bytes", "Bytes of retained ledger segments.", nil, float64(st.Bytes)) + s.tel.Set("vbilling_ledger_segments", "Retained ledger segments.", nil, float64(st.Segments)) + if s.disp != nil { + for _, d := range s.disp.Statuses() { + l := map[string]string{"destination": d.Name} + s.tel.Set("vbilling_destination_lag_records", "Ledger records not yet acknowledged by a destination.", l, float64(d.LagRecords)) + ready := 0.0 + if d.Ready { + ready = 1 + } + s.tel.Set("vbilling_destination_ready", "1 once a destination is bootstrapped and receiving events.", l, ready) + } + } + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + s.tel.WriteText(w) +} + +// --- status, catalog, tenants --- + +func (s *Server) status(w http.ResponseWriter, r *http.Request) { + out := map[string]any{ + "version": s.version, + "region": s.cfg.Region, + "cluster": s.cfg.ClusterName, + "adapters": s.cfg.Adapters, + "window": s.cfg.CollectionInterval.String(), + "enforcement_mode": s.cfg.EnforcementMode, + "ledger": s.spool.Stats(), + "tenants": len(s.tenants.All()), + } + if s.ctrl != nil { + out["ready"] = s.ctrl.Ready() + out["tenant_clusters"] = len(s.ctrl.Clusters()) + out["last_window_end"] = s.ctrl.LastWindow() + } + if s.disp != nil { + out["destinations"] = s.disp.Statuses() + } + writeJSON(w, http.StatusOK, out) +} + +func (s *Server) catalog(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"schema_version": usage.SchemaVersion, "metrics": usage.Catalog()}) +} + +func (s *Server) listTenants(w http.ResponseWriter, r *http.Request) { + type row struct { + usage.Tenant + BillingState *enforcement.TenantState `json:"billing_state,omitempty"` + } + var out []row + for _, t := range s.tenants.All() { + rw := row{Tenant: t} + if s.enf != nil { + st := s.enf.StateOf(t.ID) + rw.BillingState = &st + } + out = append(out, rw) + } + writeJSON(w, http.StatusOK, map[string]any{"tenants": out}) +} + +// --- usage line items --- + +func (s *Server) usage(w http.ResponseWriter, r *http.Request) { + now := s.now().UTC() + from, to, err := s.parseRange(r, monthStart(now), now) + if err != nil { + writeErr(w, http.StatusBadRequest, "%v", err) + return + } + groupBy := []string{"tenant"} + if v := r.URL.Query().Get("group_by"); v != "" { + groupBy = nil + for _, k := range strings.Split(v, ",") { + if k = strings.TrimSpace(k); k != "" && k != "metric" && k != "unit" { + groupBy = append(groupBy, k) + } + } + } + keep := filter(r) + type agg struct { + group []string + metric string + unit string + quantity float64 + events int + } + rows := map[string]*agg{} + total := 0 + err = s.spool.Scan(from, to, func(e *usage.Event) error { + if e.WindowStart.Before(from) || !e.WindowStart.Before(to) || !keep(e) { + return nil + } + vals := make([]string, len(groupBy)) + for i, k := range groupBy { + vals[i] = field(e, k) + } + key := strings.Join(append(vals, e.Metric), "\x00") + a, ok := rows[key] + if !ok { + a = &agg{group: vals, metric: e.Metric, unit: e.Unit} + rows[key] = a + } + a.quantity += e.Quantity + a.events++ + total++ + return nil + }) + if err != nil { + writeErr(w, http.StatusInternalServerError, "scan ledger: %v", err) + return + } + keys := make([]string, 0, len(rows)) + for k := range rows { + keys = append(keys, k) + } + sort.Strings(keys) + + if r.URL.Query().Get("format") == "csv" { + w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="usage-%s-%s.csv"`, from.Format("20060102"), to.Format("20060102"))) + cw := csv.NewWriter(w) + cw.Write(append(append([]string{}, groupBy...), "metric", "unit", "quantity", "events")) + for _, k := range keys { + a := rows[k] + cw.Write(append(append([]string{}, a.group...), a.metric, a.unit, strconv.FormatFloat(usage.Round(a.quantity), 'f', -1, 64), strconv.Itoa(a.events))) + } + cw.Flush() + return + } + out := make([]map[string]any, 0, len(keys)) + for _, k := range keys { + a := rows[k] + row := map[string]any{"metric": a.metric, "unit": a.unit, "quantity": usage.Round(a.quantity), "events": a.events} + for i, g := range groupBy { + row[g] = a.group[i] + } + out = append(out, row) + } + writeJSON(w, http.StatusOK, map[string]any{"from": from, "to": to, "group_by": groupBy, "events": total, "rows": out, + "note": "Unrated usage from the vBilling ledger; prices and invoices come from your billing backend."}) +} + +// --- raw export --- + +func (s *Server) exportEvents(w http.ResponseWriter, r *http.Request) { + now := s.now().UTC() + from, to, err := s.parseRange(r, monthStart(now), now) + if err != nil { + writeErr(w, http.StatusBadRequest, "%v", err) + return + } + limit := math.MaxInt + if v := r.URL.Query().Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + limit = n + } + } + keep := filter(r) + format := r.URL.Query().Get("format") + var cw *csv.Writer + if format == "csv" { + w.Header().Set("Content-Type", "text/csv") + cw = csv.NewWriter(w) + cw.Write([]string{"id", "tenant", "metric", "quantity", "unit", "window_start", "window_end", "region", "project", "sku", "resource_id", "dimensions", "source"}) + } else { + w.Header().Set("Content-Type", "application/x-ndjson") + } + enc := json.NewEncoder(w) + n := 0 + errStop := errors.New("limit reached") + err = s.spool.Scan(from, to, func(e *usage.Event) error { + if e.WindowStart.Before(from) || !e.WindowStart.Before(to) || !keep(e) { + return nil + } + if n >= limit { + return errStop + } + n++ + if cw != nil { + dims := make([]string, 0, len(e.Dimensions)) + for k, v := range e.Dimensions { + dims = append(dims, k+"="+v) + } + sort.Strings(dims) + return cw.Write([]string{e.ID, e.Tenant, e.Metric, strconv.FormatFloat(e.Quantity, 'f', -1, 64), e.Unit, + e.WindowStart.Format(time.RFC3339), e.WindowEnd.Format(time.RFC3339), e.Region, e.Project, e.SKU, e.ResourceID, strings.Join(dims, ";"), e.Source}) + } + return enc.Encode(e) + }) + if cw != nil { + cw.Flush() + } + if err != nil && !errors.Is(err, errStop) { + log.Printf("[api] export: %v", err) + } +} + +// --- ingest --- + +const ( + maxIngestBody = 10 << 20 + maxIngestEvents = 5000 + ingestMaxAge = 34 * 24 * time.Hour // the strictest backend window (Metronome) + ingestMaxSkew = 5 * time.Minute +) + +// ingest accepts usage from custom emitters: inference gateways (tokens), +// Slurm accounting, storage or network exporters. Events need a declared +// metric (builtin or CUSTOM_METRICS) and a closed window; an omitted ID is +// derived from the event identity, so resends deduplicate. +func (s *Server) ingest(w http.ResponseWriter, r *http.Request) { + if s.cfg.IngestToken == "" { + writeErr(w, http.StatusForbidden, "ingest is disabled: set INGEST_TOKEN (or API_TOKEN) to enable it") + return + } + if !tokenOK(bearer(r), s.cfg.IngestToken) { + writeErr(w, http.StatusUnauthorized, "missing or invalid bearer token") + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, maxIngestBody+1)) + if err != nil || len(body) > maxIngestBody { + writeErr(w, http.StatusRequestEntityTooLarge, "body must be at most %d bytes", maxIngestBody) + return + } + var events []usage.Event + trimmed := strings.TrimSpace(string(body)) + if strings.HasPrefix(trimmed, "[") { + err = json.Unmarshal(body, &events) + } else { + var req struct { + Events []usage.Event `json:"events"` + } + err = json.Unmarshal(body, &req) + events = req.Events + } + if err != nil { + writeErr(w, http.StatusBadRequest, "invalid JSON: %v", err) + return + } + if len(events) == 0 || len(events) > maxIngestEvents { + writeErr(w, http.StatusBadRequest, "send 1 to %d events", maxIngestEvents) + return + } + client := r.Header.Get("X-VBilling-Client") + if client == "" { + client = "api" + } + source := "ingest:" + client + now := s.now().UTC() + type rejection struct { + Index int `json:"index"` + ID string `json:"id,omitempty"` + Error string `json:"error"` + } + var valid []usage.Event + var rejected []rejection + for i := range events { + e := events[i] + e.Source = source + e.RecordedAt = now + if e.Region == "" { + e.Region = s.cfg.Region + } + e.Finalize() + var problems []string + if _, ok := usage.Lookup(e.Metric); !ok { + problems = append(problems, fmt.Sprintf("unknown metric %q (declare it in CUSTOM_METRICS)", e.Metric)) + } + if e.WindowEnd.After(now.Add(ingestMaxSkew)) { + problems = append(problems, "window_end is in the future") + } + if e.WindowStart.Before(now.Add(-ingestMaxAge)) { + problems = append(problems, "window_start is older than 34 days") + } + if err := e.Validate(); err != nil { + problems = append(problems, err.Error()) + } + if len(problems) > 0 { + rejected = append(rejected, rejection{Index: i, ID: e.ID, Error: strings.Join(problems, "; ")}) + continue + } + valid = append(valid, e) + } + var accepted, dups []string + if len(valid) > 0 { + if accepted, dups, err = s.spool.AppendIngest(source, valid); err != nil { + writeErr(w, http.StatusServiceUnavailable, "ledger write failed: %v", err) + return + } + } + status := http.StatusOK + if len(valid) == 0 { + status = http.StatusBadRequest + } + s.tel.Add("vbilling_ingested_events_total", "Events accepted through the ingest API.", map[string]string{"client": client}, float64(len(accepted))) + writeJSON(w, status, map[string]any{"accepted": len(accepted), "duplicates": len(dups), "rejected": rejected, "ids": accepted}) +} + +// --- reconciliation --- + +type reconcileRow struct { + Tenant string `json:"tenant"` + Metric string `json:"metric"` + Ledger float64 `json:"ledger"` + DeadLettered float64 `json:"dead_lettered"` + Expected float64 `json:"expected"` + Recorded float64 `json:"recorded"` + Difference float64 `json:"difference"` + Match bool `json:"match"` +} + +type reconcileResult struct { + Destination string `json:"destination"` + OK bool `json:"ok"` + Error string `json:"error,omitempty"` + Rows []reconcileRow `json:"rows"` +} + +// reconcile recomputes usage from the ledger and compares it with what each +// backend recorded. Default range: the previous full UTC day, which every +// backend can answer for (Metronome only reports whole UTC days). +func (s *Server) reconcile(w http.ResponseWriter, r *http.Request) { + today := s.now().UTC().Truncate(24 * time.Hour) + from, to, err := s.parseRange(r, today.Add(-24*time.Hour), today) + if err != nil { + writeErr(w, http.StatusBadRequest, "%v", err) + return + } + only := r.URL.Query().Get("destination") + keep := filter(r) + var results []reconcileResult + for _, d := range s.disp.Destinations() { + if only != "" && d.Name() != only { + continue + } + rec, ok := d.(destinations.Reconciler) + if !ok { + if only != "" { + results = append(results, reconcileResult{Destination: d.Name(), Error: "destination does not support reconciliation"}) + } + continue + } + results = append(results, s.reconcileOne(r.Context(), d, rec, from, to, keep)) + } + writeJSON(w, http.StatusOK, map[string]any{"from": from, "to": to, "results": results}) +} + +func (s *Server) reconcileOne(ctx context.Context, d destinations.Destination, rec destinations.Reconciler, from, to time.Time, keep func(*usage.Event) bool) reconcileResult { + res := reconcileResult{Destination: d.Name()} + type key struct{ tenant, metric string } + ledger, dead := map[key]float64{}, map[key]float64{} + in := func(e *usage.Event) bool { + return !e.WindowStart.Before(from) && e.WindowStart.Before(to) && destinations.Accepts(d, e.Metric) && keep(e) + } + err := s.spool.Scan(from, to, func(e *usage.Event) error { + if in(e) { + ledger[key{e.Tenant, e.Metric}] += e.Quantity + } + return nil + }) + if err != nil { + res.Error = err.Error() + return res + } + if dls, err := s.spool.DeadLetters(d.Name()); err == nil { + for i := range dls { + if in(&dls[i].Event) { + dead[key{dls[i].Event.Tenant, dls[i].Event.Metric}] += dls[i].Event.Quantity + } + } + } + tenantSet, metricSet := map[string]bool{}, map[string]bool{} + for k := range ledger { + tenantSet[k.tenant], metricSet[k.metric] = true, true + } + q := destinations.TotalsQuery{From: from, To: to} + for t := range tenantSet { + tt, _ := s.tenants.Get(t) + q.Tenants = append(q.Tenants, tt) + } + for m := range metricSet { + q.Metrics = append(q.Metrics, m) + } + sort.Slice(q.Tenants, func(i, j int) bool { return q.Tenants[i].ID < q.Tenants[j].ID }) + sort.Strings(q.Metrics) + recorded := map[key]float64{} + if len(q.Tenants) > 0 { + totals, err := rec.RecordedTotals(ctx, q) + if err != nil { + res.Error = err.Error() + return res + } + for _, t := range totals { + recorded[key{t.Tenant, t.Metric}] += t.Quantity + } + } + keys := map[key]bool{} + for k := range ledger { + keys[k] = true + } + for k := range recorded { + keys[k] = true + } + res.OK = true + for k := range keys { + expected := ledger[k] - dead[k] + diff := recorded[k] - expected + match := math.Abs(diff) <= math.Max(1e-6, 1e-3*math.Abs(expected)) + res.OK = res.OK && match + res.Rows = append(res.Rows, reconcileRow{Tenant: k.tenant, Metric: k.metric, Ledger: usage.Round(ledger[k]), DeadLettered: usage.Round(dead[k]), + Expected: usage.Round(expected), Recorded: usage.Round(recorded[k]), Difference: usage.Round(diff), Match: match}) + } + sort.Slice(res.Rows, func(i, j int) bool { + if res.Rows[i].Tenant != res.Rows[j].Tenant { + return res.Rows[i].Tenant < res.Rows[j].Tenant + } + return res.Rows[i].Metric < res.Rows[j].Metric + }) + return res +} + +// --- ledger, destinations, dead letters --- + +func (s *Server) verify(w http.ResponseWriter, r *http.Request) { + rep := s.spool.Verify() + status := http.StatusOK + if !rep.OK { + status = http.StatusConflict + } + writeJSON(w, status, rep) +} + +func (s *Server) listDestinations(w http.ResponseWriter, r *http.Request) { + type row struct { + pipeline.Status + DeadLetters int `json:"dead_letters"` + } + var out []row + for _, st := range s.disp.Statuses() { + dls, _ := s.spool.DeadLetters(st.Name) + out = append(out, row{Status: st, DeadLetters: len(dls)}) + } + writeJSON(w, http.StatusOK, map[string]any{"destinations": out}) +} + +func (s *Server) destination(name string) destinations.Destination { + for _, d := range s.disp.Destinations() { + if d.Name() == name { + return d + } + } + return nil +} + +func (s *Server) deadLetters(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if s.destination(name) == nil { + writeErr(w, http.StatusNotFound, "no destination %q", name) + return + } + dls, err := s.spool.DeadLetters(name) + if err != nil { + writeErr(w, http.StatusInternalServerError, "%v", err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"destination": name, "dead_letters": dls}) +} + +// replayDeadLetters resends parked events (after fixing the cause, for +// example creating a missing customer). Event IDs are stable, so a replay +// can never double-bill. +func (s *Server) replayDeadLetters(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + d := s.destination(name) + if d == nil { + writeErr(w, http.StatusNotFound, "no destination %q", name) + return + } + dls, err := s.spool.DeadLetters(name) + if err != nil { + writeErr(w, http.StatusInternalServerError, "%v", err) + return + } + if len(dls) == 0 { + writeJSON(w, http.StatusOK, map[string]any{"replayed": 0}) + return + } + events := make([]usage.Event, len(dls)) + for i := range dls { + events[i] = dls[i].Event + } + err = d.SendEvents(r.Context(), events) + var partial *destinations.PartialError + switch { + case err == nil: + s.spool.ClearDeadLetters(name) + writeJSON(w, http.StatusOK, map[string]any{"replayed": len(events)}) + case errors.As(err, &partial): + s.spool.ClearDeadLetters(name) + for _, ev := range events { + if reason, bad := partial.Rejected[ev.ID]; bad { + s.spool.AddDeadLetter(name, ev, reason) + } + } + writeJSON(w, http.StatusOK, map[string]any{"replayed": len(events) - len(partial.Rejected), "still_rejected": len(partial.Rejected)}) + default: + writeErr(w, http.StatusBadGateway, "replay failed, dead letters kept: %v", err) + } +} + +// setCursor rewinds or advances a destination, e.g. {"seq": 0} replays the +// retained ledger into a newly added backend. +func (s *Server) setCursor(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if s.destination(name) == nil { + writeErr(w, http.StatusNotFound, "no destination %q", name) + return + } + var req struct { + Seq *uint64 `json:"seq"` + } + if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&req); err != nil || req.Seq == nil { + writeErr(w, http.StatusBadRequest, `body must be {"seq": }`) + return + } + if err := s.spool.SetCursor(name, *req.Seq); err != nil { + writeErr(w, http.StatusInternalServerError, "%v", err) + return + } + log.Printf("[api] cursor for %s set to %d", name, *req.Seq) + writeJSON(w, http.StatusOK, map[string]any{"destination": name, "cursor": s.spool.Cursor(name)}) +} + +// --- billing state --- + +func (s *Server) billingStates(w http.ResponseWriter, r *http.Request) { + if s.enf == nil { + writeJSON(w, http.StatusOK, map[string]any{"enforcement": "disabled", "states": []any{}}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"mode": s.cfg.EnforcementMode, "states": s.enf.States()}) +} + +func (s *Server) setBillingState(w http.ResponseWriter, r *http.Request) { + if s.enf == nil { + writeErr(w, http.StatusNotFound, "enforcement is disabled") + return + } + var req struct { + State enforcement.State `json:"state"` + Reason string `json:"reason"` + } + if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&req); err != nil { + writeErr(w, http.StatusBadRequest, "invalid JSON: %v", err) + return + } + if req.Reason == "" { + req.Reason = "set by operator" + } + tenant := r.PathValue("tenant") + if err := s.enf.Set(r.Context(), tenant, req.State, req.Reason, "api", "manual"); err != nil { + writeErr(w, http.StatusBadRequest, "%v", err) + return + } + writeJSON(w, http.StatusOK, s.enf.StateOf(tenant)) +} + +// --- webhooks --- + +func (s *Server) webhookResult(w http.ResponseWriter, sig enforcement.Signal, err error) { + switch { + case err == nil: + writeJSON(w, http.StatusOK, map[string]any{"applied": true, "tenant": sig.Tenant, "state": sig.State}) + case errors.Is(err, enforcement.ErrSignature): + writeErr(w, http.StatusBadRequest, "%v", err) + case errors.Is(err, enforcement.ErrIgnored), errors.Is(err, enforcement.ErrUnknownCustomer), errors.Is(err, enforcement.ErrDuplicate): + writeJSON(w, http.StatusOK, map[string]any{"applied": false, "reason": err.Error()}) + default: + log.Printf("[api] webhook %s: %v", sig.Type, err) + writeErr(w, http.StatusInternalServerError, "%v", err) // the provider retries + } +} + +func (s *Server) stripeWebhook(w http.ResponseWriter, r *http.Request) { + if s.enf == nil || s.cfg.StripeWebhookSecret == "" { + writeErr(w, http.StatusNotFound, "stripe webhooks are not enabled (set STRIPE_WEBHOOK_SECRET)") + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + writeErr(w, http.StatusBadRequest, "%v", err) + return + } + sig, err := s.enf.HandleStripe(r.Context(), body, r.Header.Get("Stripe-Signature")) + s.webhookResult(w, sig, err) +} + +func (s *Server) metronomeWebhook(w http.ResponseWriter, r *http.Request) { + if s.enf == nil || s.cfg.MetronomeWebhookSecret == "" { + writeErr(w, http.StatusNotFound, "metronome webhooks are not enabled (set METRONOME_WEBHOOK_SECRET)") + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + writeErr(w, http.StatusBadRequest, "%v", err) + return + } + date := r.Header.Get("X-Metronome-Date") + if date == "" { + date = r.Header.Get("Date") + } + sig, err := s.enf.HandleMetronome(r.Context(), body, date, r.Header.Get("Metronome-Webhook-Signature")) + s.webhookResult(w, sig, err) +} diff --git a/internal/api/server_test.go b/internal/api/server_test.go new file mode 100644 index 0000000..8a376c7 --- /dev/null +++ b/internal/api/server_test.go @@ -0,0 +1,338 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" + "github.com/vclusterlabs-experiments/vbilling/internal/enforcement" + "github.com/vclusterlabs-experiments/vbilling/internal/pipeline" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/telemetry" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +var now = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + +// recDest is a destination that records totals for reconciliation. +type recDest struct { + mu sync.Mutex + recorded map[string]float64 // tenant|metric -> quantity + reject map[string]bool + sent int +} + +func (d *recDest) Name() string { return "fakebill" } +func (d *recDest) Bootstrap(context.Context, []usage.MetricDef) error { return nil } +func (d *recDest) EnsureTenant(context.Context, usage.Tenant) error { return nil } +func (d *recDest) RemoveTenant(context.Context, usage.Tenant) error { return nil } +func (d *recDest) Accepts(m string) bool { return destinations.BillableOnly(m) } +func (d *recDest) SendEvents(_ context.Context, evs []usage.Event) error { + d.mu.Lock() + defer d.mu.Unlock() + rej := map[string]string{} + for _, e := range evs { + if d.reject[e.ID] { + rej[e.ID] = "unknown customer" + continue + } + d.recorded[e.Tenant+"|"+e.Metric] += e.Quantity + d.sent++ + } + if len(rej) > 0 { + return &destinations.PartialError{Rejected: rej} + } + return nil +} +func (d *recDest) RecordedTotals(_ context.Context, q destinations.TotalsQuery) ([]destinations.Total, error) { + d.mu.Lock() + defer d.mu.Unlock() + var out []destinations.Total + for _, t := range q.Tenants { + for _, m := range q.Metrics { + if v, ok := d.recorded[t.ID+"|"+m]; ok { + out = append(out, destinations.Total{Tenant: t.ID, Metric: m, Quantity: v}) + } + } + } + return out, nil +} + +type env struct { + srv *httptest.Server + sp *spool.Spool + dest *recDest + disp *pipeline.Dispatcher + enf *enforcement.Enforcer +} + +func setup(t *testing.T, mutate ...func(*config.Config)) *env { + t.Helper() + cfg := &config.Config{Region: "ap-southeast-2", ClusterName: "syd-1", Adapters: []string{"fakebill"}, CollectionInterval: time.Minute, + IngestToken: "ingest-secret", EnforcementMode: "observe", StripeWebhookSecret: "whsec_api"} + for _, m := range mutate { + m(cfg) + } + sp, err := spool.Open(t.TempDir(), spool.Options{NoSync: true, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { sp.Close() }) + dest := &recDest{recorded: map[string]float64{}, reject: map[string]bool{}} + reg := pipeline.NewTenantRegistry() + reg.Upsert(usage.Tenant{ID: "acme", DisplayName: "Acme"}) + disp := pipeline.New(sp, []destinations.Destination{dest}, pipeline.Options{IdlePoll: 5 * time.Millisecond, MinBackoff: time.Millisecond}, nil) + tel := telemetry.New() + enf := enforcement.New(nil, enforcement.Options{Mode: "observe", StripeWebhookSecret: cfg.StripeWebhookSecret, Now: func() time.Time { return now }, + Telemetry: tel, Resolvers: map[string]destinations.CustomerResolver{"stripe": resolver{"cus_acme": "acme"}}}) + s := New(cfg, sp, disp, reg, nil, enf, tel, "test") + s.now = func() time.Time { return now } + srv := httptest.NewServer(s.Handler()) + t.Cleanup(srv.Close) + return &env{srv: srv, sp: sp, dest: dest, disp: disp, enf: enf} +} + +type resolver map[string]string + +func (r resolver) TenantForCustomer(_ context.Context, id string) (string, bool) { + v, ok := r[id] + return v, ok +} + +func (e *env) commit(t *testing.T, windows int, start time.Time) { + t.Helper() + for i := 0; i < windows; i++ { + ws := start.Add(time.Duration(i) * time.Minute) + var evs []usage.Event + for _, sku := range []string{"NVIDIA-H100", "NVIDIA-L40S"} { + ev := usage.Event{Tenant: "acme", Metric: usage.MetricGPUHours, Quantity: 8.0 / 60, SKU: sku, Region: "ap-southeast-2", + WindowStart: ws, WindowEnd: ws.Add(time.Minute), Dimensions: map[string]string{usage.DimTenantCluster: "vcluster-a-train"}, RecordedAt: now} + ev.Finalize() + evs = append(evs, ev) + } + util := usage.Event{Tenant: "acme", Metric: usage.MetricGPUUtilization, Quantity: 1, WindowStart: ws, WindowEnd: ws.Add(time.Minute), RecordedAt: now} + util.Finalize() + evs = append(evs, util) + if err := e.sp.AppendWindow(spool.SourceCollector, ws.Add(time.Minute), evs, ""); err != nil { + t.Fatal(err) + } + } +} + +func (e *env) get(t *testing.T, path string) (int, string) { + t.Helper() + resp, err := http.Get(e.srv.URL + path) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) +} + +func (e *env) deliver(t *testing.T) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + go e.disp.Run(ctx) + for e.sp.Cursor("fakebill") < e.sp.Committed() { + if ctx.Err() != nil { + t.Fatal("delivery timed out") + } + time.Sleep(2 * time.Millisecond) + } +} + +func TestUsageLineItemsAndCSV(t *testing.T) { + e := setup(t) + e.sp.Cursor("fakebill") + e.commit(t, 60, now.Add(-2*time.Hour)) + code, body := e.get(t, "/api/v1/usage?group_by=tenant,sku&metric="+usage.MetricGPUHours) + if code != 200 { + t.Fatalf("usage: %d %s", code, body) + } + var resp struct { + Rows []map[string]any `json:"rows"` + } + json.Unmarshal([]byte(body), &resp) + if len(resp.Rows) != 2 || resp.Rows[0]["sku"] != "NVIDIA-H100" || resp.Rows[0]["quantity"].(float64) != usage.Round(60*usage.Round(8.0/60)) { + t.Fatalf("rows: %+v", resp.Rows) + } + code, body = e.get(t, "/api/v1/usage?format=csv&group_by=tenant,day") + if code != 200 || !strings.HasPrefix(body, "tenant,day,metric,unit,quantity,events\n") || !strings.Contains(body, "acme,2026-10-05,vcluster_gpu_hours,gpu-hours,15.99999996,120") { + t.Fatalf("csv: %d\n%s", code, body) + } + code, body = e.get(t, "/api/v1/events?format=csv&limit=3") + if code != 200 || strings.Count(body, "\n") != 4 { + t.Fatalf("event export: %d\n%s", code, body) + } +} + +func TestIngestValidatesDedupesAndAuthenticates(t *testing.T) { + defer func() { recover() }() + usage.RegisterCustom(usage.MetricDef{Code: "inference_output_tokens", Unit: "tokens", GroupKeys: []string{"model"}}) + e := setup(t) + post := func(token, body string) (int, map[string]any) { + req, _ := http.NewRequest("POST", e.srv.URL+"/api/v1/events", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("X-VBilling-Client", "model-gateway") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out map[string]any + json.NewDecoder(resp.Body).Decode(&out) + return resp.StatusCode, out + } + ws := now.Add(-5 * time.Minute).Format(time.RFC3339) + we := now.Add(-4 * time.Minute).Format(time.RFC3339) + good := fmt.Sprintf(`{"tenant":"acme","metric":"inference_output_tokens","quantity":125000,"window_start":%q,"window_end":%q,"dimensions":{"model":"llama-3-70b"}}`, ws, we) + bad := fmt.Sprintf(`{"tenant":"acme","metric":"made_up","quantity":1,"window_start":%q,"window_end":%q}`, ws, we) + future := fmt.Sprintf(`{"tenant":"acme","metric":"inference_output_tokens","quantity":1,"window_start":%q,"window_end":%q}`, we, now.Add(time.Hour).Format(time.RFC3339)) + + if code, _ := post("wrong", `{"events":[]}`); code != 401 { + t.Fatalf("bad token: %d", code) + } + code, out := post("ingest-secret", `{"events":[`+good+`,`+bad+`,`+future+`]}`) + if code != 200 || out["accepted"].(float64) != 1 || len(out["rejected"].([]any)) != 2 { + t.Fatalf("ingest: %d %+v", code, out) + } + code, out = post("ingest-secret", `[`+good+`]`) // resend without an ID: derived ID dedupes + if code != 200 || out["accepted"].(float64) != 0 || out["duplicates"].(float64) != 1 { + t.Fatalf("resend: %d %+v", code, out) + } + var stored *usage.Event + e.sp.Scan(now.Add(-time.Hour), now, func(ev *usage.Event) error { stored = ev; return nil }) + if stored == nil || stored.Source != "ingest:model-gateway" || stored.Region != "ap-southeast-2" || stored.Unit != "tokens" { + t.Fatalf("stored event: %+v", stored) + } + + disabled := setup(t, func(c *config.Config) { c.IngestToken = "" }) + req, _ := http.NewRequest("POST", disabled.srv.URL+"/api/v1/events", strings.NewReader(good)) + resp, _ := http.DefaultClient.Do(req) + if resp.StatusCode != 403 { + t.Fatalf("ingest without a configured token must be disabled, got %d", resp.StatusCode) + } +} + +func TestReconcileMatchesAndExplainsDeadLetters(t *testing.T) { + e := setup(t) + e.sp.Cursor("fakebill") + day := now.Truncate(24 * time.Hour).Add(-24 * time.Hour) + e.commit(t, 30, day.Add(time.Hour)) + // Reject one event so it is dead-lettered; reconciliation must explain it. + var victim usage.Event + e.sp.Scan(day, day.Add(24*time.Hour), func(ev *usage.Event) error { + if victim.ID == "" && ev.Metric == usage.MetricGPUHours { + victim = *ev + } + return nil + }) + e.dest.reject[victim.ID] = true + e.deliver(t) + + code, body := e.get(t, "/api/v1/reconcile") + if code != 200 { + t.Fatalf("reconcile: %d %s", code, body) + } + var resp struct { + Results []reconcileResult `json:"results"` + } + json.Unmarshal([]byte(body), &resp) + if len(resp.Results) != 1 || !resp.Results[0].OK || len(resp.Results[0].Rows) != 1 { + t.Fatalf("reconcile result: %s", body) + } + row := resp.Results[0].Rows[0] + if row.DeadLettered == 0 || !row.Match || row.Metric != usage.MetricGPUHours { + t.Fatalf("row: %+v", row) + } + + // Drift in the backend shows up as a mismatch. + e.dest.mu.Lock() + e.dest.recorded["acme|"+usage.MetricGPUHours] -= 1 + e.dest.mu.Unlock() + _, body = e.get(t, "/api/v1/reconcile") + json.Unmarshal([]byte(body), &resp) + if resp.Results[0].OK || resp.Results[0].Rows[0].Match { + t.Fatalf("drift not detected: %s", body) + } + + // Fix the cause and replay the dead letter: it reconciles again. + delete(e.dest.reject, victim.ID) + e.dest.mu.Lock() + e.dest.recorded["acme|"+usage.MetricGPUHours] += 1 + e.dest.mu.Unlock() + resp2, _ := http.Post(e.srv.URL+"/api/v1/destinations/fakebill/dead-letters/replay", "application/json", nil) + if resp2.StatusCode != 200 { + t.Fatalf("replay status %d", resp2.StatusCode) + } + _, body = e.get(t, "/api/v1/reconcile") + json.Unmarshal([]byte(body), &resp) + if !resp.Results[0].OK || resp.Results[0].Rows[0].DeadLettered != 0 { + t.Fatalf("after replay: %s", body) + } +} + +func TestVerifyDestinationsCursorAndWebhook(t *testing.T) { + e := setup(t) + e.commit(t, 3, now.Add(-time.Hour)) + if code, body := e.get(t, "/api/v1/ledger/verify"); code != 200 || !strings.Contains(body, `"ok": true`) { + t.Fatalf("verify: %d %s", code, body) + } + resp, _ := http.Post(e.srv.URL+"/api/v1/destinations/fakebill/cursor", "application/json", strings.NewReader(`{"seq":0}`)) + if resp.StatusCode != 200 || e.sp.Cursor("fakebill") != 0 { + t.Fatalf("cursor rewind: %d", resp.StatusCode) + } + + body := []byte(`{"id":"evt_1","type":"invoice.payment_failed","data":{"object":{"customer":"cus_acme","attempt_count":1}}}`) + req, _ := http.NewRequest("POST", e.srv.URL+"/webhooks/stripe", bytes.NewReader(body)) + req.Header.Set("Stripe-Signature", stripe.SignatureHeader(body, "whsec_api", now)) + resp, err := http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != 200 { + t.Fatalf("webhook: %v %d", err, resp.StatusCode) + } + if e.enf.StateOf("acme").State != enforcement.Delinquent { + t.Fatal("webhook did not update billing state") + } + req, _ = http.NewRequest("POST", e.srv.URL+"/webhooks/stripe", bytes.NewReader(body)) + req.Header.Set("Stripe-Signature", "t=1,v1=bad") + resp, _ = http.DefaultClient.Do(req) + if resp.StatusCode != 400 { + t.Fatalf("bad signature: %d", resp.StatusCode) + } + if code, _ := e.get(t, "/webhooks/metronome"); code != 405 && code != 404 { + t.Fatalf("GET on webhook: %d", code) + } + code, mbody := e.get(t, "/metrics") + if code != 200 || !strings.Contains(mbody, "vbilling_ledger_committed_seq") || !strings.Contains(mbody, `vbilling_tenant_billing_state{state="delinquent",tenant="acme"} 1`) { + t.Fatalf("metrics: %s", mbody) + } +} + +func TestAPITokenProtectsReadEndpoints(t *testing.T) { + e := setup(t, func(c *config.Config) { c.APIToken = "read-secret" }) + if code, _ := e.get(t, "/api/v1/usage"); code != 401 { + t.Fatalf("unauthenticated usage read: %d", code) + } + req, _ := http.NewRequest("GET", e.srv.URL+"/api/v1/usage", nil) + req.Header.Set("Authorization", "Bearer read-secret") + resp, _ := http.DefaultClient.Do(req) + if resp.StatusCode != 200 { + t.Fatalf("authenticated read: %d", resp.StatusCode) + } + if code, _ := e.get(t, "/healthz"); code != 200 { + t.Fatal("healthz must stay open for probes") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 4e32db7..da9dac4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,10 @@ +// Package config loads vBilling settings from environment variables. +// Every v0.1 variable keeps working; ADAPTER is still accepted as a single +// entry of ADAPTERS. package config import ( + "fmt" "os" "strconv" "strings" @@ -8,57 +12,233 @@ import ( ) type Config struct { - // Adapter selects the billing destination ("lago", "noop", future: "metronome", "stripe", "openmeter"). - Adapter string + // Adapters lists the destinations usage fans out to, e.g. + // ["metronome", "webhook"]. Built-in: lago, stripe, metronome, webhook, noop. + Adapters []string - // Lago connection (used when Adapter == "lago") - LagoAPIURL string - LagoAPIKey string + // Where this vBilling instance meters. Region is stamped on every event; + // a node's topology.kubernetes.io/region label wins when present. + Region string + Zone string + ClusterName string // name of the control plane cluster (event source, webhook source) + TenantClass string // default tenant class, e.g. public, enterprise, government, dev + + // Durable ledger. + DataDir string + Retention time.Duration + + // Collection. CollectionInterval is the metering window size. + CollectionInterval time.Duration + ReconcileInterval time.Duration + MaxBackfill time.Duration + CPUMemoryBasis string // usage | requests | max + MeterControlPlane bool + MeterByNamespace bool + GPUResources []string + SKULabel string + CapacityTypeLabel string + UnhealthyTaints []string + PrometheusURL string + PrometheusHeaders map[string]string // extra request headers, e.g. X-Scope-OrgID or Authorization + PromTokenFile string // bearer token file, re-read on every query + EgressQuery string + GPUUtilQuery string // replaces the DCGM utilization query; "none" disables it + GPUCountQuery string + PrometheusPreset string // "vcluster-platform": vCluster Platform fleet observability labels + PrometheusMetrics string // PROMETHEUS_METRICS_FILE: operator-defined metrics from PromQL + DRAGPUDrivers []string + WatchDeletions bool // bill pods and nodes deleted between windows - // Collection intervals - CollectionInterval time.Duration // how often to scrape metrics and send events - ReconcileInterval time.Duration // how often to discover new/removed vClusters + // Tenant clusters with their own nodes (Private Nodes, Auto Nodes, Standalone). + TenantAPI bool + TenantKubeconfigSecret string + TenantAdminFallback bool + TenantClustersFile string + PrivateNodeBilling string // node | usage + TenantExcludeNamespaces []string + WatchNamespaces []string + DedicatedNodeLabel string // extra node selector for dedicated nodes, "key=%s" (%s = cluster name) + RegionFromNode bool // use topology.kubernetes.io/region from nodes instead of REGION + TenantSource string // cluster | project + PlatformCluster string + OffboardGrace time.Duration + CustomMetrics string - // Billing plan + // Billing defaults. DefaultPlanCode string BillingCurrency string - // Optional: Prometheus for DCGM + network metrics - PrometheusURL string + // HTTP API. + ListenAddr string + APIToken string + IngestToken string - // Node cost attribution - SpotDiscountPercent float64 // e.g. 60 means spot nodes cost 40% of on-demand + // Enforcement of billing state on tenant clusters: observe | annotate | enforce. + EnforcementMode string + EnforcementRules string // "event.type=state,..." overrides + + // Lago. + LagoAPIURL string + LagoAPIKey string - // Namespace filter (empty = all namespaces) - WatchNamespaces []string + // Stripe (Billing Meters). + StripeAPIKey string + StripeAPIBase string + StripeAPIVersion string + StripeWebhookSecret string + StripeSplitMetersBy []string + StripeAutoSubscribe bool + StripeMaxRPS float64 + StripeCancelOnRemove bool + + // Metronome. + MetronomeAPIToken string + MetronomeAPIBase string + MetronomeWebhookSecret string + MetronomeRateCard string // rate card ID or alias for automatic contracts + MetronomeStripeLink bool // create the Stripe customer and link it for invoicing + MetronomeStripeCollect string // charge_automatically | send_invoice + + // Webhook (CloudEvents). + WebhookURL string + WebhookSecret string + WebhookHeaders map[string]string + + // Deprecated: discounts belong in the rating engine, keyed on the + // capacity_type dimension. Kept so v0.1 Helm values still parse. + SpotDiscountPercent float64 } func Load() *Config { c := &Config{ - Adapter: envOr("ADAPTER", "lago"), - LagoAPIURL: envOr("LAGO_API_URL", "http://localhost:3000"), - LagoAPIKey: envOr("LAGO_API_KEY", ""), + Adapters: envList("ADAPTERS", nil), + Region: envOr("REGION", "default"), + Zone: os.Getenv("ZONE"), + ClusterName: envOr("CLUSTER_NAME", "control-plane"), + TenantClass: os.Getenv("DEFAULT_TENANT_CLASS"), + + DataDir: envOr("DATA_DIR", "/var/lib/vbilling"), + Retention: envDuration("RETENTION", 7*24*time.Hour), + CollectionInterval: envDuration("COLLECTION_INTERVAL", 60*time.Second), ReconcileInterval: envDuration("RECONCILE_INTERVAL", 30*time.Second), - DefaultPlanCode: envOr("DEFAULT_PLAN_CODE", "vcluster-standard"), - BillingCurrency: envOr("BILLING_CURRENCY", "USD"), - PrometheusURL: envOr("PROMETHEUS_URL", ""), - SpotDiscountPercent: envFloat("SPOT_DISCOUNT_PERCENT", 60), - } - - // Parse watch namespaces from comma-separated env var - if ns := os.Getenv("WATCH_NAMESPACES"); ns != "" { - for _, n := range strings.Split(ns, ",") { - n = strings.TrimSpace(n) - if n != "" { - c.WatchNamespaces = append(c.WatchNamespaces, n) - } - } - } + MaxBackfill: envDuration("MAX_BACKFILL", 6*time.Hour), + CPUMemoryBasis: envOr("CPU_MEMORY_BASIS", "usage"), + MeterControlPlane: envBool("METER_CONTROL_PLANE", false), + MeterByNamespace: envBool("METER_BY_NAMESPACE", false), + GPUResources: envList("GPU_RESOURCES", []string{"nvidia.com/gpu", "amd.com/gpu"}), + SKULabel: os.Getenv("SKU_LABEL"), + CapacityTypeLabel: envOr("CAPACITY_TYPE_LABEL", "vbilling.vcluster.com/capacity-type"), + UnhealthyTaints: envList("UNHEALTHY_NODE_TAINTS", []string{"node.kubernetes.io/not-ready", "node.kubernetes.io/unreachable"}), + PrometheusURL: os.Getenv("PROMETHEUS_URL"), + PrometheusHeaders: envMap("PROMETHEUS_HEADERS"), + PromTokenFile: os.Getenv("PROMETHEUS_BEARER_TOKEN_FILE"), + EgressQuery: os.Getenv("EGRESS_QUERY"), + GPUUtilQuery: os.Getenv("GPU_UTIL_QUERY"), + GPUCountQuery: os.Getenv("GPU_COUNT_QUERY"), + PrometheusPreset: os.Getenv("PROMETHEUS_PRESET"), + PrometheusMetrics: os.Getenv("PROMETHEUS_METRICS_FILE"), + DRAGPUDrivers: envList("DRA_GPU_DRIVERS", []string{"gpu.nvidia.com", "gpu.amd.com"}), + WatchDeletions: envBool("WATCH_DELETIONS", true), + + TenantAPI: envBool("TENANT_API", true), + TenantKubeconfigSecret: envOr("TENANT_KUBECONFIG_SECRET", "vbilling-kubeconfig"), + TenantAdminFallback: envBool("TENANT_ADMIN_FALLBACK", false), + TenantClustersFile: os.Getenv("TENANT_CLUSTERS_FILE"), + PrivateNodeBilling: envOr("PRIVATE_NODE_BILLING", "node"), + TenantExcludeNamespaces: envList("TENANT_EXCLUDE_NAMESPACES", []string{"kube-system"}), + WatchNamespaces: envList("WATCH_NAMESPACES", nil), + DedicatedNodeLabel: os.Getenv("VCLUSTER_NODE_LABEL"), + RegionFromNode: envBool("REGION_FROM_NODE", false), + TenantSource: envOr("TENANT_SOURCE", "cluster"), + PlatformCluster: os.Getenv("PLATFORM_CLUSTER"), + OffboardGrace: envDuration("OFFBOARD_GRACE", time.Hour), + CustomMetrics: os.Getenv("CUSTOM_METRICS"), + + DefaultPlanCode: envOr("DEFAULT_PLAN_CODE", "vcluster-standard"), + BillingCurrency: envOr("BILLING_CURRENCY", "USD"), + + ListenAddr: envOr("LISTEN_ADDR", ":8080"), + APIToken: secretEnv("API_TOKEN"), + IngestToken: secretEnv("INGEST_TOKEN"), + + EnforcementMode: envOr("ENFORCEMENT_MODE", "observe"), + EnforcementRules: os.Getenv("ENFORCEMENT_RULES"), + + LagoAPIURL: envOr("LAGO_API_URL", "http://localhost:3000"), + LagoAPIKey: secretEnv("LAGO_API_KEY"), + + StripeAPIKey: secretEnv("STRIPE_API_KEY"), + StripeAPIBase: envOr("STRIPE_API_BASE", "https://api.stripe.com"), + StripeAPIVersion: envOr("STRIPE_API_VERSION", "2026-09-30.endive"), + StripeWebhookSecret: secretEnv("STRIPE_WEBHOOK_SECRET"), + StripeSplitMetersBy: envList("STRIPE_SPLIT_METERS_BY", []string{"sku"}), + StripeAutoSubscribe: envBool("STRIPE_AUTO_SUBSCRIBE", false), + StripeMaxRPS: envFloat("STRIPE_MAX_RPS", 0), + StripeCancelOnRemove: envBool("STRIPE_CANCEL_ON_REMOVE", false), + MetronomeAPIToken: secretEnv("METRONOME_API_TOKEN"), + MetronomeAPIBase: envOr("METRONOME_API_BASE", "https://api.metronome.com"), + MetronomeWebhookSecret: secretEnv("METRONOME_WEBHOOK_SECRET"), + MetronomeRateCard: os.Getenv("METRONOME_RATE_CARD"), + MetronomeStripeLink: envBool("METRONOME_STRIPE_LINK", false), + MetronomeStripeCollect: envOr("METRONOME_STRIPE_COLLECTION_METHOD", "charge_automatically"), + + WebhookURL: os.Getenv("WEBHOOK_URL"), + WebhookSecret: secretEnv("WEBHOOK_SECRET"), + WebhookHeaders: envMap("WEBHOOK_HEADERS"), + + SpotDiscountPercent: envFloat("SPOT_DISCOUNT_PERCENT", 0), + } + if len(c.Adapters) == 0 { + c.Adapters = []string{envOr("ADAPTER", "lago")} + } + if c.IngestToken == "" { + c.IngestToken = c.APIToken + } return c } +// Validate rejects settings that would silently produce wrong bills. +func (c *Config) Validate() error { + if c.CollectionInterval < 10*time.Second || c.CollectionInterval > time.Hour { + return fmt.Errorf("COLLECTION_INTERVAL %s must be between 10s and 1h", c.CollectionInterval) + } + if time.Hour%c.CollectionInterval != 0 { + return fmt.Errorf("COLLECTION_INTERVAL %s must divide an hour evenly so windows align with billing hours", c.CollectionInterval) + } + switch c.CPUMemoryBasis { + case "usage", "requests", "max": + default: + return fmt.Errorf("CPU_MEMORY_BASIS %q must be usage, requests or max", c.CPUMemoryBasis) + } + switch c.TenantSource { + case "cluster", "project": + default: + return fmt.Errorf("TENANT_SOURCE %q must be cluster or project", c.TenantSource) + } + switch c.EnforcementMode { + case "observe", "annotate", "enforce": + default: + return fmt.Errorf("ENFORCEMENT_MODE %q must be observe, annotate or enforce", c.EnforcementMode) + } + switch c.PrivateNodeBilling { + case "node", "usage": + default: + return fmt.Errorf("PRIVATE_NODE_BILLING %q must be node or usage", c.PrivateNodeBilling) + } + switch c.PrometheusPreset { + case "", "default", "vcluster-platform": + default: + return fmt.Errorf("PROMETHEUS_PRESET %q must be empty, default or vcluster-platform", c.PrometheusPreset) + } + return nil +} + +// secretEnv reads a key or token. Secrets mounted from files usually end in a +// newline, which would make every request's Authorization header invalid. +func secretEnv(key string) string { return strings.TrimSpace(os.Getenv(key)) } + func envOr(key, fallback string) string { if v := os.Getenv(key); v != "" { return v @@ -68,8 +248,7 @@ func envOr(key, fallback string) string { func envDuration(key string, fallback time.Duration) time.Duration { if v := os.Getenv(key); v != "" { - d, err := time.ParseDuration(v) - if err == nil { + if d, err := time.ParseDuration(v); err == nil { return d } } @@ -78,10 +257,49 @@ func envDuration(key string, fallback time.Duration) time.Duration { func envFloat(key string, fallback float64) float64 { if v := os.Getenv(key); v != "" { - f, err := strconv.ParseFloat(v, 64) - if err == nil { + if f, err := strconv.ParseFloat(v, 64); err == nil { return f } } return fallback } + +func envBool(key string, fallback bool) bool { + if v := os.Getenv(key); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return fallback +} + +// envList parses a comma-separated list. An explicitly empty value ("none") +// yields an empty, non-nil list so defaults can be switched off. +func envList(key string, fallback []string) []string { + v, ok := os.LookupEnv(key) + if !ok || strings.TrimSpace(v) == "" { + return fallback + } + out := []string{} + if strings.TrimSpace(v) == "none" { + return out + } + for _, p := range strings.Split(v, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +// envMap parses "k1=v1,k2=v2". +func envMap(key string) map[string]string { + out := map[string]string{} + for _, kv := range envList(key, nil) { + k, v, ok := strings.Cut(kv, "=") + if ok { + out[strings.TrimSpace(k)] = strings.TrimSpace(v) + } + } + return out +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..a4fd023 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,66 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +func TestLoadDefaultsAndLegacyAdapter(t *testing.T) { + t.Setenv("ADAPTER", "noop") + c := Load() + if strings.Join(c.Adapters, ",") != "noop" || c.CollectionInterval != time.Minute || c.CPUMemoryBasis != "usage" || c.EnforcementMode != "observe" { + t.Fatalf("defaults: %+v", c) + } + if err := c.Validate(); err != nil { + t.Fatal(err) + } + t.Setenv("ADAPTERS", "metronome, webhook") + t.Setenv("API_TOKEN", "tok") + t.Setenv("GPU_RESOURCES", "none") + c = Load() + if strings.Join(c.Adapters, ",") != "metronome,webhook" || c.IngestToken != "tok" || len(c.GPUResources) != 0 { + t.Fatalf("overrides: %+v", c) + } +} + +func TestValidateRejectsUnsafeSettings(t *testing.T) { + for name, mutate := range map[string]func(*Config){ + "window too small": func(c *Config) { c.CollectionInterval = 5 * time.Second }, + "window not hour-aligned": func(c *Config) { c.CollectionInterval = 7 * time.Minute }, + "bad basis": func(c *Config) { c.CPUMemoryBasis = "limits" }, + "bad mode": func(c *Config) { c.EnforcementMode = "delete-everything" }, + "bad tenant source": func(c *Config) { c.TenantSource = "owner" }, + } { + c := Load() + mutate(c) + if err := c.Validate(); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestPrometheusAuthAndGPUQueries(t *testing.T) { + t.Setenv("PROMETHEUS_HEADERS", "X-Scope-OrgID=tenant-1, Authorization=Basic dmI6cHc=") + t.Setenv("PROMETHEUS_BEARER_TOKEN_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/token") + t.Setenv("GPU_UTIL_QUERY", "none") + c := Load() + if c.PrometheusHeaders["X-Scope-OrgID"] != "tenant-1" || c.PrometheusHeaders["Authorization"] != "Basic dmI6cHc=" { + t.Fatalf("headers: %v", c.PrometheusHeaders) + } + if c.PromTokenFile == "" || c.GPUUtilQuery != "none" || c.GPUCountQuery != "" { + t.Fatalf("prometheus settings: %+v", c) + } +} + +// Secrets created with kubectl create secret --from-file keep the file's +// trailing newline; an API key with a newline breaks every request. +func TestSecretsFromFilesAreTrimmed(t *testing.T) { + t.Setenv("STRIPE_API_KEY", "sk_test_123\n") + t.Setenv("METRONOME_API_TOKEN", " tok \r\n") + t.Setenv("WEBHOOK_SECRET", "whsec_abc\n") + c := Load() + if c.StripeAPIKey != "sk_test_123" || c.MetronomeAPIToken != "tok" || c.WebhookSecret != "whsec_abc" { + t.Fatalf("secrets not trimmed: %q %q %q", c.StripeAPIKey, c.MetronomeAPIToken, c.WebhookSecret) + } +} diff --git a/internal/controller/controller.go b/internal/controller/controller.go index f100381..10f76ae 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -1,311 +1,415 @@ +// Package controller runs vBilling's two loops: tenant reconciliation and +// window-aligned metering into the durable ledger. package controller import ( "context" - "fmt" "log" + "path/filepath" + "sort" + "strings" "sync" "time" "github.com/vclusterlabs-experiments/vbilling/internal/config" "github.com/vclusterlabs-experiments/vbilling/internal/destinations" "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" "github.com/vclusterlabs-experiments/vbilling/internal/metrics" + "github.com/vclusterlabs-experiments/vbilling/internal/pipeline" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/telemetry" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" ) -// Controller is the main billing reconciliation loop. -// It discovers Tenant Clusters, ensures they exist in the configured billing -// destination, collects resource metrics, and ships usage events. -type Controller struct { - cfg *config.Config - dest destinations.Destination - discoverer *discovery.Discoverer - collector *metrics.Collector - - mu sync.Mutex - known map[string]*tracked // key = ExternalID +// Discoverer abstracts discovery for tests. +type Discoverer interface { + Discover(ctx context.Context) ([]discovery.TenantCluster, error) } -type tracked struct { - VCluster discovery.VCluster - Subscribed bool - LastCollection time.Time +// Collector abstracts metering for tests. +type Collector interface { + Collect(ctx context.Context, targets []metrics.Target, windows []metrics.Window) (map[time.Time][]usage.Event, metrics.Stats, error) } -func New(cfg *config.Config, dest destinations.Destination, disc *discovery.Discoverer, coll *metrics.Collector) *Controller { - return &Controller{ - cfg: cfg, - dest: dest, - discoverer: disc, - collector: coll, - known: make(map[string]*tracked), - } +// tenantFiller re-meters windows a tenant cluster's own API missed. +type tenantFiller interface { + CollectTenantAPI(ctx context.Context, targets []metrics.Target, windows []metrics.Window) (map[time.Time][]usage.Event, metrics.Stats, error) } -// Run starts the controller with two loops: -// 1. Reconcile loop: discovers Tenant Clusters and manages billing entities -// 2. Collection loop: scrapes metrics and ships events to the destination -func (c *Controller) Run(ctx context.Context) error { - log.Printf("[controller] starting (adapter=%s, reconcile=%s, collection=%s)", - c.dest.Name(), c.cfg.ReconcileInterval, c.cfg.CollectionInterval) +// SourceTenantFill is the ledger source of windows filled after a tenant API outage. +const SourceTenantFill = "tenant-api-fill" - c.reconcile(ctx) +// Controller owns tenant reconciliation and metering. +type Controller struct { + cfg *config.Config + disc Discoverer + coll Collector + spool *spool.Spool + tenants *pipeline.TenantRegistry + dests []destinations.Destination + tel *telemetry.Registry + now func() time.Time + + mu sync.Mutex + clusters []discovery.TenantCluster + lastSeen map[string]time.Time // tenant -> last time it had a tenant cluster + discovered bool + lastWindow time.Time + + // gaps: tenant cluster external ID -> end of the first window its own + // API could not be read for. Persisted, so a restart still fills them. + gaps *kv.Store +} - reconcileTicker := time.NewTicker(c.cfg.ReconcileInterval) - collectionTicker := time.NewTicker(c.cfg.CollectionInterval) - defer reconcileTicker.Stop() - defer collectionTicker.Stop() +func New(cfg *config.Config, disc Discoverer, coll Collector, sp *spool.Spool, tenants *pipeline.TenantRegistry, + dests []destinations.Destination, tel *telemetry.Registry) *Controller { + if tel == nil { + tel = telemetry.New() + } + gaps := kv.Memory() + if cfg.DataDir != "" { + if s, err := kv.Open(filepath.Join(cfg.DataDir, "tenant-api-gaps.json")); err != nil { + log.Printf("[controller] tenant API gaps kept in memory only: %v", err) + } else { + gaps = s + } + } + return &Controller{cfg: cfg, disc: disc, coll: coll, spool: sp, tenants: tenants, dests: dests, tel: tel, + now: time.Now, lastSeen: map[string]time.Time{}, gaps: gaps} +} +// Run reconciles tenants every ReconcileInterval and closes a metering +// window at every CollectionInterval boundary until ctx ends. +func (c *Controller) Run(ctx context.Context) error { + log.Printf("[controller] starting (adapters=%v, window=%s, reconcile=%s, region=%s)", + c.cfg.Adapters, c.cfg.CollectionInterval, c.cfg.ReconcileInterval, c.cfg.Region) + c.Reconcile(ctx) + c.CollectDue(ctx) + + reconcile := time.NewTicker(c.cfg.ReconcileInterval) + defer reconcile.Stop() + collect := time.NewTimer(c.untilNextWindow()) + defer collect.Stop() for { select { case <-ctx.Done(): - log.Println("[controller] shutting down") return ctx.Err() - case <-reconcileTicker.C: - c.reconcile(ctx) - case <-collectionTicker.C: - c.collectAndSend(ctx) + case <-reconcile.C: + c.Reconcile(ctx) + case <-collect.C: + c.CollectDue(ctx) + collect.Reset(c.untilNextWindow()) } } } -func (c *Controller) reconcile(ctx context.Context) { - vclusters, err := c.discoverer.Discover(ctx) - if err != nil { - log.Printf("[controller] discovery error: %v", err) - return - } +// settle gives the API server a moment to reflect state at the boundary. +const settle = 2 * time.Second + +func (c *Controller) untilNextWindow() time.Duration { + now := c.now() + next := now.Truncate(c.cfg.CollectionInterval).Add(c.cfg.CollectionInterval).Add(settle) + return next.Sub(now) +} +// Ready reports whether discovery has succeeded at least once. +func (c *Controller) Ready() bool { c.mu.Lock() defer c.mu.Unlock() + return c.discovered +} - alive := make(map[string]bool) - - for _, vc := range vclusters { - extID := vc.ExternalID() - alive[extID] = true +// Clusters returns the tenant clusters from the last successful discovery. +func (c *Controller) Clusters() []discovery.TenantCluster { + c.mu.Lock() + defer c.mu.Unlock() + return append([]discovery.TenantCluster(nil), c.clusters...) +} - if _, exists := c.known[extID]; exists { - continue +// ClustersOf returns the tenant clusters billed to a tenant. +func (c *Controller) ClustersOf(tenant string) []discovery.TenantCluster { + var out []discovery.TenantCluster + for _, cl := range c.Clusters() { + if cl.TenantID(discovery.TenantSource(c.cfg.TenantSource)) == tenant { + out = append(out, cl) } - - log.Printf("[controller] new Tenant Cluster discovered: %s/%s", vc.Namespace, vc.Name) - - if err := c.dest.EnsureTenant(ctx, tenantFor(vc, c.cfg.BillingCurrency)); err != nil { - log.Printf("[controller] ensure tenant %s: %v", extID, err) - continue - } - - c.known[extID] = &tracked{VCluster: vc, Subscribed: true} } + return out +} - for extID, t := range c.known { - if alive[extID] { - continue - } - log.Printf("[controller] Tenant Cluster removed: %s/%s", t.VCluster.Namespace, t.VCluster.Name) - if t.Subscribed { - if err := c.dest.RemoveTenant(ctx, extID); err != nil { - log.Printf("[controller] remove tenant %s: %v", extID, err) - } - } - delete(c.known, extID) - } +// LastWindow is the end of the last committed metering window. +func (c *Controller) LastWindow() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.lastWindow } -func tenantFor(vc discovery.VCluster, currency string) destinations.Tenant { - return destinations.Tenant{ - ExternalID: vc.ExternalID(), - DisplayName: vc.DisplayName(), - Currency: currency, - Metadata: map[string]string{ - "vcluster_name": vc.Name, - "vcluster_namespace": vc.Namespace, - "vcluster_uid": vc.UID, - "created_at": vc.CreatedAt.Format(time.RFC3339), - }, - CreatedAt: vc.CreatedAt, +// Reconcile refreshes tenant clusters and tenants and offboards tenants that +// have been gone for longer than OFFBOARD_GRACE. A discovery error changes +// nothing: an API hiccup must never look like deleted tenants. +func (c *Controller) Reconcile(ctx context.Context) { + clusters, err := c.disc.Discover(ctx) + if err != nil { + log.Printf("[controller] discovery failed, keeping previous view: %v", err) + c.tel.Add("vbilling_discovery_failures_total", "Failed tenant-cluster discovery runs.", nil, 1) + return + } + now := c.now() + tenants := c.tenantsFrom(clusters) + for _, t := range tenants { + c.tenants.Upsert(t) } -} -func (c *Controller) collectAndSend(ctx context.Context) { c.mu.Lock() - all := make([]*tracked, 0, len(c.known)) - for _, t := range c.known { - all = append(all, t) + c.clusters = clusters + c.discovered = true + for _, t := range tenants { + c.lastSeen[t.ID] = now } - c.mu.Unlock() - - if len(all) == 0 { - return + var gone []usage.Tenant + current := map[string]bool{} + for _, t := range tenants { + current[t.ID] = true } - - intervalHours := c.cfg.CollectionInterval.Hours() - now := time.Now() - var events []destinations.UsageEvent - - for _, t := range all { - m, err := c.collector.Collect(ctx, t.VCluster.Namespace) - if err != nil { - log.Printf("[controller] metrics collection failed for %s: %v", t.VCluster.Namespace, err) - continue - } - events = append(events, eventsForTenant(t.VCluster, m, intervalHours, now)...) - - c.mu.Lock() - if existing, ok := c.known[t.VCluster.ExternalID()]; ok { - existing.LastCollection = now + for id, seen := range c.lastSeen { + if !current[id] && now.Sub(seen) >= c.cfg.OffboardGrace { + t, _ := c.tenants.Get(id) + gone = append(gone, t) } - c.mu.Unlock() } + c.mu.Unlock() - if len(events) == 0 { - return - } - if err := c.dest.SendEvents(ctx, events); err != nil { - log.Printf("[controller] send events: %v", err) - } else { - log.Printf("[controller] sent %d billing events to %s", len(events), c.dest.Name()) + for _, t := range gone { + ok := true + for _, d := range c.dests { + if err := d.RemoveTenant(ctx, t); err != nil { + log.Printf("[controller] offboard %s from %s: %v (will retry)", t.ID, d.Name(), err) + ok = false + } + } + if ok { + log.Printf("[controller] tenant %s offboarded after %s without tenant clusters", t.ID, c.cfg.OffboardGrace) + c.tenants.Delete(t.ID) + c.mu.Lock() + delete(c.lastSeen, t.ID) + c.mu.Unlock() + } } + c.tel.Set("vbilling_tenant_clusters", "Tenant clusters being metered.", nil, float64(len(clusters))) + c.tel.Set("vbilling_tenants", "Billing tenants with at least one tenant cluster.", nil, float64(len(tenants))) } -// eventsForTenant builds the canonical usage events for one Tenant Cluster. -func eventsForTenant(vc discovery.VCluster, m *metrics.VClusterMetrics, intervalHours float64, now time.Time) []destinations.UsageEvent { - extID := vc.ExternalID() - uid := vc.UID - ts := now.Unix() - costMul := m.NodeCostMultiplier - - commonProps := func(extra map[string]any) map[string]any { - props := map[string]any{ - "vcluster_name": vc.Name, - "vcluster_namespace": vc.Namespace, +// tenantsFrom groups tenant clusters into billing tenants. +func (c *Controller) tenantsFrom(clusters []discovery.TenantCluster) []usage.Tenant { + src := discovery.TenantSource(c.cfg.TenantSource) + byID := map[string]*usage.Tenant{} + var ids []string + for i := range clusters { + cl := &clusters[i] + id := cl.TenantID(src) + t, ok := byID[id] + if !ok { + t = &usage.Tenant{ID: id, Region: c.cfg.Region, CreatedAt: cl.CreatedAt, Metadata: map[string]string{}, ProviderIDs: map[string]string{}} + byID[id] = t + ids = append(ids, id) + } + t.Clusters = append(t.Clusters, cl.ExternalID()) + if cl.CreatedAt.Before(t.CreatedAt) { + t.CreatedAt = cl.CreatedAt + } + first := func(cur *string, vals ...string) { + for _, v := range vals { + if *cur == "" && v != "" { + *cur = v + } + } + } + first(&t.DisplayName, cl.Meta("display-name"), cl.DisplayName) + first(&t.Email, cl.Meta("email")) + first(&t.Currency, cl.Meta("currency")) + first(&t.Project, cl.Project) + first(&t.Class, cl.Meta("tenant-class"), c.cfg.TenantClass) + first(&t.Plan, cl.Meta("plan")) + if v := cl.Meta("stripe-customer-id"); v != "" { + t.ProviderIDs["stripe"] = v + } + if v := cl.Meta("metronome-customer-id"); v != "" { + t.ProviderIDs["metronome"] = v } - for k, v := range extra { - props[k] = v + if v := cl.Meta("cost-center"); v != "" { + t.Metadata["cost_center"] = v } - return props } - - var events []destinations.UsageEvent - add := func(metric, suffix string, value float64, props map[string]any) { - events = append(events, destinations.UsageEvent{ - TenantExternalID: extID, - MetricCode: metric, - Value: roundFloat(value), - Unit: unitFor(metric), - Timestamp: now, - TransactionID: fmt.Sprintf("%s-%s-%d", uid, suffix, ts), - Properties: props, - }) + out := make([]usage.Tenant, 0, len(ids)) + sort.Strings(ids) + for _, id := range ids { + t := byID[id] + sort.Strings(t.Clusters) + if t.DisplayName == "" { + t.DisplayName = id + if len(t.Clusters) == 1 && strings.HasPrefix(id, "vcluster-") { + t.DisplayName = "Tenant cluster " + strings.TrimPrefix(id, "vcluster-") + } + } + t.Metadata["region"] = t.Region + t.Metadata["clusters"] = strings.Join(t.Clusters, ",") + if t.Project != "" { + t.Metadata["project"] = t.Project + } + if len(t.ProviderIDs) == 0 { + t.ProviderIDs = nil + } + out = append(out, *t) } + return out +} - if m.CPUCores > 0 { - add(destinations.MetricCPUCoreHours, "cpu", m.CPUCores*intervalHours*costMul, commonProps(map[string]any{ - "raw_cpu_cores": roundFloat(m.CPUCores), - "cost_multiplier": roundFloat(costMul), - })) +// CollectDue meters every closed window since the watermark (bounded by +// MAX_BACKFILL) and commits each one atomically to the ledger. +func (c *Controller) CollectDue(ctx context.Context) { + if !c.Ready() { + return } - if m.MemoryGB() > 0 { - add(destinations.MetricMemoryGBHours, "mem", m.MemoryGB()*intervalHours*costMul, commonProps(map[string]any{ - "raw_memory_gb": roundFloat(m.MemoryGB()), - })) + size := c.cfg.CollectionInterval + latestEnd := c.now().UTC().Truncate(size) + firstEnd := latestEnd + note := "" + if wm, ok := c.spool.Watermark(spool.SourceCollector); ok { + if !latestEnd.After(wm) { + return + } + firstEnd = wm.Add(size) + // The oldest window we backfill starts MAX_BACKFILL before the latest end. + if earliest := latestEnd.Add(-c.cfg.MaxBackfill).Truncate(size).Add(size); firstEnd.Before(earliest) { + gap := earliest.Sub(firstEnd) + note = "gap: " + gap.String() + " older than MAX_BACKFILL not metered" + log.Printf("[controller] %d window(s) before %s are older than MAX_BACKFILL=%s and will not be metered", + int(gap/size), earliest.Add(-size).Format(time.RFC3339), c.cfg.MaxBackfill) + firstEnd = earliest + } } - if m.StorageGB() > 0 { - add(destinations.MetricStorageGBHours, "stor", m.StorageGB()*intervalHours, commonProps(map[string]any{ - "raw_storage_gb": roundFloat(m.StorageGB()), - })) + var windows []metrics.Window + for end := firstEnd; !end.After(latestEnd); end = end.Add(size) { + windows = append(windows, metrics.Window{Start: end.Add(-size), End: end, Latest: end.Equal(latestEnd)}) } - add(destinations.MetricInstanceHours, "inst", intervalHours, commonProps(nil)) - - for gpuType, count := range m.GPUCountByType() { - add(destinations.MetricGPUHours, "gpu-"+sanitize(gpuType), float64(count)*intervalHours, commonProps(map[string]any{ - "gpu_count": count, - "gpu_type": gpuType, - })) + src := discovery.TenantSource(c.cfg.TenantSource) + var targets []metrics.Target + for _, cl := range c.Clusters() { + t, _ := c.tenants.Get(cl.TenantID(src)) + targets = append(targets, metrics.Target{Cluster: cl, Tenant: t.ID, Project: cl.Project, TenantClass: t.Class}) } - if len(m.GPUUtilization) > 0 { - var sum float64 - for _, g := range m.GPUUtilization { - sum += g.Utilization + started := time.Now() + byWindow, st, err := c.coll.Collect(ctx, targets, windows) + if err != nil { + log.Printf("[controller] metering failed for %d window(s), will retry: %v", len(windows), err) + c.tel.Add("vbilling_collection_failures_total", "Failed metering runs (windows are retried).", nil, 1) + return + } + total := 0 + for i, w := range windows { + n := "" + if i == 0 { + n = note + } + evs := byWindow[w.End] + if err := c.spool.AppendWindow(spool.SourceCollector, w.End, evs, n); err != nil { + log.Printf("[controller] commit window %s: %v", w.End.Format(time.RFC3339), err) + return } - avg := sum / float64(len(m.GPUUtilization)) - add(destinations.MetricGPUUtilization, "gpuutil", avg*intervalHours, commonProps(map[string]any{ - "avg_utilization_pct": roundFloat(avg), - "gpu_count": len(m.GPUUtilization), - })) + total += len(evs) + c.mu.Lock() + c.lastWindow = w.End + c.mu.Unlock() } - - if m.NetworkEgressGB() > 0 { - add(destinations.MetricNetworkEgressGB, "net", m.NetworkEgressGB(), commonProps(nil)) + c.fillTenantGaps(ctx, st, windows, targets) + if len(windows) > 1 { + log.Printf("[controller] backfilled %d window(s) after a gap", len(windows)-1) + c.tel.Add("vbilling_backfilled_windows_total", "Windows metered after a gap (restart or outage).", nil, float64(len(windows)-1)) } + log.Printf("[controller] window %s: %d events for %d tenant cluster(s) (%d pods, %d GPU pods, %d nodes down)", + latestEnd.Format("15:04:05"), total, len(targets), st.Pods, st.GPUPods, st.NodesDown) + c.tel.Add("vbilling_events_metered_total", "Usage events written to the ledger by the collector.", nil, float64(total)) + c.tel.Set("vbilling_last_window_end_timestamp_seconds", "End of the last committed metering window.", nil, float64(latestEnd.Unix())) + c.tel.Set("vbilling_collection_duration_seconds", "Duration of the last metering run.", nil, time.Since(started).Seconds()) + c.tel.Set("vbilling_nodes_down", "Nodes excluded from billing as not ready or unhealthy.", nil, float64(st.NodesDown)) +} - if m.LoadBalancerCount > 0 { - add(destinations.MetricLBHours, "lb", float64(m.LoadBalancerCount)*intervalHours, commonProps(map[string]any{ - "lb_count": m.LoadBalancerCount, - })) +// fillTenantGaps records tenant clusters whose own API failed in this run +// and, once it answers again, meters the windows it missed. Those events go +// to the ledger as late events, deduplicated by ID. A gap is the span of +// failed windows only (first|last window end), so a fill repeated after a +// crash never touches windows that were committed normally. Windows older +// than MAX_BACKFILL are not filled. +func (c *Controller) fillTenantGaps(ctx context.Context, st metrics.Stats, windows []metrics.Window, targets []metrics.Target) { + failed := map[string]bool{} + first, last := windows[0].End, windows[len(windows)-1].End + for _, id := range st.TenantAPIFailed { + failed[id] = true + from := first + if g, ok := parseGap(c.gaps, id); ok { + from = g[0] + } else { + log.Printf("[controller] %s: tenant API unreachable from window %s, will fill once it answers", id, first.Format(time.RFC3339)) + } + _ = c.gaps.Set(id, from.Format(time.RFC3339)+"|"+last.Format(time.RFC3339)) } - - if m.HasPrivateNodes() { - for gpuType, count := range m.PrivateNodeGPUsByType() { - add(destinations.MetricGPUHours, "pn-gpu-"+sanitize(gpuType), float64(count)*intervalHours, commonProps(map[string]any{ - "gpu_count": count, - "gpu_type": gpuType, - "billing_mode": "private_node", - })) + filler, ok := c.coll.(tenantFiller) + if !ok { + return + } + size := c.cfg.CollectionInterval + earliest := last.Add(-c.cfg.MaxBackfill) + for _, t := range targets { + id := t.Cluster.ExternalID() + g, ok := parseGap(c.gaps, id) + if !ok || failed[id] { + continue + } + var missed []metrics.Window + for end := g[0]; !end.After(g[1]); end = end.Add(size) { + if end.After(earliest) { + missed = append(missed, metrics.Window{Start: end.Add(-size), End: end}) + } + } + if len(missed) == 0 { + _ = c.gaps.Delete(id) + continue } - if m.PrivateNodeTotalCPU() > 0 { - add(destinations.MetricCPUCoreHours, "pn-cpu", float64(m.PrivateNodeTotalCPU())*intervalHours, commonProps(map[string]any{ - "raw_cpu_cores": roundFloat(float64(m.PrivateNodeTotalCPU())), - "billing_mode": "private_node", - })) + byWindow, fst, err := filler.CollectTenantAPI(ctx, []metrics.Target{t}, missed) + if err != nil || len(fst.TenantAPIFailed) > 0 { + continue // still unreachable: try again next window } - if m.PrivateNodeTotalMemoryGB() > 0 { - add(destinations.MetricMemoryGBHours, "pn-mem", m.PrivateNodeTotalMemoryGB()*intervalHours, commonProps(map[string]any{ - "raw_memory_gb": roundFloat(m.PrivateNodeTotalMemoryGB()), - "billing_mode": "private_node", - })) + var evs []usage.Event + for _, w := range missed { + evs = append(evs, byWindow[w.End]...) + } + accepted, dups, err := c.spool.AppendIngest(SourceTenantFill, evs) + if err != nil { + log.Printf("[controller] %s: fill %d window(s): %v", id, len(missed), err) + continue } - add(destinations.MetricPrivateNodeHours, "pn-hours", float64(m.PrivateNodeCount())*intervalHours, commonProps(map[string]any{ - "node_count": m.PrivateNodeCount(), - "billing_mode": "private_node", - })) + _ = c.gaps.Delete(id) + log.Printf("[controller] %s: filled %d window(s) missed during a tenant API outage (%d events, %d already in the ledger)", + id, len(missed), len(accepted), len(dups)) + c.tel.Add("vbilling_tenant_api_filled_windows_total", "Windows filled after a tenant API outage.", nil, float64(len(missed))) } - - return events } -func unitFor(metric string) string { - switch metric { - case destinations.MetricCPUCoreHours: - return "core-hours" - case destinations.MetricMemoryGBHours, destinations.MetricStorageGBHours: - return "gb-hours" - case destinations.MetricInstanceHours, destinations.MetricGPUHours, - destinations.MetricLBHours, destinations.MetricPrivateNodeHours: - return "hours" - case destinations.MetricGPUUtilization: - return "util-hours" - case destinations.MetricNetworkEgressGB: - return "gb" - default: - return "" +// parseGap reads a "first|last" window-end span. +func parseGap(store *kv.Store, id string) ([2]time.Time, bool) { + v, ok := store.Get(id) + if !ok { + return [2]time.Time{}, false } -} - -func roundFloat(f float64) float64 { - return float64(int64(f*1000000)) / 1000000 -} - -func sanitize(s string) string { - out := make([]byte, 0, len(s)) - for i := 0; i < len(s); i++ { - c := s[i] - if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' { - out = append(out, c) - } + a, b, found := strings.Cut(v, "|") + from, err1 := time.Parse(time.RFC3339, a) + to, err2 := time.Parse(time.RFC3339, b) + if !found || err1 != nil || err2 != nil || to.Before(from) { + _ = store.Delete(id) + return [2]time.Time{}, false } - return string(out) + return [2]time.Time{from, to}, true } diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go new file mode 100644 index 0000000..775ea57 --- /dev/null +++ b/internal/controller/controller_test.go @@ -0,0 +1,331 @@ +package controller + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/metrics" + "github.com/vclusterlabs-experiments/vbilling/internal/pipeline" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +type fakeDisc struct { + clusters []discovery.TenantCluster + err error +} + +func (f *fakeDisc) Discover(context.Context) ([]discovery.TenantCluster, error) { + return f.clusters, f.err +} + +type fakeColl struct { + mu sync.Mutex + calls [][]metrics.Window + targets []metrics.Target + err error +} + +func (f *fakeColl) Collect(_ context.Context, targets []metrics.Target, windows []metrics.Window) (map[time.Time][]usage.Event, metrics.Stats, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, windows) + f.targets = targets + if f.err != nil { + return nil, metrics.Stats{}, f.err + } + out := map[time.Time][]usage.Event{} + for _, w := range windows { + for _, tg := range targets { + e := usage.Event{Tenant: tg.Tenant, Metric: usage.MetricInstanceHours, Quantity: w.End.Sub(w.Start).Hours(), + WindowStart: w.Start, WindowEnd: w.End, Dimensions: map[string]string{usage.DimTenantCluster: tg.Cluster.ExternalID()}} + e.Finalize() + out[w.End] = append(out[w.End], e) + } + } + return out, metrics.Stats{}, nil +} + +type fakeDest struct { + mu sync.Mutex + removed []string + failRm int +} + +func (d *fakeDest) Name() string { return "fake" } +func (d *fakeDest) Bootstrap(context.Context, []usage.MetricDef) error { return nil } +func (d *fakeDest) EnsureTenant(context.Context, usage.Tenant) error { return nil } +func (d *fakeDest) SendEvents(context.Context, []usage.Event) error { return nil } +func (d *fakeDest) RemoveTenant(_ context.Context, t usage.Tenant) error { + d.mu.Lock() + defer d.mu.Unlock() + if d.failRm > 0 { + d.failRm-- + return errors.New("backend down") + } + d.removed = append(d.removed, t.ID) + return nil +} + +var t0 = time.Date(2026, 10, 5, 10, 0, 30, 0, time.UTC) + +func cluster(ns, name string, meta map[string]string) discovery.TenantCluster { + return discovery.TenantCluster{Name: name, Namespace: ns, Ready: true, CreatedAt: t0.Add(-time.Hour), Annotations: meta} +} + +type harness struct { + c *Controller + disc *fakeDisc + coll *fakeColl + dest *fakeDest + sp *spool.Spool + reg *pipeline.TenantRegistry + now time.Time +} + +func newHarness(t *testing.T, mutate ...func(*config.Config)) *harness { + t.Helper() + cfg := &config.Config{CollectionInterval: time.Minute, ReconcileInterval: 30 * time.Second, MaxBackfill: time.Hour, + OffboardGrace: time.Hour, Region: "ap-southeast-2", TenantSource: "cluster", Adapters: []string{"fake"}} + for _, m := range mutate { + m(cfg) + } + sp, err := spool.Open(t.TempDir(), spool.Options{NoSync: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { sp.Close() }) + h := &harness{disc: &fakeDisc{}, coll: &fakeColl{}, dest: &fakeDest{}, sp: sp, reg: pipeline.NewTenantRegistry(), now: t0} + h.c = New(cfg, h.disc, h.coll, sp, h.reg, []destinations.Destination{h.dest}, nil) + h.c.now = func() time.Time { return h.now } + return h +} + +func TestTenantsGroupClustersByLabelAndProject(t *testing.T) { + h := newHarness(t) + h.disc.clusters = []discovery.TenantCluster{ + cluster("team-a", "train", map[string]string{discovery.AnnotationPrefix + "tenant": "acme", discovery.AnnotationPrefix + "display-name": "Acme AI", + discovery.AnnotationPrefix + "stripe-customer-id": "cus_123", discovery.AnnotationPrefix + "plan": "gpu-pro"}), + cluster("team-b", "infer", map[string]string{discovery.AnnotationPrefix + "tenant": "acme", discovery.AnnotationPrefix + "email": "billing@acme.test"}), + cluster("solo", "dev", nil), + } + h.c.Reconcile(context.Background()) + all := h.reg.All() + if len(all) != 2 { + t.Fatalf("tenants: %+v", all) + } + acme, _ := h.reg.Get("acme") + if acme.DisplayName != "Acme AI" || acme.Email != "billing@acme.test" || acme.Plan != "gpu-pro" || acme.ProviderIDs["stripe"] != "cus_123" || + strings.Join(acme.Clusters, ",") != "vcluster-team-a-train,vcluster-team-b-infer" || acme.Region != "ap-southeast-2" { + t.Fatalf("acme: %+v", acme) + } + if solo, ok := h.reg.Get("vcluster-solo-dev"); !ok || solo.DisplayName != "Tenant cluster solo-dev" { + t.Fatalf("default per-cluster tenant: %+v", solo) + } + + // Project mode: clusters in one vCluster Platform project share a customer. + p := newHarness(t, func(c *config.Config) { c.TenantSource = "project" }) + a, b := cluster("loft-p1-v-a", "a", nil), cluster("loft-p1-v-b", "b", nil) + a.Project, b.Project = "research", "research" + p.disc.clusters = []discovery.TenantCluster{a, b} + p.c.Reconcile(context.Background()) + if got := p.reg.All(); len(got) != 1 || got[0].ID != "project-research" || len(got[0].Clusters) != 2 { + t.Fatalf("project tenants: %+v", got) + } +} + +func TestDiscoveryFailureNeverOffboards(t *testing.T) { + h := newHarness(t) + h.disc.clusters = []discovery.TenantCluster{cluster("team-a", "train", nil)} + h.c.Reconcile(context.Background()) + + h.disc.err = errors.New("apiserver timeout") + h.now = h.now.Add(3 * time.Hour) + h.c.Reconcile(context.Background()) + if len(h.dest.removed) != 0 || len(h.c.Clusters()) != 1 { + t.Fatal("a failed discovery must keep the previous view") + } +} + +func TestOffboardingWaitsForGraceAndRetries(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + h.disc.clusters = []discovery.TenantCluster{cluster("team-a", "train", nil)} + h.c.Reconcile(ctx) + + h.disc.clusters = nil // tenant cluster deleted + h.now = h.now.Add(30 * time.Minute) + h.c.Reconcile(ctx) + if len(h.dest.removed) != 0 { + t.Fatal("offboarded before the grace period") + } + h.dest.failRm = 1 + h.now = h.now.Add(31 * time.Minute) + h.c.Reconcile(ctx) // backend fails: retried next time + h.c.Reconcile(ctx) + if strings.Join(h.dest.removed, ",") != "vcluster-team-a-train" { + t.Fatalf("removed = %v", h.dest.removed) + } + h.c.Reconcile(ctx) + if len(h.dest.removed) != 1 { + t.Fatal("tenant offboarded twice") + } + if _, ok := h.reg.Get("vcluster-team-a-train"); ok { + t.Fatal("offboarded tenant still registered") + } +} + +func TestCollectBackfillsGapsAndRetriesFailures(t *testing.T) { + h := newHarness(t, func(c *config.Config) { c.MaxBackfill = 10 * time.Minute }) + ctx := context.Background() + h.disc.clusters = []discovery.TenantCluster{cluster("team-a", "train", nil)} + h.c.Reconcile(ctx) + + h.c.CollectDue(ctx) // first run ever: only the last closed window + if len(h.coll.calls) != 1 || len(h.coll.calls[0]) != 1 || !h.coll.calls[0][0].End.Equal(t0.Truncate(time.Minute)) { + t.Fatalf("first collection windows: %+v", h.coll.calls) + } + h.c.CollectDue(ctx) // same window: nothing to do + if len(h.coll.calls) != 1 { + t.Fatal("window collected twice") + } + + h.now = h.now.Add(5 * time.Minute) // vBilling was down for 5 minutes + h.c.CollectDue(ctx) + last := h.coll.calls[len(h.coll.calls)-1] + if len(last) != 5 || !last[4].Latest || last[0].Latest { + t.Fatalf("expected 5 windows with only the last live, got %+v", last) + } + + h.coll.err = errors.New("apiserver unavailable") + h.now = h.now.Add(2 * time.Minute) + h.c.CollectDue(ctx) + h.coll.err = nil + h.c.CollectDue(ctx) + last = h.coll.calls[len(h.coll.calls)-1] + if len(last) != 2 { + t.Fatalf("failed windows must be retried, got %d windows", len(last)) + } + + h.now = h.now.Add(3 * time.Hour) // longer than MAX_BACKFILL + h.c.CollectDue(ctx) + last = h.coll.calls[len(h.coll.calls)-1] + if len(last) != 10 { + t.Fatalf("backfill must be capped at MAX_BACKFILL (10 windows), got %d", len(last)) + } + wm, _ := h.sp.Watermark(spool.SourceCollector) + if !wm.Equal(h.now.Truncate(time.Minute)) { + t.Fatalf("watermark %s, want %s", wm, h.now.Truncate(time.Minute)) + } + if h.coll.targets[0].Tenant != "vcluster-team-a-train" { + t.Fatalf("collector target tenant: %+v", h.coll.targets[0]) + } +} + +// gapColl fails the tenant API part on demand and meters private nodes when +// asked to fill. +type gapColl struct { + fakeColl + tenantDown bool + fills [][]metrics.Window +} + +func (g *gapColl) Collect(ctx context.Context, targets []metrics.Target, windows []metrics.Window) (map[time.Time][]usage.Event, metrics.Stats, error) { + out, st, err := g.fakeColl.Collect(ctx, targets, windows) + if g.tenantDown { + for _, tg := range targets { + st.TenantAPIFailed = append(st.TenantAPIFailed, tg.Cluster.ExternalID()) + } + } + return out, st, err +} + +func (g *gapColl) CollectTenantAPI(_ context.Context, targets []metrics.Target, windows []metrics.Window) (map[time.Time][]usage.Event, metrics.Stats, error) { + g.fills = append(g.fills, windows) + out := map[time.Time][]usage.Event{} + if g.tenantDown { + return out, metrics.Stats{TenantAPIFailed: []string{targets[0].Cluster.ExternalID()}}, nil + } + for _, w := range windows { + e := usage.Event{Tenant: targets[0].Tenant, Metric: usage.MetricPrivateNodeHours, Quantity: w.End.Sub(w.Start).Hours(), ResourceID: "gpu-1", + WindowStart: w.Start, WindowEnd: w.End, Dimensions: map[string]string{usage.DimTenantCluster: targets[0].Cluster.ExternalID(), usage.DimBillingMode: "private_node"}} + e.Finalize() + out[w.End] = append(out[w.End], e) + } + return out, metrics.Stats{}, nil +} + +func TestTenantAPIOutageWindowsAreFilledExactlyOnce(t *testing.T) { + h := newHarness(t) + gc := &gapColl{tenantDown: true} + h.c.coll = gc + ctx := context.Background() + h.disc.clusters = []discovery.TenantCluster{cluster("team-p", "private", nil)} + h.c.Reconcile(ctx) + id := "vcluster-team-p-private" + + h.c.CollectDue(ctx) // W1: tenant API down + h.now = h.now.Add(2 * time.Minute) + h.c.CollectDue(ctx) // W2, W3: still down + w1, w3 := t0.Truncate(time.Minute), t0.Truncate(time.Minute).Add(2*time.Minute) + if span, ok := h.c.gaps.Get(id); !ok || span != w1.Format(time.RFC3339)+"|"+w3.Format(time.RFC3339) { + t.Fatalf("gap = %q, %v; want the failed windows W1..W3", span, ok) + } + if len(gc.fills) != 0 { + t.Fatal("no fill may run while the tenant API is still down") + } + + gc.tenantDown = false + h.now = h.now.Add(time.Minute) + h.c.CollectDue(ctx) // W4 live; W1..W3 filled + if len(gc.fills) != 1 || len(gc.fills[0]) != 3 { + t.Fatalf("fill windows = %+v, want the 3 windows missed", gc.fills) + } + if _, ok := h.c.gaps.Get(id); ok { + t.Fatal("gap must be cleared once filled") + } + countPrivate := func() int { + n := 0 + recs, err := h.sp.Read(0, 10000) + if err != nil { + t.Fatal(err) + } + for _, r := range recs { + if r.Event != nil && r.Event.Metric == usage.MetricPrivateNodeHours { + n++ + } + } + return n + } + if n := countPrivate(); n != 3 { + t.Fatalf("ledger has %d filled private-node events, want 3", n) + } + // A repeated fill (for example after a crash between append and clearing + // the gap) must not bill twice. + _ = h.c.gaps.Set(id, w1.Format(time.RFC3339)+"|"+w3.Format(time.RFC3339)) + h.now = h.now.Add(time.Minute) + h.c.CollectDue(ctx) + if n := countPrivate(); n != 3 { + t.Fatalf("after a repeated fill the ledger has %d private-node events, want still 3", n) + } +} + +func TestTenantAPIGapsSurviveRestart(t *testing.T) { + dir := t.TempDir() + cfg := &config.Config{CollectionInterval: time.Minute, MaxBackfill: time.Hour, DataDir: dir} + a := New(cfg, &fakeDisc{}, &fakeColl{}, nil, pipeline.NewTenantRegistry(), nil, nil) + if err := a.gaps.Set("vcluster-team-p-private", "2026-10-05T10:00:00Z|2026-10-05T10:02:00Z"); err != nil { + t.Fatal(err) + } + b := New(cfg, &fakeDisc{}, &fakeColl{}, nil, pipeline.NewTenantRegistry(), nil, nil) + if g, ok := parseGap(b.gaps, "vcluster-team-p-private"); !ok || !g[1].Equal(time.Date(2026, 10, 5, 10, 2, 0, 0, time.UTC)) { + t.Fatalf("gap after restart = %v, %v", g, ok) + } +} diff --git a/internal/destinations/destination.go b/internal/destinations/destination.go index 328d04e..57c5490 100644 --- a/internal/destinations/destination.go +++ b/internal/destinations/destination.go @@ -1,62 +1,119 @@ // Package destinations defines the billing-adapter contract. // -// vBilling collects usage metrics from Tenant Clusters and ships them to a -// pluggable billing backend. Each backend (Lago, Metronome, Stripe Meters, -// OpenMeter, custom) implements Destination. The active backend is chosen via -// the ADAPTER env var; default is "lago". +// vBilling meters tenant clusters, writes canonical usage events to its +// durable ledger, and fans them out to one or more destinations: billing +// engines (Stripe, Metronome, Lago), data platforms (webhook), or a dry-run +// sink (noop). Destinations are selected with ADAPTERS=a,b,c. package destinations import ( "context" + "errors" "fmt" + "sort" "sync" "time" "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" ) -// Canonical metric codes emitted by the controller. Adapters translate these -// to whatever the destination expects. -const ( - MetricCPUCoreHours = "vcluster_cpu_core_hours" - MetricMemoryGBHours = "vcluster_memory_gb_hours" - MetricStorageGBHours = "vcluster_storage_gb_hours" - MetricInstanceHours = "vcluster_instance_hours" - MetricGPUHours = "vcluster_gpu_hours" - MetricGPUUtilization = "vcluster_gpu_utilization" - MetricNetworkEgressGB = "vcluster_network_egress_gb" - MetricLBHours = "vcluster_lb_hours" - MetricPrivateNodeHours = "vcluster_private_node_hours" -) +// Destination is the contract every adapter implements. All methods must be +// idempotent: vBilling retries them freely. +type Destination interface { + Name() string + // Bootstrap creates what the backend needs to accept the catalog + // (meters, billable metrics, plan charges). + Bootstrap(ctx context.Context, metrics []usage.MetricDef) error + // EnsureTenant creates or updates the billing customer. + EnsureTenant(ctx context.Context, t usage.Tenant) error + // RemoveTenant runs once a tenant has had no tenant clusters for the + // offboarding grace period. Adapters decide what that means; deleting + // customers with open invoices is never the right answer. + RemoveTenant(ctx context.Context, t usage.Tenant) error + // SendEvents delivers a batch. Return Permanent(err) only when the + // backend will never accept these events as sent (bad payload); every + // other error is retried with backoff. + SendEvents(ctx context.Context, events []usage.Event) error +} -// Tenant is a billable entity (one Tenant Cluster). -type Tenant struct { - ExternalID string - DisplayName string - Currency string - Metadata map[string]string - CreatedAt time.Time +// EventFilter is implemented by destinations that only take some metrics +// (billing engines skip informational metrics they have no meter for). +type EventFilter interface { + Accepts(metric string) bool } -// UsageEvent is one metered observation for a tenant. Adapters serialize this -// into the destination's wire format. -type UsageEvent struct { - TenantExternalID string - MetricCode string - Value float64 - Unit string - Timestamp time.Time - TransactionID string - Properties map[string]any +// Reconciler is implemented by destinations that can report what they +// recorded, so vBilling can compare it with its own ledger. +type Reconciler interface { + RecordedTotals(ctx context.Context, q TotalsQuery) ([]Total, error) } -// Destination is the contract every billing adapter implements. -type Destination interface { - Name() string - Bootstrap(ctx context.Context) error - EnsureTenant(ctx context.Context, t Tenant) error - RemoveTenant(ctx context.Context, externalID string) error - SendEvents(ctx context.Context, events []UsageEvent) error +// TotalsQuery asks a backend for recorded usage. +type TotalsQuery struct { + From, To time.Time + Tenants []usage.Tenant + Metrics []string +} + +// Total is one tenant+metric aggregate. +type Total struct { + Tenant string `json:"tenant"` + Metric string `json:"metric"` + Quantity float64 `json:"quantity"` +} + +// CustomerResolver maps a backend customer ID (cus_..., a Metronome UUID) +// back to a vBilling tenant, so inbound billing webhooks can be acted on. +type CustomerResolver interface { + TenantForCustomer(ctx context.Context, customerID string) (string, bool) +} + +// PermanentError marks a payload the backend will never accept. +type PermanentError struct{ Err error } + +func (e *PermanentError) Error() string { return "permanent: " + e.Err.Error() } +func (e *PermanentError) Unwrap() error { return e.Err } + +// PartialError reports events a backend permanently rejected inside an +// otherwise accepted batch. Every event not listed was accepted. Adapters +// return it only when nothing in the batch needs a retry. +type PartialError struct { + Rejected map[string]string // event ID -> reason +} + +func (e *PartialError) Error() string { + return fmt.Sprintf("%d event(s) rejected", len(e.Rejected)) +} + +// Permanent wraps err as non-retryable. +func Permanent(err error) error { + if err == nil { + return nil + } + return &PermanentError{Err: err} +} + +// IsPermanent reports whether err (or anything it wraps) is permanent. +func IsPermanent(err error) bool { + var p *PermanentError + return errors.As(err, &p) +} + +// Accepts applies a destination's EventFilter, accepting everything when the +// destination has none. +func Accepts(d Destination, metric string) bool { + if f, ok := d.(EventFilter); ok { + return f.Accepts(metric) + } + return true +} + +// BillableOnly is an EventFilter for billing engines: builtin billable +// metrics plus operator-declared custom metrics. +func BillableOnly(metric string) bool { + def, ok := usage.Lookup(metric) + return ok && def.Billable } // Factory builds a Destination from config. @@ -81,17 +138,40 @@ func New(name string, cfg *config.Config) (Destination, error) { f, ok := registry[name] mu.RUnlock() if !ok { - return nil, fmt.Errorf("unknown billing adapter %q (known: %v)", name, registered()) + return nil, fmt.Errorf("unknown billing adapter %q (known: %v)", name, Registered()) } return f(cfg) } -func registered() []string { +// NewAll constructs every adapter in names, failing on the first error. +func NewAll(names []string, cfg *config.Config) ([]Destination, error) { + seen := map[string]bool{} + var out []Destination + for _, n := range names { + if seen[n] { + return nil, fmt.Errorf("adapter %q listed twice", n) + } + seen[n] = true + d, err := New(n, cfg) + if err != nil { + return nil, fmt.Errorf("adapter %s: %w", n, err) + } + out = append(out, d) + } + if len(out) == 0 { + return nil, errors.New("no adapters configured") + } + return out, nil +} + +// Registered lists known adapter names. +func Registered() []string { mu.RLock() defer mu.RUnlock() names := make([]string, 0, len(registry)) for n := range registry { names = append(names, n) } + sort.Strings(names) return names } diff --git a/internal/destinations/destination_test.go b/internal/destinations/destination_test.go index 7007c3c..bba66f6 100644 --- a/internal/destinations/destination_test.go +++ b/internal/destinations/destination_test.go @@ -2,19 +2,22 @@ package destinations import ( "context" + "errors" + "fmt" "strings" "testing" "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" ) type fakeDest struct{ name string } -func (f *fakeDest) Name() string { return f.name } -func (f *fakeDest) Bootstrap(context.Context) error { return nil } -func (f *fakeDest) EnsureTenant(context.Context, Tenant) error { return nil } -func (f *fakeDest) RemoveTenant(context.Context, string) error { return nil } -func (f *fakeDest) SendEvents(context.Context, []UsageEvent) error { return nil } +func (f *fakeDest) Name() string { return f.name } +func (f *fakeDest) Bootstrap(context.Context, []usage.MetricDef) error { return nil } +func (f *fakeDest) EnsureTenant(context.Context, usage.Tenant) error { return nil } +func (f *fakeDest) RemoveTenant(context.Context, usage.Tenant) error { return nil } +func (f *fakeDest) SendEvents(context.Context, []usage.Event) error { return nil } func TestRegistryReturnsRegisteredAdapter(t *testing.T) { Register("test-fake", func(*config.Config) (Destination, error) { @@ -28,6 +31,9 @@ func TestRegistryReturnsRegisteredAdapter(t *testing.T) { if d.Name() != "test-fake" { t.Errorf("got Name=%q, want %q", d.Name(), "test-fake") } + if _, err := NewAll([]string{"test-fake", "test-fake"}, &config.Config{}); err == nil { + t.Error("duplicate adapters must be rejected") + } } func TestRegistryReturnsErrorForUnknownAdapter(t *testing.T) { @@ -39,3 +45,35 @@ func TestRegistryReturnsErrorForUnknownAdapter(t *testing.T) { t.Errorf("error %q does not mention unknown adapter", err.Error()) } } + +func TestStatusErrorClassification(t *testing.T) { + for status, permanent := range map[int]bool{400: true, 413: true, 422: true, 401: false, 403: false, 404: false, 409: false, 429: false, 500: false, 503: false} { + err := StatusError("POST", "/x", status, []byte("body")) + if IsPermanent(err) != permanent { + t.Errorf("HTTP %d permanent=%v, want %v", status, IsPermanent(err), permanent) + } + var he *HTTPError + if !errors.As(err, &he) || he.Status != status { + t.Errorf("HTTP %d not unwrappable to HTTPError", status) + } + } + if !IsPermanent(fmt.Errorf("wrapped: %w", Permanent(errors.New("x")))) { + t.Error("wrapped permanent error not detected") + } +} + +func TestBillableOnly(t *testing.T) { + if !BillableOnly(usage.MetricGPUHours) || BillableOnly(usage.MetricGPUUtilization) || BillableOnly("unknown_metric") { + t.Error("BillableOnly classification wrong") + } +} + +func TestSentCacheEvictsOldest(t *testing.T) { + c := NewSentCache(2) + c.Add("a") + c.Add("b") + c.Add("c") + if c.Has("a") || !c.Has("b") || !c.Has("c") { + t.Error("cache did not evict the oldest entry") + } +} diff --git a/internal/destinations/httpx.go b/internal/destinations/httpx.go new file mode 100644 index 0000000..6bc3374 --- /dev/null +++ b/internal/destinations/httpx.go @@ -0,0 +1,44 @@ +package destinations + +import ( + "fmt" + "net/http" + "strings" + "time" +) + +// HTTPError is a non-2xx response from a backend API. +type HTTPError struct { + Method string + URL string + Status int + Body string +} + +func (e *HTTPError) Error() string { + body := strings.TrimSpace(e.Body) + if len(body) > 512 { + body = body[:512] + "..." + } + return fmt.Sprintf("%s %s: HTTP %d: %s", e.Method, e.URL, e.Status, body) +} + +// StatusError classifies a non-2xx response. Payload rejections (400, 413, +// 422) are permanent so the dispatcher can isolate the bad event. Auth +// failures, missing endpoints, rate limits and server errors are retried: +// they are configuration or availability problems, and dead-lettering +// every event because of an expired API key would lose revenue. +func StatusError(method, url string, status int, body []byte) error { + err := &HTTPError{Method: method, URL: url, Status: status, Body: string(body)} + switch status { + case http.StatusBadRequest, http.StatusRequestEntityTooLarge, http.StatusUnprocessableEntity: + return Permanent(err) + default: + return err + } +} + +// NewHTTPClient returns the client adapters share. +func NewHTTPClient() *http.Client { + return &http.Client{Timeout: 30 * time.Second} +} diff --git a/internal/destinations/lago/adapter.go b/internal/destinations/lago/adapter.go index 1603490..9f36224 100644 --- a/internal/destinations/lago/adapter.go +++ b/internal/destinations/lago/adapter.go @@ -2,17 +2,20 @@ package lago import ( "context" + "errors" "fmt" "log" + "strings" "github.com/vclusterlabs-experiments/vbilling/internal/config" "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" ) func init() { destinations.Register("lago", func(cfg *config.Config) (destinations.Destination, error) { if cfg.LagoAPIKey == "" { - return nil, fmt.Errorf("LAGO_API_KEY is required when ADAPTER=lago") + return nil, fmt.Errorf("LAGO_API_KEY is required when the lago adapter is enabled") } return &Adapter{ client: NewClient(cfg.LagoAPIURL, cfg.LagoAPIKey), @@ -29,77 +32,109 @@ type Adapter struct { func (a *Adapter) Name() string { return "lago" } -func (a *Adapter) Bootstrap(ctx context.Context) error { - return Bootstrap(ctx, a.client, a.cfg) +// Accepts billable metrics plus GPU utilization, which v0.1 created as a +// Lago billable metric and existing plans may still reference. +func (a *Adapter) Accepts(metric string) bool { + return destinations.BillableOnly(metric) || metric == MetricGPUUtilization } -func (a *Adapter) EnsureTenant(ctx context.Context, t destinations.Tenant) error { +func (a *Adapter) Bootstrap(ctx context.Context, catalog []usage.MetricDef) error { + return Bootstrap(ctx, a.client, a.cfg, catalog) +} + +func (a *Adapter) EnsureTenant(ctx context.Context, t usage.Tenant) error { meta := make([]CustomerMeta, 0, len(t.Metadata)) for k, v := range t.Metadata { meta = append(meta, CustomerMeta{Key: k, Value: v}) } + currency := t.Currency + if currency == "" { + currency = a.cfg.BillingCurrency + } cust := Customer{ - ExternalID: t.ExternalID, + ExternalID: t.ID, Name: t.DisplayName, - Currency: t.Currency, + Email: t.Email, + Currency: currency, Metadata: meta, } if _, err := a.client.UpsertCustomer(ctx, cust); err != nil { - // Metadata keys may already exist on the customer — retry without metadata. + // Metadata keys may already exist on the customer; retry without metadata. cust.Metadata = nil if _, err2 := a.client.UpsertCustomer(ctx, cust); err2 != nil { return fmt.Errorf("upsert customer: %w", err2) } } - subID := subscriptionIDFor(t.ExternalID) - if _, err := a.client.GetCurrentUsage(ctx, t.ExternalID, subID); err == nil { - // Subscription already exists. - return nil + subID := subscriptionIDFor(t.ID) + if _, err := a.client.GetCurrentUsage(ctx, t.ID, subID); err == nil { + return nil // subscription already exists } - - sub := Subscription{ - ExternalCustomerID: t.ExternalID, - PlanCode: a.cfg.DefaultPlanCode, - ExternalID: subID, + plan := t.Plan + if plan == "" { + plan = a.cfg.DefaultPlanCode } + sub := Subscription{ExternalCustomerID: t.ID, PlanCode: plan, ExternalID: subID} if _, err := a.client.CreateSubscription(ctx, sub); err != nil { - log.Printf("[lago] warning: could not create subscription %s: %v", subID, err) + // Returned, not swallowed: without a subscription Lago drops the + // tenant's events, so EnsureTenant must be retried. + return fmt.Errorf("create subscription %s on plan %s: %w", subID, plan, err) } + log.Printf("[lago] subscribed tenant %s to plan %s", t.ID, plan) return nil } -func (a *Adapter) RemoveTenant(ctx context.Context, externalID string) error { - return a.client.TerminateSubscription(ctx, subscriptionIDFor(externalID)) +// RemoveTenant terminates the subscription, which makes Lago issue the final +// invoice. vBilling only calls it after the offboarding grace period. +func (a *Adapter) RemoveTenant(ctx context.Context, t usage.Tenant) error { + return a.client.TerminateSubscription(ctx, subscriptionIDFor(t.ID)) } -func (a *Adapter) SendEvents(ctx context.Context, events []destinations.UsageEvent) error { - if len(events) == 0 { - return nil +// ToEvent converts a canonical event to Lago's wire format. Dimensions and +// top-level attributes become string properties, so Lago charge filters can +// price on gpu_type, sku, region or capacity_type. +func ToEvent(ev *usage.Event) Event { + props := map[string]interface{}{} + for k, v := range ev.Properties { + props[k] = v } - lagoEvents := make([]Event, 0, len(events)) - for _, ev := range events { - props := map[string]interface{}{} - for k, v := range ev.Properties { + for k, v := range ev.Dimensions { + props[k] = v + } + set := func(k, v string) { + if v != "" { props[k] = v } - // The metric's primary value is what Lago aggregates on. Lago's - // billable metrics use a sum_agg over a named property field, so we - // repeat the value under the metric's field name. Existing properties - // (gpu_type, vcluster_name, etc.) flow through untouched. - props[fieldNameFor(ev.MetricCode)] = ev.Value - lagoEvents = append(lagoEvents, Event{ - TransactionID: ev.TransactionID, - ExternalSubscriptionID: subscriptionIDFor(ev.TenantExternalID), - Code: ev.MetricCode, - Timestamp: ev.Timestamp.Unix(), - Properties: props, - }) } + set("region", ev.Region) + set("project", ev.Project) + set("sku", ev.SKU) + set("resource_id", ev.ResourceID) + if tc := ev.Dim(usage.DimTenantCluster); tc != "" { + // v0.1 property names, kept for existing Lago filters. + set("vcluster", tc) + } + // Lago's sum_agg aggregates a named property: repeat the quantity under + // the field name Bootstrap registered for this metric. + props[fieldNameFor(ev.Metric)] = ev.Quantity // a JSON number, as v0.1 sent it + return Event{ + TransactionID: ev.ID, + ExternalSubscriptionID: subscriptionIDFor(ev.Tenant), + Code: ev.Metric, + Timestamp: ev.WindowStart.Unix(), + Properties: props, + } +} - // Lago's batch endpoint caps at 100 events per call. Each event carries a - // unique TransactionID, so Lago dedupes on retry — partial-failure recovery - // just resends the whole window. +func (a *Adapter) SendEvents(ctx context.Context, events []usage.Event) error { + lagoEvents := make([]Event, 0, len(events)) + for i := range events { + if a.Accepts(events[i].Metric) { + lagoEvents = append(lagoEvents, ToEvent(&events[i])) + } + } + // Lago's batch endpoint caps at 100 events per call. Transaction IDs are + // deterministic, so a resent window is deduplicated by Lago. for i := 0; i < len(lagoEvents); i += 100 { end := i + 100 if end > len(lagoEvents) { @@ -107,17 +142,25 @@ func (a *Adapter) SendEvents(ctx context.Context, events []destinations.UsageEve } batch := lagoEvents[i:end] if err := a.client.SendEvents(ctx, batch); err != nil { + if isDuplicate(err) && len(batch) == 1 { + continue // already recorded + } return fmt.Errorf("send %d events: %w", len(batch), err) } - log.Printf("[lago] sent %d billing events", len(batch)) } return nil } -// subscriptionIDFor derives a stable Lago subscription ID from a tenant external ID. +// isDuplicate recognizes Lago's "transaction_id already exists" rejection. +func isDuplicate(err error) bool { + var httpErr *destinations.HTTPError + return errors.As(err, &httpErr) && httpErr.Status == 422 && strings.Contains(httpErr.Body, "value_already_exist") +} + +// subscriptionIDFor derives a stable Lago subscription ID from a tenant ID. // Tenants are 1:1 with subscriptions in vBilling's Lago model. -func subscriptionIDFor(externalID string) string { - return "sub-" + externalID +func subscriptionIDFor(tenantID string) string { + return "sub-" + tenantID } // fieldNameFor maps a canonical metric code to the Lago BillableMetric.field_name @@ -147,5 +190,7 @@ func fieldNameFor(code string) string { } } -// Compile-time check. -var _ destinations.Destination = (*Adapter)(nil) +var ( + _ destinations.Destination = (*Adapter)(nil) + _ destinations.EventFilter = (*Adapter)(nil) +) diff --git a/internal/destinations/lago/adapter_test.go b/internal/destinations/lago/adapter_test.go new file mode 100644 index 0000000..dce8d04 --- /dev/null +++ b/internal/destinations/lago/adapter_test.go @@ -0,0 +1,73 @@ +package lago + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +func TestToEventCarriesDimensionsForChargeFilters(t *testing.T) { + ws := time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) + e := usage.Event{Tenant: "acme", Metric: usage.MetricGPUHours, Quantity: 8.0 / 60, SKU: "NVIDIA-H100", Region: "ap-southeast-2", + WindowStart: ws, WindowEnd: ws.Add(time.Minute), + Dimensions: map[string]string{usage.DimTenantCluster: "vcluster-a-train", usage.DimCapacityType: usage.CapacitySpot}} + e.Finalize() + le := ToEvent(&e) + if le.TransactionID != e.ID || le.ExternalSubscriptionID != "sub-acme" || le.Code != usage.MetricGPUHours || le.Timestamp != ws.Unix() { + t.Fatalf("event: %+v", le) + } + if le.Properties["gpu_hours"] != 0.133333333 { + t.Errorf("gpu_hours = %v, want the number 0.133333333", le.Properties["gpu_hours"]) + } + for k, want := range map[string]string{"sku": "NVIDIA-H100", "region": "ap-southeast-2", "capacity_type": "spot", "vcluster": "vcluster-a-train"} { + if le.Properties[k] != want { + t.Errorf("property %s = %v, want %s", k, le.Properties[k], want) + } + } +} + +func TestDuplicateTransactionIDCountsAsDelivered(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + body, _ := io.ReadAll(r.Body) + var req struct { + Events []Event `json:"events"` + } + json.Unmarshal(body, &req) + if len(req.Events) == 1 && strings.HasSuffix(req.Events[0].TransactionID, "dup") { + w.WriteHeader(422) + io.WriteString(w, `{"status":422,"error":"Unprocessable Entity","code":"validation_errors","error_details":{"transaction_id":["value_already_exist"]}}`) + return + } + w.WriteHeader(200) + })) + defer srv.Close() + a := &Adapter{client: NewClient(srv.URL, "key"), cfg: &config.Config{}} + ev := usage.Event{ID: "vb1_dup", Tenant: "acme", Metric: usage.MetricCPUCoreHours, Quantity: 1, WindowStart: time.Now().Add(-time.Minute), WindowEnd: time.Now()} + if err := a.SendEvents(context.Background(), []usage.Event{ev}); err != nil { + t.Fatalf("duplicate must be treated as delivered: %v", err) + } + // A 422 for another reason is a permanent rejection the dispatcher isolates. + srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(422) + io.WriteString(w, `{"code":"validation_errors","error_details":{"code":["metric_not_found"]}}`) + })) + defer srv2.Close() + b := &Adapter{client: NewClient(srv2.URL, "key"), cfg: &config.Config{}} + if err := b.SendEvents(context.Background(), []usage.Event{ev}); !destinations.IsPermanent(err) { + t.Fatalf("expected permanent error, got %v", err) + } + if b.Accepts(usage.MetricGPUDowntimeHours) || !b.Accepts(usage.MetricGPUUtilization) { + t.Fatal("lago filter: downtime is informational; utilization is kept for v0.1 plans") + } +} diff --git a/internal/destinations/lago/bootstrap.go b/internal/destinations/lago/bootstrap.go index 6e58e14..3aa17e5 100644 --- a/internal/destinations/lago/bootstrap.go +++ b/internal/destinations/lago/bootstrap.go @@ -6,25 +6,26 @@ import ( "strings" "github.com/vclusterlabs-experiments/vbilling/internal/config" - "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" ) // Re-exports of canonical metric codes for in-package use. const ( - MetricCPUCoreHours = destinations.MetricCPUCoreHours - MetricMemoryGBHours = destinations.MetricMemoryGBHours - MetricStorageGBHours = destinations.MetricStorageGBHours - MetricInstanceHours = destinations.MetricInstanceHours - MetricGPUHours = destinations.MetricGPUHours - MetricGPUUtilization = destinations.MetricGPUUtilization - MetricNetworkEgressGB = destinations.MetricNetworkEgressGB - MetricLBHours = destinations.MetricLBHours - MetricPrivateNodeHours = destinations.MetricPrivateNodeHours + MetricCPUCoreHours = usage.MetricCPUCoreHours + MetricMemoryGBHours = usage.MetricMemoryGBHours + MetricStorageGBHours = usage.MetricStorageGBHours + MetricInstanceHours = usage.MetricInstanceHours + MetricGPUHours = usage.MetricGPUHours + MetricGPUUtilization = usage.MetricGPUUtilization + MetricNetworkEgressGB = usage.MetricNetworkEgressGB + MetricLBHours = usage.MetricLBHours + MetricPrivateNodeHours = usage.MetricPrivateNodeHours ) // Bootstrap creates all billable metrics and a default plan in Lago. -// It is idempotent — safe to call on every startup. -func Bootstrap(ctx context.Context, client *Client, cfg *config.Config) error { +// It is idempotent: safe to call on every startup. Custom metrics from the +// catalog get a billable metric whose field name is their code. +func Bootstrap(ctx context.Context, client *Client, cfg *config.Config, catalog []usage.MetricDef) error { log.Println("[bootstrap] setting up Lago billing configuration...") // Step 1: Create billable metrics @@ -94,6 +95,16 @@ func Bootstrap(ctx context.Context, client *Client, cfg *config.Config) error { }, } + for _, def := range catalog { + if !def.Custom { + continue + } + metrics = append(metrics, BillableMetric{ + Name: def.Name, Code: def.Code, Description: def.Description, + AggregationType: "sum_agg", FieldName: fieldNameFor(def.Code), + }) + } + metricIDs := make(map[string]string) // code -> lago_id for _, m := range metrics { existing, err := client.GetBillableMetric(ctx, m.Code) @@ -125,52 +136,13 @@ func Bootstrap(ctx context.Context, client *Client, cfg *config.Config) error { log.Println("[bootstrap] Configure your pricing in the Lago UI or API — all charges default to $0") - charges := []Charge{ - { - BillableMetricID: metricIDs[MetricCPUCoreHours], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricMemoryGBHours], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricStorageGBHours], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricInstanceHours], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricGPUHours], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricGPUUtilization], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricNetworkEgressGB], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricLBHours], - ChargeModel: "standard", - Properties: map[string]string{"amount": "0"}, - }, - { - BillableMetricID: metricIDs[MetricPrivateNodeHours], + var charges []Charge + for _, m := range metrics { + charges = append(charges, Charge{ + BillableMetricID: metricIDs[m.Code], ChargeModel: "standard", Properties: map[string]string{"amount": "0"}, - }, + }) } // Filter out charges with empty metric IDs (metric creation may have failed) diff --git a/internal/destinations/lago/client.go b/internal/destinations/lago/client.go index 696b57d..5830ebe 100644 --- a/internal/destinations/lago/client.go +++ b/internal/destinations/lago/client.go @@ -9,6 +9,8 @@ import ( "log" "net/http" "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" ) // Client is an HTTP client for the Lago billing API. @@ -40,11 +42,11 @@ type BillableMetric struct { } type Charge struct { - BillableMetricID string `json:"billable_metric_id,omitempty"` - BillableMetricCode string `json:"billable_metric_code,omitempty"` - ChargeModel string `json:"charge_model"` - PayInAdvance bool `json:"pay_in_advance"` - Properties map[string]string `json:"properties"` + BillableMetricID string `json:"billable_metric_id,omitempty"` + BillableMetricCode string `json:"billable_metric_code,omitempty"` + ChargeModel string `json:"charge_model"` + PayInAdvance bool `json:"pay_in_advance"` + Properties map[string]string `json:"properties"` } type Plan struct { @@ -59,11 +61,11 @@ type Plan struct { } type Customer struct { - ExternalID string `json:"external_id"` - Name string `json:"name"` - Email string `json:"email,omitempty"` - Currency string `json:"currency,omitempty"` - Metadata []CustomerMeta `json:"metadata,omitempty"` + ExternalID string `json:"external_id"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + Currency string `json:"currency,omitempty"` + Metadata []CustomerMeta `json:"metadata,omitempty"` } type CustomerMeta struct { @@ -234,11 +236,7 @@ func (c *Client) post(ctx context.Context, path string, body interface{}, result respBody, _ := io.ReadAll(resp.Body) if resp.StatusCode >= 400 { - var apiErr apiError - if json.Unmarshal(respBody, &apiErr) == nil && apiErr.Code != "" { - return fmt.Errorf("lago API error %d: %s - %s", resp.StatusCode, apiErr.Code, apiErr.Message) - } - return fmt.Errorf("lago API error %d: %s", resp.StatusCode, string(respBody)) + return destinations.StatusError(http.MethodPost, path, resp.StatusCode, respBody) } if result != nil && len(respBody) > 0 { @@ -247,7 +245,6 @@ func (c *Client) post(ctx context.Context, path string, body interface{}, result } } - log.Printf("[lago] POST %s -> %d", path, resp.StatusCode) return nil } diff --git a/internal/destinations/metronome/adapter.go b/internal/destinations/metronome/adapter.go new file mode 100644 index 0000000..a7c165d --- /dev/null +++ b/internal/destinations/metronome/adapter.go @@ -0,0 +1,421 @@ +package metronome + +import ( + "context" + "errors" + "fmt" + "log" + "path/filepath" + "regexp" + "sort" + "strconv" + "sync" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +func init() { + destinations.Register("metronome", func(cfg *config.Config) (destinations.Destination, error) { + if cfg.MetronomeAPIToken == "" { + return nil, errors.New("METRONOME_API_TOKEN is required when the metronome adapter is enabled") + } + state, err := kv.Open(filepath.Join(cfg.DataDir, "state-metronome.json")) + if err != nil { + return nil, fmt.Errorf("open metronome state: %w", err) + } + var link *stripe.Adapter + if cfg.MetronomeStripeLink { + if cfg.StripeAPIKey == "" { + return nil, errors.New("METRONOME_STRIPE_LINK needs STRIPE_API_KEY to create Stripe customers") + } + sstate, err := kv.Open(filepath.Join(cfg.DataDir, "state-metronome-stripe.json")) + if err != nil { + return nil, err + } + link = stripe.New(stripe.NewClient(cfg.StripeAPIBase, cfg.StripeAPIKey, cfg.StripeAPIVersion, cfg.StripeMaxRPS), cfg, sstate) + } + return New(NewClient(cfg.MetronomeAPIBase, cfg.MetronomeAPIToken), cfg, state, link), nil + }) +} + +// Metronome accepts events backdated up to 34 days; keep an hour of margin. +const maxEventAge = 34*24*time.Hour - time.Hour + +// placeholder fills group-key properties an event does not carry: Metronome +// metrics require every group-key property to exist. +const placeholder = "none" + +var uuidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// Adapter ships usage to Metronome. Tenants become customers whose ingest +// alias is the vBilling tenant ID, so events are attributed even if they +// arrive before the customer exists. +type Adapter struct { + c *Client + cfg *config.Config + state *kv.Store + stripe *stripe.Adapter // nil unless METRONOME_STRIPE_LINK + now func() time.Time + + mu sync.Mutex + metrics map[string]string // metric code -> billable metric ID +} + +// New builds the adapter (exported for tests). link may be nil. +func New(c *Client, cfg *config.Config, state *kv.Store, link *stripe.Adapter) *Adapter { + return &Adapter{c: c, cfg: cfg, state: state, stripe: link, now: time.Now, metrics: map[string]string{}} +} + +func (a *Adapter) Name() string { return "metronome" } + +// StripeLink returns the Stripe customer manager used for invoicing (nil +// unless METRONOME_STRIPE_LINK). It resolves Stripe webhooks to tenants. +func (a *Adapter) StripeLink() *stripe.Adapter { return a.stripe } + +func (a *Adapter) Accepts(metric string) bool { return destinations.BillableOnly(metric) } + +// GroupKeys is the single compound group key vBilling creates per metric: +// the metric's pricing dimensions plus tenant_cluster for presentation. +// Metronome requires pricing + presentation keys to be one compound key. +func GroupKeys(def usage.MetricDef) []string { + keys := append([]string(nil), def.GroupKeys...) + if !def.Custom { + keys = append(keys, usage.DimTenantCluster) + } + seen := map[string]bool{} + out := keys[:0] + for _, k := range keys { + if !seen[k] { + seen[k] = true + out = append(out, k) + } + } + return out +} + +func metricName(def usage.MetricDef) string { return def.Name + " [" + def.Code + "]" } + +// Bootstrap creates one SUM billable metric per billable metric code, +// filtered on event_type and grouped on GroupKeys. Existing metrics are +// matched by stored ID or by name, never duplicated. +func (a *Adapter) Bootstrap(ctx context.Context, metrics []usage.MetricDef) error { + existing, err := a.c.ListBillableMetrics(ctx) + if err != nil { + return fmt.Errorf("list billable metrics: %w", err) + } + byID, byName := map[string]bool{}, map[string]string{} + for _, m := range existing { + if m.ArchivedAt != "" { + continue + } + byID[m.ID] = true + byName[m.Name] = m.ID + } + var errs []error + for _, def := range metrics { + if !def.Billable { + continue + } + if id, ok := a.state.Get("bm/" + def.Code); ok && byID[id] { + a.setMetric(def.Code, id) + continue + } + if id, ok := byName[metricName(def)]; ok { + a.setMetric(def.Code, id) + continue + } + yes := true + filters := []PropertyFilter{{Name: "value", Exists: &yes}} + keys := GroupKeys(def) + for _, k := range keys { + filters = append(filters, PropertyFilter{Name: k, Exists: &yes}) + } + bm := BillableMetric{ + Name: metricName(def), + AggregationType: "SUM", + AggregationKey: "value", + EventTypeFilter: &EventTypeFilter{InValues: []string{def.Code}}, + PropertyFilters: filters, + } + if len(keys) > 0 { + bm.GroupKeys = [][]string{keys} + } + id, err := a.c.CreateBillableMetric(ctx, bm) + if err != nil { + errs = append(errs, fmt.Errorf("create billable metric %s: %w", def.Code, err)) + continue + } + log.Printf("[metronome] created billable metric %s (%s)", def.Code, id) + a.setMetric(def.Code, id) + } + return errors.Join(errs...) +} + +func (a *Adapter) setMetric(code, id string) { + a.mu.Lock() + a.metrics[code] = id + a.mu.Unlock() + _ = a.state.Set("bm/"+code, id) + _ = a.state.Set("bmcode/"+id, code) +} + +// EnsureTenant creates the Metronome customer (and the linked Stripe +// customer when configured) and, with a rate card configured, a contract. +func (a *Adapter) EnsureTenant(ctx context.Context, t usage.Tenant) error { + cusID, err := a.ensureCustomer(ctx, t) + if err != nil { + return err + } + rateCard := t.Plan + if rateCard == "" { + rateCard = a.cfg.MetronomeRateCard + } + if rateCard == "" { + return nil + } + return a.ensureContract(ctx, t, cusID, rateCard) +} + +func (a *Adapter) ensureCustomer(ctx context.Context, t usage.Tenant) (string, error) { + if id := t.ProviderIDs["metronome"]; id != "" { + a.remember(t.ID, id) + return id, nil + } + if id, ok := a.state.Get("cus/" + t.ID); ok { + return id, nil + } + if cus, err := a.c.FindCustomerByAlias(ctx, t.ID); err != nil { + return "", fmt.Errorf("find customer %s: %w", t.ID, err) + } else if cus != nil { + a.remember(t.ID, cus.ID) + return cus.ID, nil + } + + req := CreateCustomerRequest{Name: t.DisplayName, IngestAliases: []string{t.ID}} + if req.Name == "" { + req.Name = t.ID + } + if a.stripe != nil { + stripeID, err := a.stripe.EnsureCustomer(ctx, t) + if err != nil { + return "", fmt.Errorf("link stripe customer for %s: %w", t.ID, err) + } + req.BillingProviderConfigs = []BillingProviderConfig{{ + BillingProvider: "stripe", + DeliveryMethod: "direct_to_billing_provider", + Configuration: map[string]string{ + "stripe_customer_id": stripeID, + "stripe_collection_method": a.cfg.MetronomeStripeCollect, + }, + }} + } + cus, err := a.c.CreateCustomer(ctx, req) + if IsConflict(err) { + // Alias taken: someone (or an earlier attempt) created it already. + if cus, ferr := a.c.FindCustomerByAlias(ctx, t.ID); ferr == nil && cus != nil { + a.remember(t.ID, cus.ID) + return cus.ID, nil + } + } + if err != nil { + return "", fmt.Errorf("create customer %s: %w", t.ID, err) + } + log.Printf("[metronome] created customer %s for tenant %s", cus.ID, t.ID) + a.remember(t.ID, cus.ID) + return cus.ID, nil +} + +func (a *Adapter) remember(tenant, cus string) { + _ = a.state.Set("cus/"+tenant, cus) + _ = a.state.Set("tenant/"+cus, tenant) +} + +func (a *Adapter) ensureContract(ctx context.Context, t usage.Tenant, cusID, rateCard string) error { + if _, ok := a.state.Get("contract/" + t.ID + "/" + rateCard); ok { + return nil + } + start := a.now().UTC().Truncate(24 * time.Hour) // UTC midnight + req := CreateContractRequest{ + CustomerID: cusID, + StartingAt: rfc3339(start), + UniquenessKey: "vbilling-" + t.ID + "-" + rateCard, + UsageStatementSchedule: &UsageStatementSchedule{Frequency: "MONTHLY", Day: "FIRST_OF_MONTH"}, + } + if uuidRe.MatchString(rateCard) { + req.RateCardID = rateCard + } else { + req.RateCardAlias = rateCard + } + if a.stripe != nil { + req.BillingProviderConfig = &BillingProviderConfig{BillingProvider: "stripe", DeliveryMethod: "direct_to_billing_provider"} + } + id, err := a.c.CreateContract(ctx, req) + if IsConflict(err) { + id, err = "existing", nil // uniqueness_key already used: the contract exists + } + if err != nil { + return fmt.Errorf("create contract for %s on %s: %w", t.ID, rateCard, err) + } + log.Printf("[metronome] contract %s for tenant %s on rate card %s", id, t.ID, rateCard) + return a.state.Set("contract/"+t.ID+"/"+rateCard, id) +} + +// RemoveTenant leaves the customer and contract in place: final usage still +// needs to invoice, and ending contracts is a commercial decision. +func (a *Adapter) RemoveTenant(ctx context.Context, t usage.Tenant) error { + log.Printf("[metronome] tenant %s offboarded; customer and contract left in place", t.ID) + return nil +} + +// TenantForCustomer resolves a webhook's customer_id to a tenant. +func (a *Adapter) TenantForCustomer(ctx context.Context, customerID string) (string, bool) { + if t, ok := a.state.Get("tenant/" + customerID); ok { + return t, true + } + cus, err := a.c.GetCustomer(ctx, customerID) + if err != nil || len(cus.IngestAliases) == 0 { + return "", false + } + a.remember(cus.IngestAliases[0], customerID) + return cus.IngestAliases[0], true +} + +// ToIngest converts a canonical event to Metronome's wire format. +func (a *Adapter) ToIngest(e *usage.Event) IngestEvent { + props := map[string]string{ + "value": strconv.FormatFloat(e.Quantity, 'f', -1, 64), + "unit": e.Unit, + "window_start": rfc3339(e.WindowStart), + "window_end": rfc3339(e.WindowEnd), + "schema_version": e.SchemaVersion, + } + set := func(k, v string) { + if v != "" { + props[k] = v + } + } + set("region", e.Region) + set("project", e.Project) + set("sku", e.SKU) + set("resource_id", e.ResourceID) + set("source", e.Source) + for k, v := range e.Dimensions { + set(k, v) + } + if def, ok := usage.Lookup(e.Metric); ok { + for _, k := range GroupKeys(def) { + if props[k] == "" { + props[k] = placeholder + } + } + } + customer := e.Tenant + if id, ok := a.state.Get("cus/" + e.Tenant); ok { + customer = id // pinned or created customer UUID + } + return IngestEvent{ + TransactionID: e.ID, + CustomerID: customer, + EventType: e.Metric, + Timestamp: rfc3339(e.WindowStart), + Properties: props, + } +} + +// SendEvents ingests in batches of 100. Retrying a batch is safe: Metronome +// ignores repeated transaction IDs for 34 days. +func (a *Adapter) SendEvents(ctx context.Context, events []usage.Event) error { + rejected := map[string]string{} + var batch []IngestEvent + for i := range events { + e := &events[i] + if !a.Accepts(e.Metric) { + continue + } + if a.now().Sub(e.WindowStart) > maxEventAge { + rejected[e.ID] = "older than Metronome's 34-day ingest window" + continue + } + batch = append(batch, a.ToIngest(e)) + } + for i := 0; i < len(batch); i += 100 { + end := i + 100 + if end > len(batch) { + end = len(batch) + } + // A permanent error here makes the dispatcher bisect the batch, so + // the age rejections above resurface on their own. + if err := a.c.Ingest(ctx, batch[i:end]); err != nil { + return err + } + } + if len(rejected) > 0 { + return &destinations.PartialError{Rejected: rejected} + } + return nil +} + +// RecordedTotals reads Metronome's usage API. Metronome only answers for +// whole UTC days, so both bounds must be UTC midnights. +func (a *Adapter) RecordedTotals(ctx context.Context, q destinations.TotalsQuery) ([]destinations.Total, error) { + from, to := q.From.UTC(), q.To.UTC() + if !from.Equal(from.Truncate(24*time.Hour)) || !to.Equal(to.Truncate(24*time.Hour)) || !to.After(from) { + return nil, fmt.Errorf("metronome reconciliation needs whole UTC days (got %s to %s)", rfc3339(from), rfc3339(to)) + } + req := UsageRequest{StartingOn: rfc3339(from), EndingBefore: rfc3339(to), WindowSize: "day"} + tenantOf := map[string]string{} + for _, t := range q.Tenants { + if id, ok := a.state.Get("cus/" + t.ID); ok { + req.CustomerIDs = append(req.CustomerIDs, id) + tenantOf[id] = t.ID + } + } + codeOf := map[string]string{} + for _, m := range q.Metrics { + if id, ok := a.state.Get("bm/" + m); ok { + req.BillableMetrics = append(req.BillableMetrics, map[string]any{"id": id}) + codeOf[id] = m + } + } + if len(req.CustomerIDs) == 0 || len(req.BillableMetrics) == 0 { + return nil, nil + } + rows, err := a.c.Usage(ctx, req) + if err != nil { + return nil, fmt.Errorf("usage: %w", err) + } + sums := map[[2]string]float64{} + for _, r := range rows { + v, err := r.Value.Float64() + if err != nil { + continue + } + sums[[2]string{tenantOf[r.CustomerID], codeOf[r.BillableMetricID]}] += v + } + var out []destinations.Total + for k, v := range sums { + if k[0] == "" || k[1] == "" { + continue + } + out = append(out, destinations.Total{Tenant: k[0], Metric: k[1], Quantity: usage.Round(v)}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Tenant != out[j].Tenant { + return out[i].Tenant < out[j].Tenant + } + return out[i].Metric < out[j].Metric + }) + return out, nil +} + +var ( + _ destinations.Destination = (*Adapter)(nil) + _ destinations.EventFilter = (*Adapter)(nil) + _ destinations.Reconciler = (*Adapter)(nil) + _ destinations.CustomerResolver = (*Adapter)(nil) +) diff --git a/internal/destinations/metronome/adapter_test.go b/internal/destinations/metronome/adapter_test.go new file mode 100644 index 0000000..c00c868 --- /dev/null +++ b/internal/destinations/metronome/adapter_test.go @@ -0,0 +1,468 @@ +package metronome + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +var now = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + +// fakeMetronome validates every request body against Metronome's OpenAPI +// request schema and models dedupe, alias conflicts and uniqueness keys. +type fakeMetronome struct { + t *testing.T + srv *httptest.Server + + mu sync.Mutex + customers map[string]*Customer // id -> customer + metrics []BillableMetric + events map[string]IngestEvent // transaction_id -> first accepted + ingests int + contracts map[string]CreateContractRequest // uniqueness key -> request + billing map[string][]BillingProviderConfig + failNext map[string]int + hideOnce bool // next alias lookup misses (eventual consistency), forcing a 409 on create + conflicts int +} + +func newFakeMetronome(t *testing.T) *fakeMetronome { + f := &fakeMetronome{t: t, customers: map[string]*Customer{}, events: map[string]IngestEvent{}, + contracts: map[string]CreateContractRequest{}, billing: map[string][]BillingProviderConfig{}, failNext: map[string]int{}} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +func uuidFor(s string) string { + h := sha256.Sum256([]byte(s)) + x := fmt.Sprintf("%x", h[:16]) + return x[:8] + "-" + x[8:12] + "-" + x[12:16] + "-" + x[16:20] + "-" + x[20:32] +} + +func (f *fakeMetronome) reply(w http.ResponseWriter, status int, v any) { + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) +} + +func (f *fakeMetronome) handle(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if r.Header.Get("Authorization") != "Bearer mtk_test" { + f.reply(w, 401, map[string]string{"message": "Unauthorized"}) + return + } + body, _ := io.ReadAll(r.Body) + key := r.Method + " " + r.URL.Path + if n := f.failNext[key]; n > 0 { + f.failNext[key] = n - 1 + f.reply(w, 503, map[string]string{"message": "unavailable"}) + return + } + switch key { + case "POST /v1/ingest": + assertMatchesSchema(f.t, "ingest", body) + var evs []IngestEvent + json.Unmarshal(body, &evs) + f.ingests++ + for _, e := range evs { + for k, v := range e.Properties { + if v == "" { + f.t.Errorf("empty property %s on %s", k, e.TransactionID) + } + } + if _, dup := f.events[e.TransactionID]; !dup { // first accepted wins + f.events[e.TransactionID] = e + } + } + w.WriteHeader(200) + + case "GET /v1/customers": + alias := r.URL.Query().Get("ingest_alias") + var out []Customer + if f.hideOnce { + f.hideOnce = false + f.reply(w, 200, map[string]any{"data": out, "next_page": nil}) + return + } + for _, c := range f.customers { + for _, a := range c.IngestAliases { + if a == alias { + out = append(out, *c) + } + } + } + f.reply(w, 200, map[string]any{"data": out, "next_page": nil}) + + case "POST /v1/customers": + assertMatchesSchema(f.t, "create_customer", body) + var req CreateCustomerRequest + json.Unmarshal(body, &req) + for _, c := range f.customers { + for _, a := range c.IngestAliases { + if a == req.IngestAliases[0] { + f.conflicts++ + f.reply(w, 409, map[string]string{"message": "ingest alias already in use"}) + return + } + } + } + c := &Customer{ID: uuidFor(req.IngestAliases[0]), Name: req.Name, IngestAliases: req.IngestAliases} + f.customers[c.ID] = c + f.billing[c.ID] = req.BillingProviderConfigs + f.reply(w, 200, map[string]any{"data": c}) + + case "GET /v1/billable-metrics": + f.reply(w, 200, map[string]any{"data": f.metrics, "next_page": nil}) + + case "POST /v1/billable-metrics/create": + assertMatchesSchema(f.t, "create_billable_metric", body) + var m BillableMetric + json.Unmarshal(body, &m) + m.ID = uuidFor(m.Name) + f.metrics = append(f.metrics, m) + f.reply(w, 200, map[string]any{"data": map[string]string{"id": m.ID}}) + + case "POST /v1/contracts/create": + assertMatchesSchema(f.t, "create_contract", body) + var req CreateContractRequest + json.Unmarshal(body, &req) + if _, used := f.contracts[req.UniquenessKey]; used { + f.reply(w, 409, map[string]string{"message": "This uniqueness key has already been used."}) + return + } + f.contracts[req.UniquenessKey] = req + f.reply(w, 200, map[string]any{"data": map[string]string{"id": uuidFor(req.UniquenessKey)}}) + + case "POST /v1/usage": + assertMatchesSchema(f.t, "usage", body) + var req UsageRequest + json.Unmarshal(body, &req) + start, _ := time.Parse(time.RFC3339, req.StartingOn) + end, _ := time.Parse(time.RFC3339, req.EndingBefore) + var rows []map[string]any + for _, cid := range req.CustomerIDs { + for _, bmRef := range req.BillableMetrics { + bmID := bmRef["id"].(string) + var metric BillableMetric + for _, m := range f.metrics { + if m.ID == bmID { + metric = m + } + } + var sum float64 + for _, e := range f.events { + ts, _ := time.Parse(time.RFC3339, e.Timestamp) + owner := e.CustomerID + if c, ok := f.customers[cid]; ok && len(c.IngestAliases) > 0 && owner == c.IngestAliases[0] { + owner = cid + } + if owner == cid && e.EventType == metric.EventTypeFilter.InValues[0] && !ts.Before(start) && ts.Before(end) { + v, _ := strconv.ParseFloat(e.Properties[metric.AggregationKey], 64) + sum += v + } + } + rows = append(rows, map[string]any{"customer_id": cid, "billable_metric_id": bmID, "value": sum}) + } + } + f.reply(w, 200, map[string]any{"data": rows, "next_page": nil}) + + default: + if strings.HasPrefix(key, "GET /v1/customers/") { + c, ok := f.customers[strings.TrimPrefix(r.URL.Path, "/v1/customers/")] + if !ok { + f.reply(w, 404, map[string]string{"message": "not found"}) + return + } + f.reply(w, 200, map[string]any{"data": c}) + return + } + f.reply(w, 404, map[string]string{"message": "unhandled " + key}) + } +} + +func newAdapter(t *testing.T, f *fakeMetronome, link *stripe.Adapter, mutate ...func(*config.Config)) *Adapter { + cfg := &config.Config{MetronomeStripeCollect: "charge_automatically"} + for _, m := range mutate { + m(cfg) + } + a := New(NewClient(f.srv.URL, "mtk_test"), cfg, kv.Memory(), link) + a.now = func() time.Time { return now } + return a +} + +func gpu(tenant, cluster string, ws time.Time, q float64) usage.Event { + e := usage.Event{Tenant: tenant, Metric: usage.MetricGPUHours, Quantity: q, SKU: "NVIDIA-H100-80GB-HBM3", + Region: "ap-southeast-2", WindowStart: ws, WindowEnd: ws.Add(time.Minute), + Dimensions: map[string]string{usage.DimTenantCluster: cluster, usage.DimCapacityType: usage.CapacityOnDemand}} + e.Finalize() + return e +} + +func TestBootstrapCreatesGroupedSumMetrics(t *testing.T) { + f := newFakeMetronome(t) + a := newAdapter(t, f, nil) + ctx := context.Background() + if err := a.Bootstrap(ctx, usage.Catalog()); err != nil { + t.Fatal(err) + } + if len(f.metrics) != len(usage.Billable()) { + t.Fatalf("created %d billable metrics", len(f.metrics)) + } + var gpuMetric BillableMetric + for _, m := range f.metrics { + if m.EventTypeFilter.InValues[0] == usage.MetricGPUHours { + gpuMetric = m + } + } + if gpuMetric.AggregationType != "SUM" || gpuMetric.AggregationKey != "value" { + t.Fatalf("gpu metric: %+v", gpuMetric) + } + if got := strings.Join(gpuMetric.GroupKeys[0], ","); got != "region,sku,capacity_type,billing_mode,tenant_cluster" { + t.Fatalf("compound group key = %s", got) + } + filtered := map[string]bool{} + for _, pf := range gpuMetric.PropertyFilters { + filtered[pf.Name] = pf.Exists != nil && *pf.Exists + } + for _, k := range append([]string{"value"}, gpuMetric.GroupKeys[0]...) { + if !filtered[k] { + t.Errorf("group key %s lacks an exists property filter (Metronome requires it)", k) + } + } + + // Restart with lost state: metrics are matched by name, not recreated. + b := newAdapter(t, f, nil) + if err := b.Bootstrap(ctx, usage.Catalog()); err != nil { + t.Fatal(err) + } + if len(f.metrics) != len(usage.Billable()) { + t.Fatalf("second bootstrap duplicated metrics: %d", len(f.metrics)) + } +} + +func TestEnsureTenantCustomerAndContract(t *testing.T) { + f := newFakeMetronome(t) + a := newAdapter(t, f, nil, func(c *config.Config) { c.MetronomeRateCard = "payg-aud" }) + ctx := context.Background() + tenant := usage.Tenant{ID: "acme", DisplayName: "Acme AI"} + for i := 0; i < 2; i++ { + if err := a.EnsureTenant(ctx, tenant); err != nil { + t.Fatal(err) + } + } + if len(f.customers) != 1 || len(f.contracts) != 1 { + t.Fatalf("customers=%d contracts=%d", len(f.customers), len(f.contracts)) + } + for _, c := range f.contracts { + if c.RateCardAlias != "payg-aud" || c.StartingAt != "2026-10-05T00:00:00Z" || c.UsageStatementSchedule.Day != "FIRST_OF_MONTH" { + t.Fatalf("contract: %+v", c) + } + } + // Lost state: alias lookup finds the customer; the uniqueness key stops a + // second contract (409 is treated as "already exists"). + b := newAdapter(t, f, nil, func(c *config.Config) { c.MetronomeRateCard = "payg-aud" }) + if err := b.EnsureTenant(ctx, tenant); err != nil { + t.Fatal(err) + } + if len(f.customers) != 1 || len(f.contracts) != 1 { + t.Fatalf("after state loss: customers=%d contracts=%d", len(f.customers), len(f.contracts)) + } + // A per-tenant plan that looks like a UUID is passed as rate_card_id. + rc := "8f0e3c1a-6d3b-4c8e-9a51-1f0d7a2b9c44" + if err := b.EnsureTenant(ctx, usage.Tenant{ID: "beta", Plan: rc}); err != nil { + t.Fatal(err) + } + if got := f.contracts["vbilling-beta-"+rc]; got.RateCardID != rc || got.RateCardAlias != "" { + t.Fatalf("uuid rate card: %+v", got) + } +} + +func TestEnsureTenantConflictResolvesExistingAlias(t *testing.T) { + f := newFakeMetronome(t) + pre := &Customer{ID: uuidFor("acme"), Name: "pre-existing", IngestAliases: []string{"acme"}} + f.customers[pre.ID] = pre + a := newAdapter(t, f, nil) + f.hideOnce = true // the first lookup misses, so creation hits the 409 path + if err := a.EnsureTenant(context.Background(), usage.Tenant{ID: "acme"}); err != nil { + t.Fatal(err) + } + if f.conflicts != 1 { + t.Fatalf("expected one 409, got %d", f.conflicts) + } + if got, _ := a.state.Get("cus/acme"); got != pre.ID { + t.Fatalf("resolved %q, want %q", got, pre.ID) + } +} + +// stripeStub implements the customer endpoints the Stripe link uses. +func stripeStub(t *testing.T) (*httptest.Server, *int) { + created := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + switch { + case r.URL.Path == "/v1/customers/search": + json.NewEncoder(w).Encode(map[string]any{"data": []any{}, "has_more": false}) + case r.Method == "POST" && r.URL.Path == "/v1/customers": + created++ + if r.Form.Get("metadata[vbilling_tenant_id]") == "" { + t.Error("stripe customer missing tenant metadata") + } + json.NewEncoder(w).Encode(map[string]any{"id": "cus_linked"}) + case r.Method == "POST" && strings.HasPrefix(r.URL.Path, "/v1/customers/"): + json.NewEncoder(w).Encode(map[string]any{"id": "cus_linked"}) + default: + w.WriteHeader(404) + } + })) + t.Cleanup(srv.Close) + return srv, &created +} + +func TestStripeLinkedCustomerForInvoicing(t *testing.T) { + f := newFakeMetronome(t) + ss, created := stripeStub(t) + cfg := &config.Config{} + link := stripe.New(stripe.NewClient(ss.URL, "sk_test_x", "2026-09-30.endive", 0), cfg, kv.Memory()) + a := newAdapter(t, f, link, func(c *config.Config) { c.MetronomeRateCard = "payg-aud" }) + if err := a.EnsureTenant(context.Background(), usage.Tenant{ID: "acme", DisplayName: "Acme"}); err != nil { + t.Fatal(err) + } + if *created != 1 { + t.Fatalf("stripe customers created: %d", *created) + } + cfgs := f.billing[uuidFor("acme")] + if len(cfgs) != 1 || cfgs[0].BillingProvider != "stripe" || cfgs[0].DeliveryMethod != "direct_to_billing_provider" || + cfgs[0].Configuration["stripe_customer_id"] != "cus_linked" || cfgs[0].Configuration["stripe_collection_method"] != "charge_automatically" { + t.Fatalf("billing provider config: %+v", cfgs) + } + for _, c := range f.contracts { + if c.BillingProviderConfig == nil || c.BillingProviderConfig.BillingProvider != "stripe" { + t.Fatalf("contract not routed to stripe: %+v", c) + } + } +} + +func TestSendEventsWireFormatBatchingAndDedupe(t *testing.T) { + f := newFakeMetronome(t) + a := newAdapter(t, f, nil) + ctx := context.Background() + if err := a.EnsureTenant(ctx, usage.Tenant{ID: "acme"}); err != nil { + t.Fatal(err) + } + var events []usage.Event + for i := 0; i < 250; i++ { + events = append(events, gpu("acme", "vcluster-team-a-train", now.Add(-time.Duration(i+1)*time.Minute), 8.0/60)) + } + cpu := usage.Event{Tenant: "beta", Metric: usage.MetricCPUCoreHours, Quantity: 0.25, WindowStart: now.Add(-time.Minute), WindowEnd: now} + cpu.Finalize() + events = append(events, cpu) + + f.failNext["POST /v1/ingest"] = 1 + if err := a.SendEvents(ctx, events); err == nil || destinations.IsPermanent(err) { + t.Fatalf("503 must be a retryable error, got %v", err) + } + if err := a.SendEvents(ctx, events); err != nil { + t.Fatal(err) + } + if err := a.SendEvents(ctx, events); err != nil { // full replay: deduped by transaction_id + t.Fatal(err) + } + if len(f.events) != 251 { + t.Fatalf("stored %d events, want 251", len(f.events)) + } + if f.ingests != 6 { // 3 batches of <=100, twice + t.Fatalf("ingest calls = %d", f.ingests) + } + e := f.events[events[0].ID] + if e.CustomerID != uuidFor("acme") || e.EventType != usage.MetricGPUHours || e.Timestamp != rfc3339(events[0].WindowStart) { + t.Fatalf("wire event: %+v", e) + } + if e.Properties["value"] != "0.133333333" || e.Properties["sku"] != "NVIDIA-H100-80GB-HBM3" || e.Properties["billing_mode"] != placeholder { + t.Fatalf("properties: %+v", e.Properties) + } + // Unknown customers are sent under their ingest alias so Metronome can + // attribute them once the customer exists. + if got := f.events[cpu.ID].CustomerID; got != "beta" { + t.Fatalf("alias customer_id = %q", got) + } +} + +func TestOldEventsRejectedWithReason(t *testing.T) { + f := newFakeMetronome(t) + a := newAdapter(t, f, nil) + old := gpu("acme", "c", now.Add(-40*24*time.Hour), 1) + err := a.SendEvents(context.Background(), []usage.Event{old, gpu("acme", "c", now.Add(-time.Hour), 1)}) + var partial *destinations.PartialError + if !errors.As(err, &partial) || !strings.Contains(partial.Rejected[old.ID], "34-day") { + t.Fatalf("expected age rejection, got %v", err) + } +} + +func TestRecordedTotalsWholeDays(t *testing.T) { + f := newFakeMetronome(t) + a := newAdapter(t, f, nil) + ctx := context.Background() + if err := a.Bootstrap(ctx, usage.Catalog()); err != nil { + t.Fatal(err) + } + if err := a.EnsureTenant(ctx, usage.Tenant{ID: "acme"}); err != nil { + t.Fatal(err) + } + day := now.Truncate(24 * time.Hour) + var events []usage.Event + for i := 0; i < 60; i++ { + events = append(events, gpu("acme", "c1", day.Add(time.Duration(i)*time.Minute), 8.0/60)) + } + if err := a.SendEvents(ctx, events); err != nil { + t.Fatal(err) + } + q := destinations.TotalsQuery{From: day, To: day.Add(24 * time.Hour), Tenants: []usage.Tenant{{ID: "acme"}}, Metrics: []string{usage.MetricGPUHours}} + totals, err := a.RecordedTotals(ctx, q) + if err != nil { + t.Fatal(err) + } + if len(totals) != 1 || totals[0].Quantity != usage.Round(60*usage.Round(8.0/60)) { + t.Fatalf("totals: %+v", totals) + } + q.From = day.Add(time.Hour) + if _, err := a.RecordedTotals(ctx, q); err == nil { + t.Fatal("non-midnight bounds must be rejected") + } +} + +func TestVerifySignatureOfficialVector(t *testing.T) { + // From docs.metronome.com/guides/platform-configuration/setup-webhooks. + body := []byte("{\n \"id\": \"b2c9e307-624e-4e7d-a5a4-1b74107d78c4\",\n \"type\": \"widget_created\",\n \"properties\": {\n \"customer_id\": \"5f794d50-085a-4db6-8d15-286e518b7225\",\n \"widget_id\": \"0891458d-b6f0-4fdd-a41e-380aae1a1e38\"\n }\n}") + date := "Mon, 02 Jan 2006 22:04:05 GMT" + sig := "b82652fa2246cf1d8a27e591f155c865f68b46c19b9213fd9c052f2419b4742b" + if err := VerifySignature(body, date, sig, "correct-horse-battery-staple", 0, now); err != nil { + t.Fatalf("official test vector rejected: %v", err) + } + if err := VerifySignature(body, date, sig, "correct-horse-battery-staple", DefaultTolerance, now); err == nil { + t.Fatal("2006 timestamp should fail the freshness check") + } + if err := VerifySignature(append(body, ' '), date, sig, "correct-horse-battery-staple", 0, now); err == nil { + t.Fatal("modified body accepted") + } + fresh := now.Format(http.TimeFormat) + if err := VerifySignature(body, fresh, Sign(body, fresh, "s"), "s", DefaultTolerance, now.Add(time.Minute)); err != nil { + t.Fatalf("fresh signature rejected: %v", err) + } +} diff --git a/internal/destinations/metronome/client.go b/internal/destinations/metronome/client.go new file mode 100644 index 0000000..2d00be5 --- /dev/null +++ b/internal/destinations/metronome/client.go @@ -0,0 +1,304 @@ +// Package metronome ships usage to Metronome for rating and invoicing, and +// keeps Metronome customers (optionally linked to Stripe for payment, tax +// and invoice delivery) and contracts in sync with vBilling tenants. +package metronome + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" +) + +// Client is a minimal Metronome REST client. +type Client struct { + base string + token string + http *http.Client +} + +func NewClient(base, token string) *Client { + return &Client{base: strings.TrimRight(base, "/"), token: token, http: destinations.NewHTTPClient()} +} + +// APIError is a Metronome error response ({"message": "..."}). +type APIError struct { + Status int + Message string `json:"message"` + Path string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("metronome %s: HTTP %d: %s", e.Path, e.Status, e.Message) +} + +// IsConflict reports a 409 (ingest alias or uniqueness key already used). +func IsConflict(err error) bool { + for err != nil { + if e, ok := err.(*APIError); ok { + return e.Status == http.StatusConflict + } + u, ok := err.(interface{ Unwrap() error }) + if !ok { + return false + } + err = u.Unwrap() + } + return false +} + +func (c *Client) do(ctx context.Context, method, path string, query url.Values, in, out any) error { + target := c.base + path + if len(query) > 0 { + target += "?" + query.Encode() + } + var body io.Reader + if in != nil { + b, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("User-Agent", "vbilling/0.2") + if in != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("metronome %s %s: %w", method, path, err) + } + defer resp.Body.Close() + b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + if out == nil || len(bytes.TrimSpace(b)) == 0 { + return nil + } + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("metronome %s %s: decode: %w", method, path, err) + } + return nil + } + apiErr := &APIError{Status: resp.StatusCode, Path: method + " " + path} + if json.Unmarshal(b, apiErr) != nil || apiErr.Message == "" { + apiErr.Message = strings.TrimSpace(string(b)) + } + apiErr.Status = resp.StatusCode + switch resp.StatusCode { + case http.StatusBadRequest, http.StatusRequestEntityTooLarge, http.StatusUnprocessableEntity: + return destinations.Permanent(apiErr) + default: // 401/403 config, 404, 408, 409 (callers decide), 429, 5xx + return apiErr + } +} + +// --- ingest --- + +// IngestEvent is one usage event in Metronome's wire format. Property values +// are strings, as Metronome recommends for numeric precision. +type IngestEvent struct { + TransactionID string `json:"transaction_id"` + CustomerID string `json:"customer_id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + Properties map[string]string `json:"properties,omitempty"` +} + +// Ingest sends up to 100 events. No Idempotency-Key: Metronome caches error +// responses under a key, and transaction_id already deduplicates for 34 days. +func (c *Client) Ingest(ctx context.Context, events []IngestEvent) error { + return c.do(ctx, http.MethodPost, "/v1/ingest", nil, events, nil) +} + +// --- customers --- + +type Customer struct { + ID string `json:"id"` + Name string `json:"name"` + IngestAliases []string `json:"ingest_aliases"` +} + +type BillingProviderConfig struct { + BillingProvider string `json:"billing_provider"` + DeliveryMethod string `json:"delivery_method,omitempty"` + Configuration map[string]string `json:"configuration,omitempty"` +} + +type CreateCustomerRequest struct { + Name string `json:"name"` + IngestAliases []string `json:"ingest_aliases"` + BillingProviderConfigs []BillingProviderConfig `json:"customer_billing_provider_configurations,omitempty"` +} + +func (c *Client) CreateCustomer(ctx context.Context, req CreateCustomerRequest) (*Customer, error) { + var resp struct { + Data Customer `json:"data"` + } + if err := c.do(ctx, http.MethodPost, "/v1/customers", nil, req, &resp); err != nil { + return nil, err + } + return &resp.Data, nil +} + +func (c *Client) FindCustomerByAlias(ctx context.Context, alias string) (*Customer, error) { + q := url.Values{} + q.Set("ingest_alias", alias) + var resp struct { + Data []Customer `json:"data"` + } + if err := c.do(ctx, http.MethodGet, "/v1/customers", q, nil, &resp); err != nil { + return nil, err + } + if len(resp.Data) == 0 { + return nil, nil + } + return &resp.Data[0], nil +} + +func (c *Client) GetCustomer(ctx context.Context, id string) (*Customer, error) { + var resp struct { + Data Customer `json:"data"` + } + if err := c.do(ctx, http.MethodGet, "/v1/customers/"+url.PathEscape(id), nil, nil, &resp); err != nil { + return nil, err + } + return &resp.Data, nil +} + +// --- billable metrics --- + +type PropertyFilter struct { + Name string `json:"name"` + Exists *bool `json:"exists,omitempty"` +} + +type EventTypeFilter struct { + InValues []string `json:"in_values"` +} + +type BillableMetric struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + AggregationType string `json:"aggregation_type"` + AggregationKey string `json:"aggregation_key,omitempty"` + EventTypeFilter *EventTypeFilter `json:"event_type_filter,omitempty"` + PropertyFilters []PropertyFilter `json:"property_filters,omitempty"` + GroupKeys [][]string `json:"group_keys,omitempty"` + ArchivedAt string `json:"archived_at,omitempty"` +} + +func (c *Client) ListBillableMetrics(ctx context.Context) ([]BillableMetric, error) { + var out []BillableMetric + q := url.Values{} + q.Set("limit", "100") + for page := 0; page < 50; page++ { + var resp struct { + Data []BillableMetric `json:"data"` + NextPage *string `json:"next_page"` + } + if err := c.do(ctx, http.MethodGet, "/v1/billable-metrics", q, nil, &resp); err != nil { + return nil, err + } + out = append(out, resp.Data...) + if resp.NextPage == nil || *resp.NextPage == "" { + break + } + q.Set("next_page", *resp.NextPage) + } + return out, nil +} + +func (c *Client) CreateBillableMetric(ctx context.Context, m BillableMetric) (string, error) { + var resp struct { + Data struct { + ID string `json:"id"` + } `json:"data"` + } + if err := c.do(ctx, http.MethodPost, "/v1/billable-metrics/create", nil, m, &resp); err != nil { + return "", err + } + return resp.Data.ID, nil +} + +// --- contracts --- + +type UsageStatementSchedule struct { + Frequency string `json:"frequency"` + Day string `json:"day"` +} + +type CreateContractRequest struct { + CustomerID string `json:"customer_id"` + StartingAt string `json:"starting_at"` + RateCardID string `json:"rate_card_id,omitempty"` + RateCardAlias string `json:"rate_card_alias,omitempty"` + UniquenessKey string `json:"uniqueness_key,omitempty"` + UsageStatementSchedule *UsageStatementSchedule `json:"usage_statement_schedule,omitempty"` + BillingProviderConfig *BillingProviderConfig `json:"billing_provider_configuration,omitempty"` +} + +func (c *Client) CreateContract(ctx context.Context, req CreateContractRequest) (string, error) { + var resp struct { + Data struct { + ID string `json:"id"` + } `json:"data"` + } + if err := c.do(ctx, http.MethodPost, "/v1/contracts/create", nil, req, &resp); err != nil { + return "", err + } + return resp.Data.ID, nil +} + +// --- usage (reconciliation) --- + +type UsageRequest struct { + StartingOn string `json:"starting_on"` + EndingBefore string `json:"ending_before"` + WindowSize string `json:"window_size"` + CustomerIDs []string `json:"customer_ids,omitempty"` + BillableMetrics []map[string]any `json:"billable_metrics,omitempty"` +} + +type UsageRow struct { + CustomerID string `json:"customer_id"` + BillableMetricID string `json:"billable_metric_id"` + Value json.Number `json:"value"` +} + +// Usage pages through POST /v1/usage. Bounds must be UTC midnights. +func (c *Client) Usage(ctx context.Context, req UsageRequest) ([]UsageRow, error) { + var out []UsageRow + q := url.Values{} + for page := 0; page < 200; page++ { + var resp struct { + Data []UsageRow `json:"data"` + NextPage *string `json:"next_page"` + } + if err := c.do(ctx, http.MethodPost, "/v1/usage", q, req, &resp); err != nil { + return nil, err + } + out = append(out, resp.Data...) + if resp.NextPage == nil || *resp.NextPage == "" { + break + } + q.Set("next_page", *resp.NextPage) + } + return out, nil +} + +func rfc3339(t time.Time) string { return t.UTC().Format(time.RFC3339) } diff --git a/internal/destinations/metronome/schema_test.go b/internal/destinations/metronome/schema_test.go new file mode 100644 index 0000000..1a974aa --- /dev/null +++ b/internal/destinations/metronome/schema_test.go @@ -0,0 +1,175 @@ +package metronome + +import ( + "encoding/json" + "fmt" + "os" + "regexp" + "testing" +) + +// The request schemas in testdata were extracted (with $refs resolved) from +// Metronome's published OpenAPI spec (docs.metronome.com/openapi.json). +// validate implements the subset of JSON Schema those schemas use, strictly: +// an object schema that lists properties rejects unknown fields, so a +// misspelled field fails here instead of being ignored by Metronome. + +var requestSchemas map[string]any + +func loadSchemas(t *testing.T) map[string]any { + t.Helper() + if requestSchemas == nil { + b, err := os.ReadFile("testdata/openapi-request-schemas.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &requestSchemas); err != nil { + t.Fatal(err) + } + } + return requestSchemas +} + +var uuidFormat = regexp.MustCompile(`^[0-9a-fA-F-]{36}$`) + +func validate(schema any, v any, path string) []string { + s, ok := schema.(map[string]any) + if !ok { + return nil + } + var errs []string + fail := func(format string, args ...any) { errs = append(errs, path+": "+fmt.Sprintf(format, args...)) } + + if v == nil { + if s["nullable"] == true { + return nil + } + } + for _, sub := range asList(s["allOf"]) { + errs = append(errs, validate(sub, v, path)...) + } + if alts := asList(s["oneOf"]); len(alts) > 0 { + matched := false + for _, alt := range alts { + if len(validate(alt, v, path)) == 0 { + matched = true + break + } + } + if !matched { + fail("matches none of the oneOf alternatives") + } + } + if enum := asList(s["enum"]); len(enum) > 0 { + found := false + for _, e := range enum { + if e == v { + found = true + } + } + if !found { + fail("%v not in enum %v", v, enum) + } + } + switch s["type"] { + case "object": + obj, ok := v.(map[string]any) + if !ok { + fail("expected object, got %T", v) + return errs + } + props, _ := s["properties"].(map[string]any) + for _, r := range asList(s["required"]) { + if _, ok := obj[r.(string)]; !ok { + fail("missing required field %q", r) + } + } + for k, val := range obj { + if ps, ok := props[k]; ok { + errs = append(errs, validate(ps, val, path+"."+k)...) + continue + } + switch ap := s["additionalProperties"].(type) { + case bool: + if !ap { + fail("unknown field %q", k) + } + case map[string]any: + errs = append(errs, validate(ap, val, path+"."+k)...) + default: + if len(props) > 0 { + fail("unknown field %q (not in Metronome's schema)", k) + } + } + } + case "array": + arr, ok := v.([]any) + if !ok { + fail("expected array, got %T", v) + return errs + } + if min, ok := s["minItems"].(float64); ok && float64(len(arr)) < min { + fail("%d items < minItems %v", len(arr), min) + } + if max, ok := s["maxItems"].(float64); ok && float64(len(arr)) > max { + fail("%d items > maxItems %v", len(arr), max) + } + for i, it := range arr { + errs = append(errs, validate(s["items"], it, fmt.Sprintf("%s[%d]", path, i))...) + } + case "string": + str, ok := v.(string) + if !ok { + fail("expected string, got %T", v) + return errs + } + if min, ok := s["minLength"].(float64); ok && float64(len(str)) < min { + fail("length %d < minLength %v", len(str), min) + } + if max, ok := s["maxLength"].(float64); ok && float64(len(str)) > max { + fail("length %d > maxLength %v", len(str), max) + } + if s["format"] == "uuid" && !uuidFormat.MatchString(str) { + fail("%q is not a uuid", str) + } + case "number", "integer": + if _, ok := v.(float64); !ok { + fail("expected number, got %T", v) + } + case "boolean": + if _, ok := v.(bool); !ok { + fail("expected boolean, got %T", v) + } + } + return errs +} + +func asList(v any) []any { + l, _ := v.([]any) + return l +} + +// assertMatchesSchema validates a captured request body. +func assertMatchesSchema(t *testing.T, name string, body []byte) { + t.Helper() + var v any + if err := json.Unmarshal(body, &v); err != nil { + t.Fatalf("%s: invalid JSON: %v", name, err) + } + schema, ok := loadSchemas(t)[name] + if !ok { + t.Fatalf("no schema %q", name) + } + for _, e := range validate(schema, v, name) { + t.Errorf("schema violation: %s", e) + } +} + +func TestValidatorCatchesTypos(t *testing.T) { + bad := []byte(`{"name":"x","ingest_alias":["typo"]}`) + var v any + json.Unmarshal(bad, &v) + if errs := validate(loadSchemas(t)["create_customer"], v, "create_customer"); len(errs) == 0 { + t.Fatal("validator accepted an unknown field") + } +} diff --git a/internal/destinations/metronome/testdata/openapi-request-schemas.json b/internal/destinations/metronome/testdata/openapi-request-schemas.json new file mode 100644 index 0000000..44010f1 --- /dev/null +++ b/internal/destinations/metronome/testdata/openapi-request-schemas.json @@ -0,0 +1 @@ +{"create_billable_metric": {"properties": {"aggregation_key": {"type": "string"}, "aggregation_type": {"enum": ["count", "Count", "COUNT", "latest", "Latest", "LATEST", "max", "Max", "MAX", "sum", "Sum", "SUM", "unique", "Unique", "UNIQUE"], "type": "string"}, "custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "event_type_filter": {"properties": {"in_values": {"items": {"type": "string"}, "type": "array"}, "not_in_values": {"items": {"type": "string"}, "type": "array"}}, "type": "object"}, "group_keys": {"items": {"items": {"type": "string"}, "type": "array"}, "type": "array"}, "name": {"type": "string"}, "property_filters": {"items": {"properties": {"exists": {"type": "boolean"}, "in_values": {"items": {"type": "string"}, "type": "array"}, "name": {"type": "string"}, "not_in_values": {"items": {"type": "string"}, "type": "array"}}, "required": ["name"], "type": "object"}, "type": "array"}, "sql": {"type": "string"}}, "required": ["name"], "type": "object"}, "create_contract": {"properties": {"billing_provider_configuration": {"properties": {"billing_provider": {"enum": ["aws_marketplace", "azure_marketplace", "gcp_marketplace", "stripe", "netsuite"], "type": "string"}, "billing_provider_configuration_id": {"format": "uuid", "type": "string"}, "delivery_method": {"enum": ["direct_to_billing_provider", "aws_sqs", "tackle", "aws_sns"], "type": "string"}}, "type": "object"}, "commits": {"items": {"properties": {"access_schedule": {"properties": {"access_type": {"enum": ["SPEND", "spend", "QUANTITY", "quantity"], "type": "string"}, "credit_type_id": {"format": "uuid", "type": "string"}, "schedule_items": {"items": {"properties": {"amount": {"$truncated": true}, "ending_before": {"$truncated": true}, "starting_at": {"$truncated": true}}, "required": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "object"}, "type": "array"}}, "required": ["schedule_items"], "type": "object"}, "amount": {"type": "number"}, "applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "hierarchy_configuration": {"properties": {"child_access": {"oneOf": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}]}}, "required": ["child_access"], "type": "object"}, "invoice_schedule": {"properties": {"credit_type_id": {"format": "uuid", "type": "string"}, "do_not_invoice": {"default": false, "type": "boolean"}, "recurring_schedule": {"properties": {"amount": {"type": {"$truncated": true}}, "amount_distribution": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "ending_before": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "frequency": {"$truncated": true}, "quantity": {"type": {"$truncated": true}}, "starting_at": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "unit_price": {"type": {"$truncated": true}}}, "required": ["starting_at", "ending_before", "frequency", "amount_distribution"], "type": "object"}, "schedule_items": {"items": {"properties": {"amount": {"$truncated": true}, "quantity": {"$truncated": true}, "timestamp": {"$truncated": true}, "unit_price": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "type": "array"}}, "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "rate_type": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"properties": {"exclude": {"items": {"$truncated": true}, "type": "array", "x-mint": {"groups": {"$truncated": true}}, "x-stainless-skip": true}, "presentation_group_values": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "pricing_group_values": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "product_id": {"format": "uuid", "type": "string"}, "product_tags": {"items": {"type": {"$truncated": true}}, "type": "array"}}, "type": "object"}, "type": "array"}, "spend_tracker_attributes": {"allOf": [{"properties": {"counts_as_discounted": {"type": {"$truncated": true}}}, "required": ["counts_as_discounted"], "type": "object"}], "x-mint": {"groups": ["ff:tb-spend-trackers"]}}, "temporary_id": {"type": "string"}, "type": {"enum": ["PREPAID", "prepaid", "POSTPAID", "postpaid"], "type": "string"}}, "required": ["type", "product_id"], "type": "object"}, "type": "array"}, "credits": {"items": {"properties": {"access_schedule": {"properties": {"access_type": {"enum": ["SPEND", "spend", "QUANTITY", "quantity"], "type": "string"}, "credit_type_id": {"format": "uuid", "type": "string"}, "schedule_items": {"items": {"properties": {"amount": {"$truncated": true}, "ending_before": {"$truncated": true}, "starting_at": {"$truncated": true}}, "required": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "object"}, "type": "array"}}, "required": ["schedule_items"], "type": "object"}, "applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "hierarchy_configuration": {"properties": {"child_access": {"oneOf": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}]}}, "required": ["child_access"], "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "rate_type": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"properties": {"exclude": {"items": {"$truncated": true}, "type": "array", "x-mint": {"groups": {"$truncated": true}}, "x-stainless-skip": true}, "presentation_group_values": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "pricing_group_values": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "product_id": {"format": "uuid", "type": "string"}, "product_tags": {"items": {"type": {"$truncated": true}}, "type": "array"}}, "type": "object"}, "type": "array"}}, "required": ["product_id", "access_schedule"], "type": "object"}, "type": "array"}, "custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "customer_id": {"format": "uuid", "type": "string"}, "discounts": {"items": {"properties": {"custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "product_id": {"format": "uuid", "type": "string"}, "schedule": {"properties": {"credit_type_id": {"format": "uuid", "type": "string"}, "do_not_invoice": {"default": false, "type": "boolean"}, "recurring_schedule": {"properties": {"amount": {"type": {"$truncated": true}}, "amount_distribution": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "ending_before": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "frequency": {"$truncated": true}, "quantity": {"type": {"$truncated": true}}, "starting_at": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "unit_price": {"type": {"$truncated": true}}}, "required": ["starting_at", "ending_before", "frequency", "amount_distribution"], "type": "object"}, "schedule_items": {"items": {"properties": {"amount": {"$truncated": true}, "quantity": {"$truncated": true}, "timestamp": {"$truncated": true}, "unit_price": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "type": "array"}}, "type": "object"}}, "required": ["product_id", "schedule"], "type": "object"}, "type": "array", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "ending_before": {"format": "date-time", "type": "string"}, "hierarchy_configuration": {"properties": {"parent": {"properties": {"contract_id": {"format": "uuid", "type": "string"}, "customer_id": {"format": "uuid", "type": "string"}}, "required": ["contract_id", "customer_id"], "type": "object"}, "parent_behavior": {"properties": {"invoice_consolidation_type": {"enum": ["CONCATENATE", "NONE", "concatenate", "none"], "type": "string"}}, "type": "object"}, "payer": {"enum": ["SELF", "PARENT", "self", "parent"], "type": "string"}, "usage_statement_behavior": {"enum": ["CONSOLIDATE", "SEPARATE", "consolidate", "separate"], "type": "string"}}, "type": "object"}, "multiplier_override_prioritization": {"enum": ["LOWEST_MULTIPLIER", "lowest_multiplier", "EXPLICIT", "explicit"], "type": "string"}, "name": {"type": "string"}, "net_payment_terms_days": {"type": "number"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "overrides": {"items": {"properties": {"applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "ending_before": {"format": "date-time", "type": "string"}, "entitled": {"type": "boolean"}, "is_commit_specific": {"type": "boolean"}, "multiplier": {"type": "number"}, "override_specifiers": {"items": {"properties": {"any_commit_or_credit_ids": {"items": {"type": {"$truncated": true}}, "type": "array"}, "billing_frequency": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "commit_ids": {"items": {"type": {"$truncated": true}}, "type": "array"}, "exclude": {"items": {"$truncated": true}, "type": "array", "x-mint": {"groups": {"$truncated": true}}, "x-stainless-skip": true}, "presentation_group_values": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "pricing_group_values": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "product_id": {"format": "uuid", "type": "string"}, "product_tags": {"items": {"type": {"$truncated": true}}, "type": "array"}, "recurring_commit_ids": {"items": {"type": {"$truncated": true}}, "type": "array"}}, "type": "object"}, "type": "array"}, "overwrite_rate": {"properties": {"credit_type_id": {"format": "uuid", "type": "string"}, "custom_rate": {"additionalProperties": true, "type": "object", "x-mint": {"groups": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}]}}, "is_prorated": {"type": "boolean", "x-mint": {"groups": [{"$truncated": true}]}}, "minimum_config": {"properties": {"minimum": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "price": {"type": "number"}, "quantity": {"type": "number", "x-mint": {"groups": [{"$truncated": true}]}}, "rate_type": {"enum": ["FLAT", "flat", "PERCENTAGE", "percentage", "SUBSCRIPTION", "subscription", "TIERED", "tiered", "TIERED_PERCENTAGE", "tiered_percentage", "CUSTOM", "custom"], "type": "string", "x-mint-enum": {"CUSTOM": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "SUBSCRIPTION": [{"$truncated": true}], "TIERED_PERCENTAGE": [{"$truncated": true}], "custom": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "subscription": [{"$truncated": true}], "tiered_percentage": [{"$truncated": true}]}}, "tiers": {"items": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}}, "required": ["rate_type"], "type": "object"}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "starting_at": {"format": "date-time", "type": "string"}, "target": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "tiers": {"items": {"properties": {"multiplier": {"type": "number"}, "size": {"type": "number"}}, "required": ["multiplier"], "type": "object"}, "type": "array"}, "type": {"enum": ["OVERWRITE", "overwrite", "MULTIPLIER", "multiplier", "TIERED", "tiered"], "type": "string"}}, "required": ["starting_at"], "type": "object"}, "type": "array"}, "package_alias": {"type": "string"}, "package_customizations": {"properties": {"add_commits": {"items": {"properties": {"access_schedule": {"properties": {"access_type": {"$truncated": true}, "credit_type_id": {"$truncated": true}, "schedule_items": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "amount": {"type": "number"}, "applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "custom_fields": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "hierarchy_configuration": {"properties": {"child_access": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "invoice_schedule": {"properties": {"credit_type_id": {"$truncated": true}, "do_not_invoice": {"$truncated": true}, "recurring_schedule": {"$truncated": true}, "schedule_items": {"$truncated": true}}, "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": [{"$truncated": true}]}}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "rate_type": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "spend_tracker_attributes": {"allOf": [{"$truncated": true}], "x-mint": {"groups": [{"$truncated": true}]}}, "temporary_id": {"type": "string"}, "type": {"enum": ["PREPAID", "prepaid", "POSTPAID", "postpaid"], "type": "string"}}, "required": ["type", "product_id"], "type": "object"}, "type": "array"}, "add_credits": {"items": {"properties": {"access_schedule": {"properties": {"access_type": {"$truncated": true}, "credit_type_id": {"$truncated": true}, "schedule_items": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "custom_fields": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "hierarchy_configuration": {"properties": {"child_access": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": [{"$truncated": true}]}}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "rate_type": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}}, "required": ["product_id", "access_schedule"], "type": "object"}, "type": "array"}, "add_overrides": {"items": {"properties": {"applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "ending_before": {"format": "date-time", "type": "string"}, "entitled": {"type": "boolean"}, "is_commit_specific": {"type": "boolean"}, "multiplier": {"type": "number"}, "override_specifiers": {"items": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "overwrite_rate": {"properties": {"credit_type_id": {"$truncated": true}, "custom_rate": {"$truncated": true}, "is_prorated": {"$truncated": true}, "minimum_config": {"$truncated": true}, "price": {"$truncated": true}, "quantity": {"$truncated": true}, "rate_type": {"$truncated": true}, "tiers": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "starting_at": {"format": "date-time", "type": "string"}, "target": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "tiers": {"items": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "type": {"enum": ["OVERWRITE", "overwrite", "MULTIPLIER", "multiplier", "TIERED", "tiered"], "type": "string"}}, "required": ["starting_at"], "type": "object"}, "type": "array"}, "add_recurring_commits": {"items": {"allOf": [{"properties": {"access_amount": {"$truncated": true}, "applicable_product_ids": {"$truncated": true}, "applicable_product_tags": {"$truncated": true}, "commit_duration": {"$truncated": true}, "ending_before": {"$truncated": true}, "hierarchy_configuration": {"$truncated": true}, "name": {"$truncated": true}, "netsuite_sales_order_id": {"$truncated": true}, "priority": {"$truncated": true}, "product_id": {"$truncated": true}, "proration": {"$truncated": true}, "rate_type": {"$truncated": true}, "recurrence_frequency": {"$truncated": true}, "rollover_fraction": {"$truncated": true}, "specifiers": {"$truncated": true}, "starting_at": {"$truncated": true}, "subscription_config": {"$truncated": true}, "temporary_id": {"$truncated": true}}, "required": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "object"}, {"properties": {"invoice_amount": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "proration_rounding": {"properties": {"$truncated": true}, "type": {"$truncated": true}}}, "type": "object"}]}, "type": "array"}, "add_recurring_credits": {"items": {"allOf": [{"properties": {"access_amount": {"$truncated": true}, "applicable_product_ids": {"$truncated": true}, "applicable_product_tags": {"$truncated": true}, "commit_duration": {"$truncated": true}, "ending_before": {"$truncated": true}, "hierarchy_configuration": {"$truncated": true}, "name": {"$truncated": true}, "netsuite_sales_order_id": {"$truncated": true}, "priority": {"$truncated": true}, "product_id": {"$truncated": true}, "proration": {"$truncated": true}, "rate_type": {"$truncated": true}, "recurrence_frequency": {"$truncated": true}, "rollover_fraction": {"$truncated": true}, "specifiers": {"$truncated": true}, "starting_at": {"$truncated": true}, "subscription_config": {"$truncated": true}, "temporary_id": {"$truncated": true}}, "required": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "object"}, {"properties": {"proration_rounding": {"properties": {"$truncated": true}, "type": {"$truncated": true}}}, "type": "object"}]}, "type": "array"}, "add_scheduled_charges": {"items": {"properties": {"custom_fields": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": [{"$truncated": true}]}}, "product_id": {"format": "uuid", "type": "string"}, "schedule": {"properties": {"credit_type_id": {"$truncated": true}, "do_not_invoice": {"$truncated": true}, "recurring_schedule": {"$truncated": true}, "schedule_items": {"$truncated": true}}, "type": "object"}}, "required": ["product_id", "schedule"], "type": "object"}, "type": "array"}, "add_spend_trackers": {"items": {"properties": {"alias": {"type": "string"}, "applicable_spend_specifiers": {"items": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "credit_type_id": {"format": "uuid", "type": "string"}, "reset_frequency": {"enum": ["BILLING_PERIOD"], "type": "string"}}, "required": ["alias", "credit_type_id", "reset_frequency", "applicable_spend_specifiers"], "type": "object"}, "type": "array", "x-mint": {"groups": ["ff:tb-spend-trackers"]}}, "add_subscriptions": {"items": {"properties": {"billing_cycle_config": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "collection_schedule": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "custom_fields": {"additionalProperties": {"$truncated": true}, "type": {"$truncated": true}}, "ending_before": {"format": "date-time", "type": "string"}, "initial_quantity": {"type": "number"}, "name": {"type": "string"}, "payment_gate_config": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}, "x-mint": {"$truncated": true}, "x-stainless-skip": {"$truncated": true}}, "proration": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "quantity_management_mode": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "seat_config": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "starting_at": {"format": "date-time", "type": "string"}, "subscription_rate": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "temporary_id": {"type": "string"}}, "required": ["subscription_rate", "collection_schedule", "proration"], "type": "object"}, "type": "array"}, "modify_subscriptions": {"items": {"properties": {"initial_quantity": {"type": "number"}, "initial_seat_ids": {"items": {"type": "string"}, "type": "array"}, "initial_unassigned_seats": {"type": "number"}, "subscription_alias": {"type": "string"}, "template_subscription_id": {"format": "uuid", "type": "string"}}, "type": "object", "x-mint": {"groups": ["ff:package-customizations"]}, "x-stainless-skip": true}, "type": "array"}, "modify_usage_statement_schedule": {"properties": {"billing_anchor_date": {"format": "date-time", "type": "string"}}, "type": "object"}, "set_prepaid_balance_threshold_configuration": {"properties": {"commit": {"allOf": [{"$truncated": true}, {"properties": {"$truncated": true}, "type": {"$truncated": true}}], "type": "object"}, "custom_credit_type_id": {"format": "uuid", "type": "string"}, "discount_configuration": {"properties": {"cap": {"allOf": {"$truncated": true}, "x-mint": {"$truncated": true}}, "payment_fraction": {"type": {"$truncated": true}}}, "required": ["payment_fraction"], "type": "object"}, "is_enabled": {"type": "boolean"}, "payment_gate_config": {"properties": {"payment_gate_type": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "precalculated_tax_config": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "stripe_config": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "tax_type": {"enum": {"$truncated": true}, "type": {"$truncated": true}}}, "required": ["payment_gate_type"], "type": "object"}, "recharge_to_amount": {"type": "number"}, "threshold_amount": {"type": "number"}, "threshold_balance_specifiers": {"items": {"properties": {"exclude": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "type": "array"}}, "required": ["is_enabled", "threshold_amount", "recharge_to_amount", "commit", "payment_gate_config"], "type": "object"}, "set_spend_threshold_configuration": {"properties": {"commit": {"allOf": [{"$truncated": true}], "type": "object"}, "discount_configuration": {"properties": {"cap": {"allOf": {"$truncated": true}, "x-mint": {"$truncated": true}}, "payment_fraction": {"type": {"$truncated": true}}}, "required": ["payment_fraction"], "type": "object"}, "is_enabled": {"type": "boolean"}, "payment_gate_config": {"properties": {"payment_gate_type": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "precalculated_tax_config": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "stripe_config": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "tax_type": {"enum": {"$truncated": true}, "type": {"$truncated": true}}}, "required": ["payment_gate_type"], "type": "object"}, "threshold_amount": {"type": "number"}}, "required": ["is_enabled", "threshold_amount", "commit", "payment_gate_config"], "type": "object"}}, "type": "object", "x-mint": {"groups": ["ff:package-customizations"]}, "x-stainless-skip": true}, "package_id": {"format": "uuid", "type": "string"}, "prepaid_balance_threshold_configuration": {"properties": {"commit": {"allOf": [{"properties": {"name": {"type": {"$truncated": true}}, "priority": {"type": {"$truncated": true}}, "product_id": {"type": {"$truncated": true}}}, "required": ["product_id"], "type": "object"}, {"properties": {"applicable_product_ids": {"items": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "applicable_product_tags": {"items": {"type": {"$truncated": true}}, "type": "array"}, "duration": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "rate_type": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"$truncated": true}, "type": "array", "x-mint": {"groups": {"$truncated": true}}}}, "type": "object"}], "type": "object"}, "custom_credit_type_id": {"format": "uuid", "type": "string"}, "discount_configuration": {"properties": {"cap": {"allOf": [{"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}], "x-mint": {"groups": ["ff:threshold-billing-discounts"]}}, "payment_fraction": {"type": "number"}}, "required": ["payment_fraction"], "type": "object"}, "is_enabled": {"type": "boolean"}, "payment_gate_config": {"properties": {"payment_gate_type": {"enum": ["NONE", "STRIPE", "EXTERNAL"], "type": "string"}, "precalculated_tax_config": {"properties": {"tax_amount": {"type": "number"}, "tax_name": {"type": "string"}}, "required": ["tax_amount"], "type": "object"}, "stripe_config": {"properties": {"invoice_metadata": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "payment_type": {"enum": [{"$truncated": true}, {"$truncated": true}], "type": "string"}}, "required": ["payment_type"], "type": "object"}, "tax_type": {"enum": ["NONE", "STRIPE", "ANROK", "PRECALCULATED"], "type": "string"}}, "required": ["payment_gate_type"], "type": "object"}, "recharge_to_amount": {"type": "number"}, "threshold_amount": {"type": "number"}, "threshold_balance_specifiers": {"items": {"properties": {"exclude": {"items": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}}, "required": ["exclude"], "type": "object"}, "type": "array"}}, "required": ["is_enabled", "threshold_amount", "recharge_to_amount", "commit", "payment_gate_config"], "type": "object"}, "professional_services": {"items": {"properties": {"custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "max_amount": {"type": "number"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "product_id": {"format": "uuid", "type": "string"}, "quantity": {"type": "number"}, "unit_price": {"type": "number"}}, "required": ["product_id", "unit_price", "quantity", "max_amount"], "type": "object"}, "type": "array", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "rate_card_alias": {"type": "string"}, "rate_card_id": {"format": "uuid", "type": "string"}, "recurring_commits": {"items": {"allOf": [{"properties": {"access_amount": {"properties": {"access_type": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "credit_type_id": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "quantity": {"type": {"$truncated": true}}, "unit_price": {"type": {"$truncated": true}}}, "required": ["unit_price"], "type": "object"}, "applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "commit_duration": {"properties": {"unit": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "value": {"type": {"$truncated": true}}}, "required": ["value"], "type": "object"}, "ending_before": {"format": "date-time", "type": "string"}, "hierarchy_configuration": {"properties": {"child_access": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": [{"$truncated": true}]}}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "proration": {"enum": ["NONE", "none", "FIRST", "first", "LAST", "last", "FIRST_AND_LAST", "first_and_last"], "type": "string"}, "rate_type": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "recurrence_frequency": {"enum": ["MONTHLY", "monthly", "QUARTERLY", "quarterly", "ANNUAL", "annual", "WEEKLY", "weekly", "DAILY", "daily"], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "starting_at": {"format": "date-time", "type": "string"}, "subscription_config": {"properties": {"access_policy": {"$truncated": true}, "allocation": {"$truncated": true}, "apply_seat_increase_config": {"$truncated": true}, "subscription_id": {"$truncated": true}}, "required": [{"$truncated": true}, {"$truncated": true}], "type": "object"}, "temporary_id": {"type": "string"}}, "required": ["product_id", "access_amount", "priority", "commit_duration", "starting_at"], "type": "object"}, {"properties": {"invoice_amount": {"properties": {"credit_type_id": {"format": "uuid", "type": "string"}, "quantity": {"type": "number"}, "unit_price": {"type": "number"}}, "required": ["unit_price", "quantity", "credit_type_id"], "type": "object"}, "proration_rounding": {"properties": {"access": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}, "invoice": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}}, "type": "object"}}, "type": "object"}]}, "type": "array"}, "recurring_credits": {"items": {"allOf": [{"properties": {"access_amount": {"properties": {"access_type": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "credit_type_id": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "quantity": {"type": {"$truncated": true}}, "unit_price": {"type": {"$truncated": true}}}, "required": ["unit_price"], "type": "object"}, "applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "commit_duration": {"properties": {"unit": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "value": {"type": {"$truncated": true}}}, "required": ["value"], "type": "object"}, "ending_before": {"format": "date-time", "type": "string"}, "hierarchy_configuration": {"properties": {"child_access": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": [{"$truncated": true}]}}, "priority": {"type": "number"}, "product_id": {"format": "uuid", "type": "string"}, "proration": {"enum": ["NONE", "none", "FIRST", "first", "LAST", "last", "FIRST_AND_LAST", "first_and_last"], "type": "string"}, "rate_type": {"enum": ["COMMIT_RATE", "commit_rate", "LIST_RATE", "list_rate"], "type": "string"}, "recurrence_frequency": {"enum": ["MONTHLY", "monthly", "QUARTERLY", "quarterly", "ANNUAL", "annual", "WEEKLY", "weekly", "DAILY", "daily"], "type": "string"}, "rollover_fraction": {"type": "number"}, "specifiers": {"items": {"properties": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "starting_at": {"format": "date-time", "type": "string"}, "subscription_config": {"properties": {"access_policy": {"$truncated": true}, "allocation": {"$truncated": true}, "apply_seat_increase_config": {"$truncated": true}, "subscription_id": {"$truncated": true}}, "required": [{"$truncated": true}, {"$truncated": true}], "type": "object"}, "temporary_id": {"type": "string"}}, "required": ["product_id", "access_amount", "priority", "commit_duration", "starting_at"], "type": "object"}, {"properties": {"proration_rounding": {"properties": {"access": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}}, "type": "object"}}, "type": "object"}]}, "type": "array"}, "reseller_royalties": {"items": {"properties": {"applicable_product_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "applicable_product_tags": {"items": {"type": "string"}, "type": "array"}, "aws_options": {"properties": {"aws_account_number": {"type": "string"}, "aws_offer_id": {"type": "string"}, "aws_payer_reference_id": {"type": "string"}}, "type": "object"}, "ending_before": {"format": "date-time", "type": "string"}, "fraction": {"type": "number"}, "gcp_options": {"properties": {"gcp_account_id": {"type": "string"}, "gcp_offer_id": {"type": "string"}}, "type": "object"}, "netsuite_reseller_id": {"type": "string"}, "reseller_contract_value": {"type": "number"}, "reseller_type": {"enum": ["AWS", "AWS_PRO_SERVICE", "GCP", "GCP_PRO_SERVICE"], "type": "string"}, "starting_at": {"format": "date-time", "type": "string"}}, "required": ["reseller_type", "starting_at", "fraction", "netsuite_reseller_id"], "type": "object"}, "type": "array", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "revenue_system_configuration": {"properties": {"delivery_method": {"enum": ["direct_to_billing_provider"], "type": "string", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}, "provider": {"enum": ["netsuite"], "type": "string", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}, "revenue_system_configuration_id": {"format": "uuid", "type": "string"}}, "type": "object", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}, "salesforce_opportunity_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "scheduled_charges": {"items": {"properties": {"custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "name": {"minLength": 1, "type": "string"}, "netsuite_sales_order_id": {"type": "string", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "product_id": {"format": "uuid", "type": "string"}, "schedule": {"properties": {"credit_type_id": {"format": "uuid", "type": "string"}, "do_not_invoice": {"default": false, "type": "boolean"}, "recurring_schedule": {"properties": {"amount": {"type": {"$truncated": true}}, "amount_distribution": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "ending_before": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "frequency": {"$truncated": true}, "quantity": {"type": {"$truncated": true}}, "starting_at": {"format": {"$truncated": true}, "type": {"$truncated": true}}, "unit_price": {"type": {"$truncated": true}}}, "required": ["starting_at", "ending_before", "frequency", "amount_distribution"], "type": "object"}, "schedule_items": {"items": {"properties": {"amount": {"$truncated": true}, "quantity": {"$truncated": true}, "timestamp": {"$truncated": true}, "unit_price": {"$truncated": true}}, "required": [{"$truncated": true}], "type": "object"}, "type": "array"}}, "type": "object"}}, "required": ["product_id", "schedule"], "type": "object"}, "type": "array"}, "scheduled_charges_on_usage_invoices": {"enum": ["ALL"], "type": "string"}, "spend_threshold_configuration": {"properties": {"commit": {"allOf": [{"properties": {"name": {"type": {"$truncated": true}}, "priority": {"type": {"$truncated": true}}, "product_id": {"type": {"$truncated": true}}}, "required": ["product_id"], "type": "object"}], "type": "object"}, "discount_configuration": {"properties": {"cap": {"allOf": [{"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}], "x-mint": {"groups": ["ff:threshold-billing-discounts"]}}, "payment_fraction": {"type": "number"}}, "required": ["payment_fraction"], "type": "object"}, "is_enabled": {"type": "boolean"}, "payment_gate_config": {"properties": {"payment_gate_type": {"enum": ["NONE", "STRIPE", "EXTERNAL"], "type": "string"}, "precalculated_tax_config": {"properties": {"tax_amount": {"type": "number"}, "tax_name": {"type": "string"}}, "required": ["tax_amount"], "type": "object"}, "stripe_config": {"properties": {"invoice_metadata": {"additionalProperties": {"type": {"$truncated": true}}, "type": "object"}, "payment_type": {"enum": [{"$truncated": true}, {"$truncated": true}], "type": "string"}}, "required": ["payment_type"], "type": "object"}, "tax_type": {"enum": ["NONE", "STRIPE", "ANROK", "PRECALCULATED"], "type": "string"}}, "required": ["payment_gate_type"], "type": "object"}, "threshold_amount": {"type": "number"}}, "required": ["is_enabled", "threshold_amount", "commit", "payment_gate_config"], "type": "object"}, "spend_trackers": {"items": {"properties": {"alias": {"type": "string"}, "applicable_spend_specifiers": {"items": {"properties": {"discounted": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "sources": {"items": {"enum": {"$truncated": true}, "type": {"$truncated": true}}, "type": "array"}, "spend_type": {"enum": [{"$truncated": true}], "type": "string"}}, "required": ["spend_type", "sources"], "type": "object"}, "type": "array"}, "credit_type_id": {"format": "uuid", "type": "string"}, "reset_frequency": {"enum": ["BILLING_PERIOD"], "type": "string"}}, "required": ["alias", "credit_type_id", "reset_frequency", "applicable_spend_specifiers"], "type": "object"}, "type": "array", "x-mint": {"groups": ["ff:tb-spend-trackers"]}}, "starting_at": {"format": "date-time", "type": "string"}, "subscriptions": {"items": {"properties": {"billing_cycle_config": {"properties": {"anchor_date": {"format": "date-time", "type": "string"}, "invoice_placement": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}}, "type": "object"}, "collection_schedule": {"enum": ["ADVANCE", "ARREARS", "advance", "arrears"], "type": "string"}, "custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "ending_before": {"format": "date-time", "type": "string"}, "initial_quantity": {"type": "number"}, "name": {"type": "string"}, "payment_gate_config": {"properties": {"payment_gate_type": {"enum": [{"$truncated": true}, {"$truncated": true}], "type": "string"}, "tax_type": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}}, "required": ["payment_gate_type"], "type": "object", "x-mint": {"groups": ["ff:payment-gated-subscriptions-enabled"]}, "x-stainless-skip": true}, "proration": {"properties": {"invoice_behavior": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "is_prorated": {"type": "boolean"}, "rounding": {"properties": {"$truncated": true}, "required": {"$truncated": true}, "type": {"$truncated": true}}}, "type": "object"}, "quantity_management_mode": {"enum": ["SEAT_BASED", "seat_based", "QUANTITY_ONLY", "quantity_only"], "type": "string"}, "seat_config": {"properties": {"initial_seat_ids": {"items": {"type": {"$truncated": true}}, "type": "array"}, "initial_unassigned_seats": {"type": "number"}, "seat_group_key": {"type": "string"}}, "required": ["seat_group_key", "initial_seat_ids"], "type": "object"}, "starting_at": {"format": "date-time", "type": "string"}, "subscription_rate": {"properties": {"billing_frequency": {"enum": [{"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}, {"$truncated": true}], "type": "string"}, "product_id": {"format": "uuid", "type": "string"}}, "required": ["billing_frequency", "product_id"], "type": "object"}, "temporary_id": {"type": "string"}}, "required": ["subscription_rate", "collection_schedule", "proration"], "type": "object"}, "type": "array"}, "total_contract_value": {"type": "number", "x-mint": {"groups": ["client_id:e3147d6d-4101-4cd1-9888-ce3afeeac5b2"]}}, "transition": {"properties": {"from_contract_id": {"format": "uuid", "type": "string"}, "future_invoice_behavior": {"properties": {"trueup": {"enum": ["remove", "as_is", "REMOVE", "AS_IS"], "nullable": true, "type": "string"}}, "type": "object"}, "type": {"enum": ["RENEWAL", "renewal"], "type": "string"}}, "required": ["type", "from_contract_id"], "type": "object"}, "uniqueness_key": {"maxLength": 128, "minLength": 1, "type": "string"}, "usage_filter": {"properties": {"group_key": {"type": "string"}, "group_values": {"items": {"type": "string"}, "type": "array"}, "starting_at": {"format": "date-time", "type": "string"}}, "required": ["group_key", "group_values"], "type": "object"}, "usage_statement_schedule": {"properties": {"billing_anchor_date": {"format": "date-time", "type": "string"}, "day": {"enum": ["FIRST_OF_MONTH", "first_of_month", "CONTRACT_START", "contract_start", "CUSTOM_DATE", "custom_date"], "type": "string"}, "frequency": {"enum": ["MONTHLY", "monthly", "QUARTERLY", "quarterly", "ANNUAL", "annual", "WEEKLY", "weekly"], "type": "string"}, "invoice_generation_starting_at": {"format": "date-time", "type": "string"}}, "required": ["frequency"], "type": "object"}}, "required": ["customer_id", "starting_at"], "type": "object"}, "create_customer": {"properties": {"custom_fields": {"additionalProperties": {"type": "string"}, "type": "object"}, "customer_billing_provider_configurations": {"items": {"properties": {"billing_provider": {"enum": ["aws_marketplace", "azure_marketplace", "gcp_marketplace", "stripe", "netsuite"], "type": "string"}, "configuration": {"additionalProperties": true, "type": "object"}, "delivery_method": {"enum": ["direct_to_billing_provider", "aws_sqs", "tackle", "aws_sns"], "type": "string"}, "delivery_method_id": {"format": "uuid", "type": "string"}, "tax_provider": {"enum": ["anrok", "avalara", "stripe"], "type": "string"}}, "required": ["billing_provider"], "type": "object"}, "type": "array"}, "customer_revenue_system_configurations": {"items": {"properties": {"configuration": {"additionalProperties": true, "type": "object"}, "delivery_method": {"enum": ["direct_to_billing_provider"], "type": "string", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}, "delivery_method_id": {"format": "uuid", "type": "string"}, "provider": {"enum": ["netsuite"], "type": "string", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}}, "required": ["provider"], "type": "object", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}, "type": "array", "x-mint": {"groups": ["ff:revenue-rec-configurations-enabled"]}}, "external_id": {"maxLength": 128, "minLength": 1, "type": "string"}, "ingest_aliases": {"items": {"maxLength": 128, "minLength": 1, "type": "string"}, "maxItems": 2000, "type": "array"}, "name": {"type": "string"}}, "required": ["name"], "type": "object"}, "ingest": {"items": {"properties": {"customer_id": {"maxLength": 1024, "minLength": 1, "type": "string"}, "event_type": {"maxLength": 1024, "minLength": 1, "type": "string"}, "properties": {"additionalProperties": true, "type": "object"}, "timestamp": {"type": "string"}, "transaction_id": {"maxLength": 128, "minLength": 1, "type": "string"}}, "required": ["customer_id", "event_type", "timestamp", "transaction_id"], "type": "object"}, "maxItems": 100, "minItems": 1, "type": "array"}, "usage": {"properties": {"billable_metrics": {"items": {"properties": {"group_by": {"properties": {"key": {"type": "string"}, "values": {"items": {"minLength": 1, "type": "string"}, "maxItems": 200, "minItems": 1, "type": "array"}}, "required": ["key"], "type": "object"}, "id": {"format": "uuid", "type": "string"}}, "required": ["id"], "type": "object"}, "type": "array"}, "customer_ids": {"items": {"format": "uuid", "type": "string"}, "type": "array"}, "ending_before": {"format": "date-time", "type": "string"}, "starting_on": {"format": "date-time", "type": "string"}, "window_size": {"enum": ["hour", "day", "none", "HOUR", "DAY", "NONE", "Hour", "Day", "None"], "type": "string"}}, "required": ["window_size", "starting_on", "ending_before"], "type": "object"}} \ No newline at end of file diff --git a/internal/destinations/metronome/webhook.go b/internal/destinations/metronome/webhook.go new file mode 100644 index 0000000..76287f9 --- /dev/null +++ b/internal/destinations/metronome/webhook.go @@ -0,0 +1,56 @@ +package metronome + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "strings" + "time" +) + +// DefaultTolerance is the replay window Metronome recommends. +const DefaultTolerance = 5 * time.Minute + +// VerifySignature checks Metronome-Webhook-Signature, which is +// hex(HMAC-SHA256(secret, X-Metronome-Date + "\n" + raw body)). Pass the +// X-Metronome-Date header (Date is accepted by Metronome only for backward +// compatibility because proxies rewrite it). tolerance <= 0 skips the +// freshness check. +func VerifySignature(body []byte, date, signature, secret string, tolerance time.Duration, now time.Time) error { + if secret == "" { + return errors.New("metronome webhook secret not configured") + } + if date == "" || signature == "" { + return errors.New("missing X-Metronome-Date or Metronome-Webhook-Signature") + } + if tolerance > 0 { + sent, err := http.ParseTime(date) + if err != nil { + return fmt.Errorf("invalid X-Metronome-Date: %w", err) + } + if age := now.Sub(sent); age > tolerance || age < -tolerance { + return fmt.Errorf("webhook date outside tolerance (%s)", age.Round(time.Second)) + } + } + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(date)) + mac.Write([]byte("\n")) + mac.Write(body) + expected := hex.EncodeToString(mac.Sum(nil)) + if !hmac.Equal([]byte(expected), []byte(strings.ToLower(strings.TrimSpace(signature)))) { + return errors.New("metronome signature mismatch") + } + return nil +} + +// Sign produces a signature for tests and local tooling. +func Sign(body []byte, date, secret string) string { + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(date)) + mac.Write([]byte("\n")) + mac.Write(body) + return hex.EncodeToString(mac.Sum(nil)) +} diff --git a/internal/destinations/noop/adapter.go b/internal/destinations/noop/adapter.go index 2ea1114..3f6d4ce 100644 --- a/internal/destinations/noop/adapter.go +++ b/internal/destinations/noop/adapter.go @@ -8,6 +8,7 @@ import ( "github.com/vclusterlabs-experiments/vbilling/internal/config" "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" ) func init() { @@ -20,25 +21,25 @@ type Adapter struct{} func (a *Adapter) Name() string { return "noop" } -func (a *Adapter) Bootstrap(ctx context.Context) error { - log.Println("[noop] bootstrap (no-op)") +func (a *Adapter) Bootstrap(ctx context.Context, metrics []usage.MetricDef) error { + log.Printf("[noop] bootstrap: %d metrics in catalog", len(metrics)) return nil } -func (a *Adapter) EnsureTenant(ctx context.Context, t destinations.Tenant) error { - log.Printf("[noop] ensure tenant %s (%s)", t.ExternalID, t.DisplayName) +func (a *Adapter) EnsureTenant(ctx context.Context, t usage.Tenant) error { + log.Printf("[noop] ensure tenant %s (%s) clusters=%v", t.ID, t.DisplayName, t.Clusters) return nil } -func (a *Adapter) RemoveTenant(ctx context.Context, externalID string) error { - log.Printf("[noop] remove tenant %s", externalID) +func (a *Adapter) RemoveTenant(ctx context.Context, t usage.Tenant) error { + log.Printf("[noop] remove tenant %s", t.ID) return nil } -func (a *Adapter) SendEvents(ctx context.Context, events []destinations.UsageEvent) error { +func (a *Adapter) SendEvents(ctx context.Context, events []usage.Event) error { for _, ev := range events { - log.Printf("[noop] event tenant=%s metric=%s value=%v ts=%s", - ev.TenantExternalID, ev.MetricCode, ev.Value, ev.Timestamp.Format("15:04:05")) + log.Printf("[noop] event tenant=%s metric=%s qty=%v %s sku=%s region=%s window=%s", + ev.Tenant, ev.Metric, ev.Quantity, ev.Unit, ev.SKU, ev.Region, ev.WindowStart.Format("15:04:05")) } return nil } diff --git a/internal/destinations/sentcache.go b/internal/destinations/sentcache.go new file mode 100644 index 0000000..db1f59e --- /dev/null +++ b/internal/destinations/sentcache.go @@ -0,0 +1,41 @@ +package destinations + +import "sync" + +// SentCache remembers recently delivered event IDs so a retried batch only +// resends what actually failed. It is bounded; the oldest IDs fall out first. +type SentCache struct { + mu sync.Mutex + set map[string]struct{} + ring []string + next int +} + +// NewSentCache keeps up to n IDs. +func NewSentCache(n int) *SentCache { + if n <= 0 { + n = 1 + } + return &SentCache{set: make(map[string]struct{}, n), ring: make([]string, n)} +} + +func (c *SentCache) Has(id string) bool { + c.mu.Lock() + defer c.mu.Unlock() + _, ok := c.set[id] + return ok +} + +func (c *SentCache) Add(id string) { + c.mu.Lock() + defer c.mu.Unlock() + if _, ok := c.set[id]; ok { + return + } + if old := c.ring[c.next]; old != "" { + delete(c.set, old) + } + c.ring[c.next] = id + c.set[id] = struct{}{} + c.next = (c.next + 1) % len(c.ring) +} diff --git a/internal/destinations/stripe/adapter.go b/internal/destinations/stripe/adapter.go new file mode 100644 index 0000000..2d9ba28 --- /dev/null +++ b/internal/destinations/stripe/adapter.go @@ -0,0 +1,593 @@ +package stripe + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash/fnv" + "log" + "math" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +func init() { + destinations.Register("stripe", func(cfg *config.Config) (destinations.Destination, error) { + if cfg.StripeAPIKey == "" { + return nil, errors.New("STRIPE_API_KEY is required when the stripe adapter is enabled") + } + state, err := kv.Open(filepath.Join(cfg.DataDir, "state-stripe.json")) + if err != nil { + return nil, fmt.Errorf("open stripe state: %w", err) + } + return New(NewClient(cfg.StripeAPIBase, cfg.StripeAPIKey, cfg.StripeAPIVersion, cfg.StripeMaxRPS), cfg, state), nil + }) +} + +// Stripe accepts meter events up to 35 days old; keep an hour of margin. +const maxEventAge = 35*24*time.Hour - time.Hour + +// Metadata keys vBilling owns on Stripe objects. +const ( + metaTenant = "vbilling_tenant_id" + metaPlan = "vbilling_plan" +) + +// Adapter ships usage to Stripe Billing Meters. +// +// Stripe meters cannot price on dimensions, so events are routed to one +// meter per (metric, split values): with the default split on "sku", H100 +// and L40S GPU-hours land on separate meters and can carry separate prices. +type Adapter struct { + c *Client + cfg *config.Config + state *kv.Store + sent *destinations.SentCache + now func() time.Time + + meterMu sync.Mutex + meters map[string]Meter // event_name -> meter + metersLoaded bool + + cusMu sync.Mutex // serializes customer creation + + priceMu sync.Mutex + prices []Price + pricesTime time.Time + + workers int +} + +// New builds the adapter around a client (exported for tests). +func New(c *Client, cfg *config.Config, state *kv.Store) *Adapter { + return &Adapter{ + c: c, cfg: cfg, state: state, + sent: destinations.NewSentCache(200_000), + now: time.Now, + meters: map[string]Meter{}, + workers: 4, + } +} + +func (a *Adapter) Name() string { return "stripe" } + +// Accepts skips informational metrics; Stripe would bill anything it meters. +func (a *Adapter) Accepts(metric string) bool { return destinations.BillableOnly(metric) } + +// --- meters --- + +func (a *Adapter) loadMetersLocked(ctx context.Context) error { + if a.metersLoaded { + return nil + } + ms, err := a.c.ListMeters(ctx) + if err != nil { + return fmt.Errorf("list meters: %w", err) + } + for _, m := range ms { + a.meters[m.EventName] = m + } + a.metersLoaded = true + return nil +} + +// ensureMeter returns the meter for an event name, creating or reactivating it. +func (a *Adapter) ensureMeter(ctx context.Context, eventName, displayName string) (Meter, error) { + a.meterMu.Lock() + defer a.meterMu.Unlock() + if err := a.loadMetersLocked(ctx); err != nil { + return Meter{}, err + } + if m, ok := a.meters[eventName]; ok { + if m.Status == "inactive" { + m, err := a.c.ReactivateMeter(ctx, m.ID) + if err != nil { + return Meter{}, fmt.Errorf("reactivate meter %s: %w", eventName, err) + } + a.meters[eventName] = m + return m, nil + } + return m, nil + } + m, err := a.c.CreateMeter(ctx, eventName, displayName) + if err != nil { + return Meter{}, fmt.Errorf("create meter %s: %w", eventName, err) + } + log.Printf("[stripe] created meter %s (%s)", eventName, m.ID) + a.meters[eventName] = m + _ = a.state.Set("meter/"+eventName, m.ID) + return m, nil +} + +// MeterName maps an event to its Stripe meter event_name. +func (a *Adapter) MeterName(e *usage.Event) string { + name := e.Metric + for _, key := range a.cfg.StripeSplitMetersBy { + if v := fieldValue(e, key); v != "" { + name += "__" + slug(v) + } + } + if len(name) > 100 { + sum := sha256.Sum256([]byte(name)) + name = name[:91] + "_" + hex.EncodeToString(sum[:4]) + } + return name +} + +func meterDisplayName(e *usage.Event, eventName string) string { + def, ok := usage.Lookup(e.Metric) + if !ok { + return eventName + } + if suffix := strings.TrimPrefix(eventName, e.Metric); suffix != "" { + return def.Name + " (" + strings.ReplaceAll(strings.TrimPrefix(suffix, "__"), "__", ", ") + ")" + } + return def.Name +} + +func fieldValue(e *usage.Event, key string) string { + switch key { + case "sku": + return e.SKU + case "region": + return e.Region + case "project": + return e.Project + default: + return e.Dim(key) + } +} + +func slug(s string) string { + var b strings.Builder + lastUnderscore := false + for _, r := range strings.ToLower(s) { + ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if ok { + b.WriteRune(r) + lastUnderscore = false + } else if !lastUnderscore { + b.WriteByte('_') + lastUnderscore = true + } + } + return strings.Trim(b.String(), "_") +} + +// Bootstrap creates (or reactivates) one meter per billable metric. Split +// meters (per SKU, per region, ...) are created on first use. +func (a *Adapter) Bootstrap(ctx context.Context, metrics []usage.MetricDef) error { + var errs []error + for _, def := range metrics { + if !def.Billable { + continue + } + if _, err := a.ensureMeter(ctx, def.Code, def.Name); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// --- customers --- + +func (a *Adapter) customerFor(ctx context.Context, t usage.Tenant) (string, error) { + if id := t.ProviderIDs["stripe"]; id != "" { + a.remember(t.ID, id) + return id, nil + } + if id, ok := a.state.Get("cus/" + t.ID); ok { + return id, nil + } + a.cusMu.Lock() + defer a.cusMu.Unlock() + if id, ok := a.state.Get("cus/" + t.ID); ok { // created while we waited + return id, nil + } + cus, err := a.c.FindCustomer(ctx, metaTenant, t.ID, t.Email) + if err != nil { + return "", fmt.Errorf("search customer %s: %w", t.ID, err) + } + if cus == nil { + f := customerForm(t) + // Hour-bucketed key: retries within the hour replay, and a cached + // 5xx cannot block creation for Stripe's full 24h key retention. + key := fmt.Sprintf("vbilling-cus-%s-%s", shortHash(t.ID), a.now().UTC().Format("2006010215")) + if cus, err = a.c.CreateCustomer(ctx, f, key); err != nil { + return "", fmt.Errorf("create customer %s: %w", t.ID, err) + } + log.Printf("[stripe] created customer %s for tenant %s", cus.ID, t.ID) + } + a.remember(t.ID, cus.ID) + return cus.ID, nil +} + +func (a *Adapter) remember(tenant, cus string) { + _ = a.state.Set("cus/"+tenant, cus) + _ = a.state.Set("tenant/"+cus, tenant) +} + +func customerForm(t usage.Tenant) url.Values { + f := url.Values{} + name := t.DisplayName + if name == "" { + name = t.ID + } + f.Set("name", truncate(name, 256)) + if t.Email != "" { + f.Set("email", t.Email) + } + f.Set("metadata["+metaTenant+"]", t.ID) + set := func(k, v string) { + if v != "" { + f.Set("metadata["+k+"]", truncate(v, 500)) + } + } + set("vbilling_region", t.Region) + set("vbilling_project", t.Project) + set("vbilling_tenant_class", t.Class) + set(metaPlan, t.Plan) + set("vbilling_clusters", strings.Join(t.Clusters, ",")) + return f +} + +// EnsureTenant creates or updates the Stripe customer and, when +// STRIPE_AUTO_SUBSCRIBE is on, subscribes it to the tenant plan's prices. +func (a *Adapter) EnsureTenant(ctx context.Context, t usage.Tenant) error { + cusID, err := a.EnsureCustomer(ctx, t) + if err != nil { + return err + } + if a.cfg.StripeAutoSubscribe { + return a.ensureSubscription(ctx, t, cusID) + } + return nil +} + +// EnsureCustomer creates or updates the tenant's Stripe customer and returns +// its ID. The Metronome adapter uses it to link customers for invoicing. +func (a *Adapter) EnsureCustomer(ctx context.Context, t usage.Tenant) (string, error) { + cusID, err := a.customerFor(ctx, t) + if err != nil { + return "", err + } + f := customerForm(t) + fp := shortHash(f.Encode()) + if prev, _ := a.state.Get("cusfp/" + t.ID); prev != fp { + if err := a.c.UpdateCustomer(ctx, cusID, f); err != nil { + if IsCode(err, "resource_missing") { + // Deleted in Stripe: forget it so the next pass recreates it. + _ = a.state.Delete("cus/" + t.ID) + _ = a.state.Delete("tenant/" + cusID) + } + return "", fmt.Errorf("update customer %s: %w", cusID, err) + } + _ = a.state.Set("cusfp/"+t.ID, fp) + } + return cusID, nil +} + +// planPrices returns active metered prices on vBilling-managed meters tagged +// metadata[vbilling_plan]=plan in the tenant's currency. +func (a *Adapter) planPrices(ctx context.Context, plan, currency string) ([]Price, error) { + a.priceMu.Lock() + if a.prices == nil || a.now().Sub(a.pricesTime) > 5*time.Minute { + ps, err := a.c.ListMeteredPrices(ctx) + if err != nil { + a.priceMu.Unlock() + return nil, fmt.Errorf("list prices: %w", err) + } + a.prices, a.pricesTime = ps, a.now() + } + all := a.prices + a.priceMu.Unlock() + + a.meterMu.Lock() + managed := map[string]bool{} + for _, m := range a.meters { + managed[m.ID] = true + } + a.meterMu.Unlock() + + var out []Price + for _, p := range all { + if !p.Active || p.Recurring == nil || p.Recurring.UsageType != "metered" || !managed[p.Recurring.Meter] { + continue + } + if p.Metadata[metaPlan] != plan || !strings.EqualFold(p.Currency, currency) { + continue + } + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (a *Adapter) ensureSubscription(ctx context.Context, t usage.Tenant, cusID string) error { + plan := t.Plan + if plan == "" { + plan = a.cfg.DefaultPlanCode + } + currency := t.Currency + if currency == "" { + currency = a.cfg.BillingCurrency + } + prices, err := a.planPrices(ctx, plan, currency) + if err != nil { + return err + } + if len(prices) == 0 { + log.Printf("[stripe] no active metered prices tagged %s=%s in %s; tenant %s not subscribed", metaPlan, plan, currency, t.ID) + return nil + } + if len(prices) > 20 { + return fmt.Errorf("plan %s has %d prices; Stripe allows 20 items per subscription", plan, len(prices)) + } + subs, err := a.c.ListSubscriptions(ctx, cusID) + if err != nil { + return fmt.Errorf("list subscriptions: %w", err) + } + for _, s := range subs { + if s.Metadata[metaTenant] != t.ID || s.Status == "canceled" || s.Status == "incomplete_expired" { + continue + } + have := map[string]bool{} + for _, it := range s.Items.Data { + have[it.Price.ID] = true + } + for _, p := range prices { + if !have[p.ID] { + if err := a.c.AddSubscriptionItem(ctx, s.ID, p.ID); err != nil { + return fmt.Errorf("add price %s to %s: %w", p.ID, s.ID, err) + } + log.Printf("[stripe] added price %s to subscription %s", p.ID, s.ID) + } + } + _ = a.state.Set("sub/"+t.ID, s.ID) + return nil + } + ids := make([]string, len(prices)) + for i, p := range prices { + ids[i] = p.ID + } + key := fmt.Sprintf("vbilling-sub-%s-%s", shortHash(t.ID+"|"+plan), a.now().UTC().Format("2006010215")) + sub, err := a.c.CreateSubscription(ctx, cusID, ids, map[string]string{metaTenant: t.ID, metaPlan: plan}, key) + if err != nil { + return fmt.Errorf("create subscription: %w", err) + } + log.Printf("[stripe] subscribed tenant %s to plan %s (%s, %d prices)", t.ID, plan, sub.ID, len(ids)) + return a.state.Set("sub/"+t.ID, sub.ID) +} + +// RemoveTenant optionally cancels the subscription at period end so final +// usage is still invoiced. Customers are never deleted. +func (a *Adapter) RemoveTenant(ctx context.Context, t usage.Tenant) error { + if !a.cfg.StripeCancelOnRemove { + log.Printf("[stripe] tenant %s offboarded; customer left in place (STRIPE_CANCEL_ON_REMOVE=false)", t.ID) + return nil + } + subID, ok := a.state.Get("sub/" + t.ID) + if !ok { + return nil + } + return a.c.CancelAtPeriodEnd(ctx, subID) +} + +// TenantForCustomer resolves an inbound webhook's customer to a tenant. +func (a *Adapter) TenantForCustomer(ctx context.Context, cusID string) (string, bool) { + if t, ok := a.state.Get("tenant/" + cusID); ok { + return t, true + } + cus, err := a.c.GetCustomer(ctx, cusID) + if err != nil || cus.Metadata[metaTenant] == "" { + return "", false + } + a.remember(cus.Metadata[metaTenant], cusID) + return cus.Metadata[metaTenant], true +} + +// --- events --- + +type job struct { + ev usage.Event + meter Meter + cus string +} + +// SendEvents posts each event to its meter. Work is partitioned by +// (customer, meter) because Stripe allows one concurrent meter-event call +// per customer per meter. +func (a *Adapter) SendEvents(ctx context.Context, events []usage.Event) error { + rejected := map[string]string{} + var jobs []job + for _, ev := range events { + if a.sent.Has(ev.ID) || !a.Accepts(ev.Metric) { + continue + } + if a.now().Sub(ev.WindowStart) > maxEventAge { + rejected[ev.ID] = "older than Stripe's 35-day meter event window" + continue + } + cus, err := a.customerFor(ctx, usage.Tenant{ID: ev.Tenant}) + if err != nil { + return err + } + name := a.MeterName(&ev) + m, err := a.ensureMeter(ctx, name, meterDisplayName(&ev, name)) + if err != nil { + if destinations.IsPermanent(err) { + rejected[ev.ID] = err.Error() + continue + } + return err + } + jobs = append(jobs, job{ev: ev, meter: m, cus: cus}) + } + + parts := make([][]job, a.workers) + for _, j := range jobs { + h := fnv.New32a() + h.Write([]byte(j.cus + "|" + j.meter.EventName)) + parts[h.Sum32()%uint32(a.workers)] = append(parts[h.Sum32()%uint32(a.workers)], j) + } + var ( + wg sync.WaitGroup + mu sync.Mutex + retryErr error + ) + for _, part := range parts { + if len(part) == 0 { + continue + } + wg.Add(1) + go func(part []job) { + defer wg.Done() + for _, j := range part { + err := a.c.CreateMeterEvent(ctx, j.meter, j.cus, FormatValue(j.ev.Quantity), j.ev.ID, j.ev.WindowStart) + switch { + case err == nil || isDuplicate(err): + a.sent.Add(j.ev.ID) + case destinations.IsPermanent(err): + mu.Lock() + rejected[j.ev.ID] = err.Error() + mu.Unlock() + default: + mu.Lock() + if retryErr == nil { + retryErr = err + } + mu.Unlock() + return // stop this partition; the batch is retried + } + } + }(part) + } + wg.Wait() + if retryErr != nil { + return retryErr + } + if len(rejected) > 0 { + return &destinations.PartialError{Rejected: rejected} + } + return nil +} + +// isDuplicate treats Stripe's duplicate-identifier rejection as delivered. +func isDuplicate(err error) bool { + var apiErr *APIError + if !asAPIError(err, &apiErr) { + return false + } + return strings.Contains(apiErr.Code, "duplicate") || + strings.Contains(strings.ToLower(apiErr.Message), "already exists with identifier") +} + +// FormatValue renders a quantity as a plain decimal with at most 15 +// significant digits (Stripe's precision guard) and 9 decimals. +func FormatValue(q float64) string { + if q == 0 || math.IsNaN(q) || math.IsInf(q, 0) { + return "0" + } + intDigits := 1 + if a := math.Abs(q); a >= 1 { + intDigits = int(math.Floor(math.Log10(a))) + 1 + } + decimals := 15 - intDigits + if decimals > 9 { + decimals = 9 + } + if decimals < 0 { + decimals = 0 + } + s := strconv.FormatFloat(q, 'f', decimals, 64) + if strings.Contains(s, ".") { + s = strings.TrimRight(strings.TrimRight(s, "0"), ".") + } + return s +} + +// --- reconciliation --- + +// RecordedTotals sums Stripe's meter event summaries per tenant and metric, +// across every split meter of that metric. Bounds are truncated to minutes. +func (a *Adapter) RecordedTotals(ctx context.Context, q destinations.TotalsQuery) ([]destinations.Total, error) { + from, to := q.From.UTC().Truncate(time.Minute), q.To.UTC().Truncate(time.Minute) + a.meterMu.Lock() + if err := a.loadMetersLocked(ctx); err != nil { + a.meterMu.Unlock() + return nil, err + } + byMetric := map[string][]Meter{} + for name, m := range a.meters { + for _, metric := range q.Metrics { + if name == metric || strings.HasPrefix(name, metric+"__") { + byMetric[metric] = append(byMetric[metric], m) + } + } + } + a.meterMu.Unlock() + + var out []destinations.Total + for _, t := range q.Tenants { + cus, ok := a.state.Get("cus/" + t.ID) + if !ok { + continue + } + for _, metric := range q.Metrics { + var sum float64 + for _, m := range byMetric[metric] { + v, err := a.c.MeterTotal(ctx, m.ID, cus, from, to) + if err != nil { + return nil, fmt.Errorf("event summaries %s/%s: %w", t.ID, m.EventName, err) + } + sum += v + } + out = append(out, destinations.Total{Tenant: t.ID, Metric: metric, Quantity: usage.Round(sum)}) + } + } + return out, nil +} + +func shortHash(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:8]) +} + +var ( + _ destinations.Destination = (*Adapter)(nil) + _ destinations.EventFilter = (*Adapter)(nil) + _ destinations.Reconciler = (*Adapter)(nil) + _ destinations.CustomerResolver = (*Adapter)(nil) +) diff --git a/internal/destinations/stripe/adapter_test.go b/internal/destinations/stripe/adapter_test.go new file mode 100644 index 0000000..885981d --- /dev/null +++ b/internal/destinations/stripe/adapter_test.go @@ -0,0 +1,414 @@ +package stripe + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +var now = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + +func newTestAdapter(t *testing.T, f *fakeStripe, statePath string, mutate ...func(*config.Config)) *Adapter { + t.Helper() + cfg := &config.Config{ + StripeSplitMetersBy: []string{"sku"}, + DefaultPlanCode: "standard", + BillingCurrency: "usd", + } + for _, m := range mutate { + m(cfg) + } + state, err := kv.Open(statePath) + if err != nil { + t.Fatal(err) + } + a := New(NewClient(f.srv.URL, "sk_test_123", "2026-09-30.endive", 0), cfg, state) + a.now = func() time.Time { return now } + return a +} + +func gpuEvent(tenant, sku string, ws time.Time, q float64) usage.Event { + e := usage.Event{Tenant: tenant, Metric: usage.MetricGPUHours, Quantity: q, SKU: sku, WindowStart: ws, WindowEnd: ws.Add(time.Minute), + Dimensions: map[string]string{usage.DimGPUType: sku}} + e.Finalize() + return e +} + +func TestBootstrapCreatesBillableMetersIdempotently(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + a := newTestAdapter(t, f, "") + ctx := context.Background() + if err := a.Bootstrap(ctx, usage.Catalog()); err != nil { + t.Fatal(err) + } + want := len(usage.Billable()) + if len(f.meters) != want { + t.Fatalf("created %d meters, want %d billable", len(f.meters), want) + } + for _, m := range f.meters { + if m.EventName == usage.MetricGPUUtilization || m.EventName == usage.MetricGPUDowntimeHours { + t.Errorf("informational metric %s got a meter", m.EventName) + } + } + // A fresh adapter (restart) finds existing meters instead of duplicating. + f.meters[0].Status = "inactive" + b := newTestAdapter(t, f, "") + if err := b.Bootstrap(ctx, usage.Catalog()); err != nil { + t.Fatal(err) + } + if len(f.meters) != want { + t.Fatalf("second bootstrap created duplicates: %d meters", len(f.meters)) + } + if f.meters[0].Status != "active" { + t.Fatal("inactive meter was not reactivated") + } +} + +func TestEnsureTenantNeverDuplicatesCustomers(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + statePath := filepath.Join(t.TempDir(), "state.json") + a := newTestAdapter(t, f, statePath) + ctx := context.Background() + tenant := usage.Tenant{ID: "acme", DisplayName: "Acme AI", Email: "billing@acme.test", Region: "ap-southeast-2", Clusters: []string{"vcluster-team-a-gpu"}} + + for i := 0; i < 3; i++ { // search is not yet consistent; local state must prevent duplicates + if err := a.EnsureTenant(ctx, tenant); err != nil { + t.Fatal(err) + } + } + if len(f.customers) != 1 { + t.Fatalf("created %d customers", len(f.customers)) + } + for _, c := range f.customers { + if c.Metadata[metaTenant] != "acme" || c.Metadata["vbilling_region"] != "ap-southeast-2" || c.Email != "billing@acme.test" { + t.Fatalf("customer metadata: %+v", c) + } + } + + // Lost local state (new PVC): search finds the existing customer. + f.makeSearchable() + b := newTestAdapter(t, f, "") + if err := b.EnsureTenant(ctx, tenant); err != nil { + t.Fatal(err) + } + if len(f.customers) != 1 { + t.Fatalf("lost state led to a duplicate customer: %d", len(f.customers)) + } + + // A pinned customer ID (tenant-cluster annotation) wins. + pinned := &Customer{ID: "cus_pinned", Metadata: map[string]string{}} + f.customers[pinned.ID] = pinned + c := newTestAdapter(t, f, "") + if err := c.EnsureTenant(ctx, usage.Tenant{ID: "beta", ProviderIDs: map[string]string{"stripe": "cus_pinned"}}); err != nil { + t.Fatal(err) + } + if got, _ := c.TenantForCustomer(ctx, "cus_pinned"); got != "beta" { + t.Fatalf("pinned customer resolves to %q", got) + } +} + +func TestSendEventsSplitsMetersPerSKU(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + a := newTestAdapter(t, f, "") + ctx := context.Background() + ws := now.Add(-10 * time.Minute) + events := []usage.Event{ + gpuEvent("acme", "NVIDIA-H100-80GB-HBM3", ws, 8.0/60), + gpuEvent("acme", "NVIDIA-L40S", ws, 2.0/60), + gpuEvent("beta", "NVIDIA-H100-80GB-HBM3", ws, 1.0/60), + } + cpu := usage.Event{Tenant: "acme", Metric: usage.MetricCPUCoreHours, Quantity: 0.5, WindowStart: ws, WindowEnd: ws.Add(time.Minute)} + cpu.Finalize() + util := usage.Event{Tenant: "acme", Metric: usage.MetricGPUUtilization, Quantity: 40, WindowStart: ws, WindowEnd: ws.Add(time.Minute)} + util.Finalize() + events = append(events, cpu, util) + + if err := a.SendEvents(ctx, events); err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, e := range f.events { + got[e.EventName+"|"+e.Customer] = e.Value + if e.Timestamp != ws.Unix() { + t.Errorf("timestamp %d, want window start %d", e.Timestamp, ws.Unix()) + } + } + if len(f.events) != 4 { + t.Fatalf("recorded %d meter events, want 4 (utilization is informational): %+v", len(f.events), f.events) + } + for _, name := range []string{"vcluster_gpu_hours__nvidia_h100_80gb_hbm3", "vcluster_gpu_hours__nvidia_l40s", "vcluster_cpu_core_hours"} { + found := false + for k := range got { + if strings.HasPrefix(k, name+"|") { + found = true + } + } + if !found { + t.Errorf("no events on meter %s (have %v)", name, got) + } + } + for k, v := range got { + if strings.HasPrefix(k, "vcluster_gpu_hours__nvidia_h100_80gb_hbm3|") && v != "0.133333333" && v != "0.016666667" { + t.Errorf("unexpected H100 value %s", v) + } + } + // Every event's identifier is the deterministic vBilling ID. + for _, e := range f.events { + if !strings.HasPrefix(e.Identifier, "vb1_") { + t.Errorf("identifier %q is not the event ID", e.Identifier) + } + } +} + +func TestRetryResendsOnlyWhatFailed(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + a := newTestAdapter(t, f, "") + a.workers = 1 // deterministic order + ctx := context.Background() + ws := now.Add(-5 * time.Minute) + var events []usage.Event + for i := 0; i < 6; i++ { + events = append(events, gpuEvent("acme", "NVIDIA-H100-80GB-HBM3", ws.Add(time.Duration(i)*time.Minute), 0.1)) + } + // Warm up customer + meter, then fail the 4th meter event. + if err := a.SendEvents(ctx, events[:3]); err != nil { + t.Fatal(err) + } + f.failNext["POST /v1/billing/meter_events"] = 1 + err := a.SendEvents(ctx, events) + if err == nil || destinations.IsPermanent(err) { + t.Fatalf("a 500 must be retryable, got %v", err) + } + if err := a.SendEvents(ctx, events); err != nil { + t.Fatal(err) + } + if len(f.events) != 6 { + t.Fatalf("recorded %d events, want exactly 6", len(f.events)) + } + // A fresh adapter (lost sent cache) resending everything hits Stripe's + // identifier dedupe, which counts as delivered. + b := newTestAdapter(t, f, "") + if err := b.SendEvents(ctx, events); err != nil { + t.Fatalf("duplicate identifiers must not fail delivery: %v", err) + } + if len(f.events) != 6 { + t.Fatalf("dedupe broken: %d events", len(f.events)) + } +} + +func TestTooOldEventsAreRejectedNotRetried(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + a := newTestAdapter(t, f, "") + old := gpuEvent("acme", "NVIDIA-H100-80GB-HBM3", now.Add(-40*24*time.Hour), 1) + fresh := gpuEvent("acme", "NVIDIA-H100-80GB-HBM3", now.Add(-time.Hour), 1) + err := a.SendEvents(context.Background(), []usage.Event{old, fresh}) + var partial *destinations.PartialError + if !errors.As(err, &partial) || len(partial.Rejected) != 1 || partial.Rejected[old.ID] == "" { + t.Fatalf("expected PartialError rejecting only the old event, got %v", err) + } + if len(f.events) != 1 { + t.Fatalf("recorded %d events", len(f.events)) + } +} + +func TestAutoSubscribeUsesPlanTaggedPrices(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + a := newTestAdapter(t, f, "", func(c *config.Config) { c.StripeAutoSubscribe = true }) + ctx := context.Background() + if err := a.Bootstrap(ctx, usage.Catalog()); err != nil { + t.Fatal(err) + } + meterID := func(name string) string { + for _, m := range f.meters { + if m.EventName == name { + return m.ID + } + } + t.Fatalf("no meter %s", name) + return "" + } + price := func(id, meter, plan, currency string) Price { + p := Price{ID: id, Active: true, Currency: currency, Metadata: map[string]string{metaPlan: plan}} + p.Recurring = &struct { + UsageType string `json:"usage_type"` + Meter string `json:"meter"` + }{UsageType: "metered", Meter: meter} + return p + } + f.prices = []Price{ + price("price_gpu", meterID(usage.MetricGPUHours), "standard", "usd"), + price("price_cpu", meterID(usage.MetricCPUCoreHours), "standard", "usd"), + price("price_aud", meterID(usage.MetricGPUHours), "standard", "aud"), // wrong currency + price("price_ent", meterID(usage.MetricGPUHours), "enterprise", "usd"), // other plan + price("price_foreign", "mtr_not_ours", "standard", "usd"), // not a vBilling meter + } + tenant := usage.Tenant{ID: "acme"} + if err := a.EnsureTenant(ctx, tenant); err != nil { + t.Fatal(err) + } + if len(f.subs) != 1 { + t.Fatalf("subscriptions: %d", len(f.subs)) + } + for _, s := range f.subs { + var ids []string + for _, it := range s.Items.Data { + ids = append(ids, it.Price.ID) + } + if strings.Join(ids, ",") != "price_cpu,price_gpu" { + t.Fatalf("subscription items %v", ids) + } + } + // A price added to the plan later is added to the existing subscription. + f.prices = append(f.prices, price("price_lb", meterID(usage.MetricLBHours), "standard", "usd")) + a.pricesTime = time.Time{} + if err := a.EnsureTenant(ctx, tenant); err != nil { + t.Fatal(err) + } + for _, s := range f.subs { + if len(s.Items.Data) != 3 { + t.Fatalf("expected 3 items after plan change, got %d", len(s.Items.Data)) + } + } + if len(f.subs) != 1 { + t.Fatal("plan change created a second subscription") + } +} + +func TestRecordedTotalsSumsSplitMeters(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + a := newTestAdapter(t, f, "") + ctx := context.Background() + ws := now.Add(-30 * time.Minute) + events := []usage.Event{ + gpuEvent("acme", "NVIDIA-H100-80GB-HBM3", ws, 0.5), + gpuEvent("acme", "NVIDIA-L40S", ws, 0.25), + gpuEvent("acme", "NVIDIA-L40S", ws.Add(time.Minute), 0.25), + } + if err := a.SendEvents(ctx, events); err != nil { + t.Fatal(err) + } + totals, err := a.RecordedTotals(ctx, destinations.TotalsQuery{ + From: now.Add(-time.Hour), To: now, Tenants: []usage.Tenant{{ID: "acme"}}, Metrics: []string{usage.MetricGPUHours}, + }) + if err != nil { + t.Fatal(err) + } + if len(totals) != 1 || totals[0].Quantity != 1.0 { + t.Fatalf("totals: %+v", totals) + } +} + +func TestMeterNameIsBoundedAndStable(t *testing.T) { + a := &Adapter{cfg: &config.Config{StripeSplitMetersBy: []string{"sku", "region"}}} + e := gpuEvent("acme", "NVIDIA-A100-SXM4-80GB-mig-1g.10gb", now, 1) + e.Region = "ap-southeast-2" + if got := a.MeterName(&e); got != "vcluster_gpu_hours__nvidia_a100_sxm4_80gb_mig_1g_10gb__ap_southeast_2" { + t.Fatalf("meter name %q", got) + } + e.SKU = strings.Repeat("x", 200) + if got := a.MeterName(&e); len(got) != 100 || got != a.MeterName(&e) { + t.Fatalf("long meter name not bounded/stable: %d", len(got)) + } +} + +func TestFormatValue(t *testing.T) { + cases := map[float64]string{ + 0: "0", + 1: "1", + 0.016666667: "0.016666667", + 1.0 / 3600: "0.000277778", + 8.0 / 60: "0.133333333", + 123456.123456789: "123456.123456789", + 1234567890.123456789: "1234567890.12346", + 12345678901234567890: "12345678901234567168", + } + for in, want := range cases { + if got := FormatValue(in); got != want { + t.Errorf("FormatValue(%v) = %q, want %q", in, got, want) + } + } +} + +func TestVerifySignature(t *testing.T) { + body := []byte(`{"id":"evt_1","type":"invoice.payment_failed"}`) + secret := "whsec_test" + header := SignatureHeader(body, secret, now) + if err := VerifySignature(body, header, secret, DefaultTolerance, now); err != nil { + t.Fatalf("valid signature rejected: %v", err) + } + if err := VerifySignature([]byte(`{"id":"evt_2"}`), header, secret, DefaultTolerance, now); err == nil { + t.Fatal("tampered body accepted") + } + if err := VerifySignature(body, header, secret, DefaultTolerance, now.Add(10*time.Minute)); err == nil { + t.Fatal("stale signature accepted") + } + rolled := header + ",v1=deadbeef" + if err := VerifySignature(body, "v1=deadbeef,"+strings.TrimPrefix(rolled, ""), secret, DefaultTolerance, now); err != nil { + t.Fatalf("multiple v1 signatures must be accepted when one matches: %v", err) + } + if err := VerifySignature(body, header, "whsec_other", DefaultTolerance, now); err == nil { + t.Fatal("wrong secret accepted") + } +} + +func TestSearchResultsMustMatchTenantMetadata(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + // A customer that search returns but that belongs to someone else. + f.customers["cus_other"] = &Customer{ID: "cus_other", Metadata: map[string]string{metaTenant: "acme-old"}} + f.searchable["cus_other"] = true + f.searchAll = true + a := newTestAdapter(t, f, "") + if err := a.EnsureTenant(context.Background(), usage.Tenant{ID: "acme"}); err != nil { + t.Fatal(err) + } + if got, _ := a.state.Get("cus/acme"); got == "cus_other" { + t.Fatal("tenant attributed to a customer whose metadata does not match") + } +} + +// Stripe does not offer customer search in every region (accounts in India +// get a 400). Lost local state must still find existing customers, by listing. +func TestCustomerLookupWithoutSearch(t *testing.T) { + f := newFakeStripe(t, func() time.Time { return now }) + f.noSearch = true + for i := 0; i < 150; i++ { // more than one page + id := fmt.Sprintf("cus_x%03d", i) + f.customers[id] = &Customer{ID: id, Metadata: map[string]string{}} + } + f.customers["cus_same_email"] = &Customer{ID: "cus_same_email", Email: "billing@acme.test", Metadata: map[string]string{metaTenant: "acme-old"}} + f.customers["cus_y_acme"] = &Customer{ID: "cus_y_acme", Email: "billing@acme.test", Metadata: map[string]string{metaTenant: "acme"}} + f.customers["cus_y_beta"] = &Customer{ID: "cus_y_beta", Metadata: map[string]string{metaTenant: "beta"}} + before := len(f.customers) + + a := newTestAdapter(t, f, "") + ctx := context.Background() + if err := a.EnsureTenant(ctx, usage.Tenant{ID: "acme", Email: "billing@acme.test"}); err != nil { + t.Fatal(err) + } + if err := a.EnsureTenant(ctx, usage.Tenant{ID: "beta"}); err != nil { // no email: full scan, page 2 + t.Fatal(err) + } + if err := a.EnsureTenant(ctx, usage.Tenant{ID: "gamma"}); err != nil { + t.Fatal(err) + } + for tenant, want := range map[string]string{"acme": "cus_y_acme", "beta": "cus_y_beta"} { + if got, _ := a.state.Get("cus/" + tenant); got != want { + t.Errorf("tenant %s resolved to %q, want %s", tenant, got, want) + } + } + if len(f.customers) != before+1 { + t.Errorf("customers %d -> %d, want only gamma created", before, len(f.customers)) + } + if n := f.count("GET /v1/customers/search"); n != 1 { + t.Errorf("search tried %d times; once unavailable it must not be retried", n) + } +} diff --git a/internal/destinations/stripe/client.go b/internal/destinations/stripe/client.go new file mode 100644 index 0000000..65b23f8 --- /dev/null +++ b/internal/destinations/stripe/client.go @@ -0,0 +1,484 @@ +// Package stripe ships usage to Stripe Billing Meters and keeps Stripe +// customers (and optionally subscriptions) in sync with vBilling tenants. +package stripe + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" +) + +// Client is a minimal Stripe REST client (form-encoded v1 API). +type Client struct { + base string + key string + version string + http *http.Client + limit *limiter + + noSearch atomic.Bool // the account has no customer search (not offered in every region) +} + +// NewClient builds a client. rps <= 0 picks a default from the key mode. +func NewClient(base, key, version string, rps float64) *Client { + base = strings.TrimRight(base, "/") + if rps <= 0 { + switch { + case !strings.Contains(base, "api.stripe.com"): + rps = 1000 // stripe-mock or a proxy + case strings.HasPrefix(key, "sk_test_") || strings.HasPrefix(key, "rk_test_"): + rps = 20 // sandboxes share a 25 req/s global limit + default: + rps = 200 + } + } + return &Client{base: base, key: key, version: version, http: destinations.NewHTTPClient(), limit: newLimiter(rps)} +} + +// APIError is a Stripe error response. +type APIError struct { + Status int + Type string `json:"type"` + Code string `json:"code"` + Message string `json:"message"` + Param string `json:"param"` + Path string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("stripe %s: HTTP %d %s/%s: %s", e.Path, e.Status, e.Type, e.Code, e.Message) +} + +// IsCode reports whether err is a Stripe API error with the given code. +func IsCode(err error, code string) bool { + var apiErr *APIError + return asAPIError(err, &apiErr) && apiErr.Code == code +} + +func asAPIError(err error, target **APIError) bool { + for err != nil { + if e, ok := err.(*APIError); ok { + *target = e + return true + } + u, ok := err.(interface{ Unwrap() error }) + if !ok { + return false + } + err = u.Unwrap() + } + return false +} + +func (c *Client) do(ctx context.Context, method, path string, form url.Values, idemKey string, out any) error { + if err := c.limit.Wait(ctx); err != nil { + return err + } + target := c.base + path + var body io.Reader + if method == http.MethodGet { + if len(form) > 0 { + target += "?" + form.Encode() + } + } else { + body = strings.NewReader(form.Encode()) + } + req, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.key) + req.Header.Set("Stripe-Version", c.version) + req.Header.Set("User-Agent", "vbilling/0.2") + if method != http.MethodGet { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if idemKey != "" { + req.Header.Set("Idempotency-Key", idemKey) + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("stripe %s %s: %w", method, path, err) + } + defer resp.Body.Close() + b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + if out == nil { + return nil + } + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("stripe %s %s: decode: %w", method, path, err) + } + return nil + } + + apiErr := &APIError{Status: resp.StatusCode, Path: method + " " + path} + var envelope struct { + Error *APIError `json:"error"` + } + if json.Unmarshal(b, &envelope) == nil && envelope.Error != nil { + envelope.Error.Status, envelope.Error.Path = apiErr.Status, apiErr.Path + apiErr = envelope.Error + } else { + apiErr.Message = strings.TrimSpace(string(b)) + } + // Stripe tells us explicitly when it knows better than the status code. + switch resp.Header.Get("Stripe-Should-Retry") { + case "true": + return apiErr + case "false": + return destinations.Permanent(apiErr) + } + switch resp.StatusCode { + case http.StatusBadRequest, http.StatusRequestEntityTooLarge, http.StatusUnprocessableEntity: + return destinations.Permanent(apiErr) + default: // 401/403 (config), 404, 409 (concurrency), 429, 5xx + return apiErr + } +} + +// --- objects --- + +type Meter struct { + ID string `json:"id"` + EventName string `json:"event_name"` + DisplayName string `json:"display_name"` + Status string `json:"status"` + CustomerMapping struct { + EventPayloadKey string `json:"event_payload_key"` + } `json:"customer_mapping"` + ValueSettings struct { + EventPayloadKey string `json:"event_payload_key"` + } `json:"value_settings"` +} + +func (m Meter) customerKey() string { + if m.CustomerMapping.EventPayloadKey != "" { + return m.CustomerMapping.EventPayloadKey + } + return "stripe_customer_id" +} + +func (m Meter) valueKey() string { + if m.ValueSettings.EventPayloadKey != "" { + return m.ValueSettings.EventPayloadKey + } + return "value" +} + +type Customer struct { + ID string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` + Deleted bool `json:"deleted"` + Metadata map[string]string `json:"metadata"` +} + +type Price struct { + ID string `json:"id"` + Active bool `json:"active"` + Currency string `json:"currency"` + Metadata map[string]string `json:"metadata"` + Recurring *struct { + UsageType string `json:"usage_type"` + Meter string `json:"meter"` + } `json:"recurring"` +} + +type Subscription struct { + ID string `json:"id"` + Status string `json:"status"` + Customer string `json:"customer"` + Metadata map[string]string `json:"metadata"` + Items struct { + Data []struct { + ID string `json:"id"` + Price Price `json:"price"` + } `json:"data"` + } `json:"items"` +} + +type list[T any] struct { + Data []T `json:"data"` + HasMore bool `json:"has_more"` + NextPage string `json:"next_page"` +} + +type idObject interface{ objectID() string } + +func (m Meter) objectID() string { return m.ID } +func (p Price) objectID() string { return p.ID } +func (s Subscription) objectID() string { return s.ID } + +// listAll pages through a v1 list endpoint (max 50 pages as a safety net). +func listAll[T idObject](ctx context.Context, c *Client, path string, params url.Values) ([]T, error) { + var out []T + if params == nil { + params = url.Values{} + } + params.Set("limit", "100") + for page := 0; page < 50; page++ { + var l list[T] + if err := c.do(ctx, http.MethodGet, path, params, "", &l); err != nil { + return nil, err + } + out = append(out, l.Data...) + if !l.HasMore || len(l.Data) == 0 { + return out, nil + } + params.Set("starting_after", l.Data[len(l.Data)-1].objectID()) + } + return out, nil +} + +// --- meters --- + +func (c *Client) ListMeters(ctx context.Context) ([]Meter, error) { + return listAll[Meter](ctx, c, "/v1/billing/meters", nil) +} + +func (c *Client) CreateMeter(ctx context.Context, eventName, displayName string) (Meter, error) { + f := url.Values{} + f.Set("display_name", truncate(displayName, 250)) + f.Set("event_name", eventName) + f.Set("default_aggregation[formula]", "sum") + f.Set("customer_mapping[type]", "by_id") + f.Set("customer_mapping[event_payload_key]", "stripe_customer_id") + f.Set("value_settings[event_payload_key]", "value") + var m Meter + err := c.do(ctx, http.MethodPost, "/v1/billing/meters", f, "vbilling-meter-"+eventName, &m) + return m, err +} + +func (c *Client) ReactivateMeter(ctx context.Context, id string) (Meter, error) { + var m Meter + err := c.do(ctx, http.MethodPost, "/v1/billing/meters/"+url.PathEscape(id)+"/reactivate", url.Values{}, "", &m) + return m, err +} + +// CreateMeterEvent records usage. The identifier deduplicates server-side; +// no Idempotency-Key is sent so a cached 5xx can never pin the event. +func (c *Client) CreateMeterEvent(ctx context.Context, m Meter, customerID, value, identifier string, ts time.Time) error { + f := url.Values{} + f.Set("event_name", m.EventName) + f.Set("payload["+m.customerKey()+"]", customerID) + f.Set("payload["+m.valueKey()+"]", value) + f.Set("identifier", identifier) + f.Set("timestamp", strconv.FormatInt(ts.Unix(), 10)) + return c.do(ctx, http.MethodPost, "/v1/billing/meter_events", f, "", nil) +} + +// MeterTotal sums a meter's aggregated value for one customer over +// [start, end). Both bounds must sit on minute boundaries. +func (c *Client) MeterTotal(ctx context.Context, meterID, customerID string, start, end time.Time) (float64, error) { + params := url.Values{} + params.Set("customer", customerID) + params.Set("start_time", strconv.FormatInt(start.Unix(), 10)) + params.Set("end_time", strconv.FormatInt(end.Unix(), 10)) + type summary struct { + ID string `json:"id"` + AggregatedValue float64 `json:"aggregated_value"` + } + var total float64 + for page := 0; page < 50; page++ { + var l list[summary] + if err := c.do(ctx, http.MethodGet, "/v1/billing/meters/"+url.PathEscape(meterID)+"/event_summaries", params, "", &l); err != nil { + return 0, err + } + for _, s := range l.Data { + total += s.AggregatedValue + } + if !l.HasMore || len(l.Data) == 0 { + break + } + params.Set("starting_after", l.Data[len(l.Data)-1].ID) + } + return total, nil +} + +// --- customers --- + +func (c *Client) GetCustomer(ctx context.Context, id string) (*Customer, error) { + var cus Customer + if err := c.do(ctx, http.MethodGet, "/v1/customers/"+url.PathEscape(id), nil, "", &cus); err != nil { + return nil, err + } + return &cus, nil +} + +// SearchCustomer finds a customer by metadata. Stripe search is eventually +// consistent (usually under a minute), so callers keep their own mapping. +func (c *Client) SearchCustomer(ctx context.Context, key, value string) (*Customer, error) { + params := url.Values{} + params.Set("query", fmt.Sprintf("metadata['%s']:'%s'", key, strings.ReplaceAll(value, "'", `\'`))) + var l list[Customer] + if err := c.do(ctx, http.MethodGet, "/v1/customers/search", params, "", &l); err != nil { + return nil, err + } + for _, cus := range l.Data { + // Trust the metadata, not the search engine: a customer is only ours + // if it carries exactly this value. Usage must never be attributed to + // another tenant's customer. + if !cus.Deleted && cus.Metadata[key] == value { + return &cus, nil + } + } + return nil, nil +} + +// FindCustomer returns the customer whose metadata[key] is value. It uses +// search where the account has it; Stripe does not offer search in every +// region (accounts in India get "The search feature is temporarily +// unavailable in your region."), so a rejected search falls back to listing +// customers, filtered by email first when one is known. +func (c *Client) FindCustomer(ctx context.Context, key, value, email string) (*Customer, error) { + if !c.noSearch.Load() { + cus, err := c.SearchCustomer(ctx, key, value) + var apiErr *APIError + if err == nil || !asAPIError(err, &apiErr) || apiErr.Status != http.StatusBadRequest { + return cus, err + } + // The query is well formed, so a 400 means search is unusable here. + c.noSearch.Store(true) + log.Printf("[stripe] customer search unavailable (%s); finding customers by listing them", apiErr.Message) + } + if email != "" { + if cus, err := c.scanCustomers(ctx, url.Values{"email": {email}}, key, value); err != nil || cus != nil { + return cus, err + } + } + return c.scanCustomers(ctx, url.Values{}, key, value) +} + +// scanCustomers pages through customers until one carries metadata[key] = value. +func (c *Client) scanCustomers(ctx context.Context, params url.Values, key, value string) (*Customer, error) { + params.Set("limit", "100") + for page := 0; page < 1000; page++ { + var l list[Customer] + if err := c.do(ctx, http.MethodGet, "/v1/customers", params, "", &l); err != nil { + return nil, err + } + for _, cus := range l.Data { + if !cus.Deleted && cus.Metadata[key] == value { + return &cus, nil + } + } + if !l.HasMore || len(l.Data) == 0 { + return nil, nil + } + params.Set("starting_after", l.Data[len(l.Data)-1].ID) + } + return nil, fmt.Errorf("no customer with %s=%s in the first 100,000 customers; pin it with the stripe-customer-id annotation", key, value) +} + +func (c *Client) CreateCustomer(ctx context.Context, f url.Values, idemKey string) (*Customer, error) { + var cus Customer + if err := c.do(ctx, http.MethodPost, "/v1/customers", f, idemKey, &cus); err != nil { + return nil, err + } + return &cus, nil +} + +func (c *Client) UpdateCustomer(ctx context.Context, id string, f url.Values) error { + return c.do(ctx, http.MethodPost, "/v1/customers/"+url.PathEscape(id), f, "", nil) +} + +// --- prices & subscriptions --- + +func (c *Client) ListMeteredPrices(ctx context.Context) ([]Price, error) { + params := url.Values{} + params.Set("active", "true") + params.Set("type", "recurring") + return listAll[Price](ctx, c, "/v1/prices", params) +} + +func (c *Client) ListSubscriptions(ctx context.Context, customerID string) ([]Subscription, error) { + params := url.Values{} + params.Set("customer", customerID) + params.Set("status", "all") + return listAll[Subscription](ctx, c, "/v1/subscriptions", params) +} + +func (c *Client) CreateSubscription(ctx context.Context, customerID string, priceIDs []string, meta map[string]string, idemKey string) (*Subscription, error) { + f := url.Values{} + f.Set("customer", customerID) + for i, p := range priceIDs { + f.Set(fmt.Sprintf("items[%d][price]", i), p) + } + for k, v := range meta { + f.Set("metadata["+k+"]", v) + } + var sub Subscription + if err := c.do(ctx, http.MethodPost, "/v1/subscriptions", f, idemKey, &sub); err != nil { + return nil, err + } + return &sub, nil +} + +func (c *Client) AddSubscriptionItem(ctx context.Context, subID, priceID string) error { + f := url.Values{} + f.Set("subscription", subID) + f.Set("price", priceID) + return c.do(ctx, http.MethodPost, "/v1/subscription_items", f, "vbilling-si-"+subID+"-"+priceID, nil) +} + +func (c *Client) CancelAtPeriodEnd(ctx context.Context, subID string) error { + f := url.Values{} + f.Set("cancel_at_period_end", "true") + return c.do(ctx, http.MethodPost, "/v1/subscriptions/"+url.PathEscape(subID), f, "", nil) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + +// limiter is a small token bucket (avoids a dependency for one function). +type limiter struct { + mu sync.Mutex + rate float64 + tokens float64 + last time.Time +} + +func newLimiter(rps float64) *limiter { return &limiter{rate: rps, tokens: 1, last: time.Now()} } + +func (l *limiter) Wait(ctx context.Context) error { + for { + l.mu.Lock() + now := time.Now() + l.tokens += now.Sub(l.last).Seconds() * l.rate + if burst := l.rate / 4; l.tokens > burst+1 { + l.tokens = burst + 1 + } + l.last = now + if l.tokens >= 1 { + l.tokens-- + l.mu.Unlock() + return nil + } + wait := time.Duration((1 - l.tokens) / l.rate * float64(time.Second)) + l.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + } + } +} diff --git a/internal/destinations/stripe/fake_test.go b/internal/destinations/stripe/fake_test.go new file mode 100644 index 0000000..e43092d --- /dev/null +++ b/internal/destinations/stripe/fake_test.go @@ -0,0 +1,378 @@ +package stripe + +import ( + "encoding/json" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// fakeStripe models the slice of Stripe's v1 API the adapter uses, with the +// constraints that matter: one meter per event_name, the 35-day timestamp +// window, 15 significant digits, identifier dedupe, eventually consistent +// customer search, idempotency-key replay, and minute-aligned summaries. +type fakeStripe struct { + t *testing.T + srv *httptest.Server + now func() time.Time + + mu sync.Mutex + meters []*Meter + customers map[string]*Customer + searchable map[string]bool // customers visible to search (eventual consistency) + events []fakeEvent + idents map[string]bool + prices []Price + subs map[string]*Subscription + idem map[string]string // idempotency key -> response body + failNext map[string]int // "METHOD path" -> number of 500s to return + searchAll bool // search ignores the query (like stripe-mock fixtures) + noSearch bool // search is not offered in the account's region + requests []string + seq int +} + +type fakeEvent struct { + EventName, Customer, Value, Identifier string + Timestamp int64 +} + +func newFakeStripe(t *testing.T, now func() time.Time) *fakeStripe { + f := &fakeStripe{ + t: t, now: now, + customers: map[string]*Customer{}, + searchable: map[string]bool{}, + idents: map[string]bool{}, + subs: map[string]*Subscription{}, + idem: map[string]string{}, + failNext: map[string]int{}, + } + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeStripe) id(prefix string) string { + f.seq++ + return fmt.Sprintf("%s_%04d", prefix, f.seq) +} + +func (f *fakeStripe) fail(w http.ResponseWriter, status int, code, msg string) { + w.WriteHeader(status) + json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"type": "invalid_request_error", "code": code, "message": msg}}) +} + +var ( + reMeterAction = regexp.MustCompile(`^/v1/billing/meters/([^/]+)/(reactivate|event_summaries)$`) + reCustomer = regexp.MustCompile(`^/v1/customers/(cus_[^/]+)$`) + reSearch = regexp.MustCompile(`^metadata\['([^']+)'\]:'(.*)'$`) +) + +func (f *fakeStripe) handle(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if r.Header.Get("Authorization") != "Bearer sk_test_123" { + f.fail(w, 401, "", "Invalid API Key provided") + return + } + if r.Header.Get("Stripe-Version") == "" { + f.t.Errorf("%s %s without Stripe-Version", r.Method, r.URL.Path) + } + r.ParseForm() + key := r.Method + " " + r.URL.Path + f.requests = append(f.requests, key) + if n := f.failNext[key]; n > 0 { + f.failNext[key] = n - 1 + f.fail(w, 500, "", "simulated outage") + return + } + if ik := r.Header.Get("Idempotency-Key"); ik != "" && r.Method == http.MethodPost { + if body, ok := f.idem[ik]; ok { + w.Header().Set("Idempotent-Replayed", "true") + w.Write([]byte(body)) + return + } + rec := httptest.NewRecorder() + f.route(rec, r) + if rec.Code < 300 { + f.idem[ik] = rec.Body.String() + } + w.WriteHeader(rec.Code) + w.Write(rec.Body.Bytes()) + return + } + f.route(w, r) +} + +func writeJSON(w http.ResponseWriter, v any) { json.NewEncoder(w).Encode(v) } + +func page[T any](r *http.Request, items []T, id func(T) string) map[string]any { + limit, _ := strconv.Atoi(r.Form.Get("limit")) + if limit <= 0 { + limit = 10 + } + start := 0 + if after := r.Form.Get("starting_after"); after != "" { + for i, it := range items { + if id(it) == after { + start = i + 1 + } + } + } + end := start + limit + if end > len(items) { + end = len(items) + } + return map[string]any{"object": "list", "data": items[start:end], "has_more": end < len(items)} +} + +func (f *fakeStripe) route(w http.ResponseWriter, r *http.Request) { + p := r.URL.Path + switch { + case r.Method == "GET" && p == "/v1/billing/meters": + var ms []Meter + for _, m := range f.meters { + ms = append(ms, *m) + } + writeJSON(w, page(r, ms, func(m Meter) string { return m.ID })) + + case r.Method == "POST" && p == "/v1/billing/meters": + name := r.Form.Get("event_name") + if name == "" || r.Form.Get("display_name") == "" || r.Form.Get("customer_mapping[type]") != "by_id" { + f.fail(w, 400, "parameter_missing", "missing meter params") + return + } + if agg := r.Form.Get("default_aggregation[formula]"); agg != "sum" && agg != "count" && agg != "last" { + f.fail(w, 400, "parameter_invalid", "bad aggregation") + return + } + if len(name) > 100 { + f.fail(w, 400, "parameter_invalid", "event_name too long") + return + } + for _, m := range f.meters { + if m.EventName == name { + f.fail(w, 400, "", "An active meter with event_name already exists") + return + } + } + m := &Meter{ID: f.id("mtr"), EventName: name, DisplayName: r.Form.Get("display_name"), Status: "active"} + m.CustomerMapping.EventPayloadKey = r.Form.Get("customer_mapping[event_payload_key]") + m.ValueSettings.EventPayloadKey = r.Form.Get("value_settings[event_payload_key]") + f.meters = append(f.meters, m) + writeJSON(w, m) + + case reMeterAction.MatchString(p): + parts := reMeterAction.FindStringSubmatch(p) + var meter *Meter + for _, m := range f.meters { + if m.ID == parts[1] { + meter = m + } + } + if meter == nil { + f.fail(w, 404, "resource_missing", "no such meter") + return + } + if parts[2] == "reactivate" { + meter.Status = "active" + writeJSON(w, meter) + return + } + start, _ := strconv.ParseInt(r.Form.Get("start_time"), 10, 64) + end, _ := strconv.ParseInt(r.Form.Get("end_time"), 10, 64) + if start%60 != 0 || end%60 != 0 { + f.fail(w, 400, "parameter_invalid", "start_time and end_time must be aligned with minute boundaries") + return + } + sum := new(big.Float) + for _, e := range f.events { + if e.EventName == meter.EventName && e.Customer == r.Form.Get("customer") && e.Timestamp >= start && e.Timestamp < end { + v, _ := new(big.Float).SetString(e.Value) + sum.Add(sum, v) + } + } + total, _ := sum.Float64() + writeJSON(w, map[string]any{"object": "list", "has_more": false, "data": []map[string]any{{"id": "mtrusg_1", "aggregated_value": total, "start_time": start, "end_time": end}}}) + + case r.Method == "POST" && p == "/v1/billing/meter_events": + name := r.Form.Get("event_name") + var meter *Meter + for _, m := range f.meters { + if m.EventName == name && m.Status == "active" { + meter = m + } + } + if meter == nil { + f.fail(w, 400, "no_meter", "no active meter for "+name) + return + } + cus := r.Form.Get("payload[" + meter.customerKey() + "]") + val := r.Form.Get("payload[" + meter.valueKey() + "]") + if cus == "" || val == "" { + f.fail(w, 400, "parameter_missing", "payload keys missing") + return + } + if _, ok := new(big.Float).SetString(val); !ok || strings.ContainsAny(val, "eE") || significantDigits(val) > 15 { + f.fail(w, 400, "meter_event_invalid_value", "invalid value "+val) + return + } + ts, _ := strconv.ParseInt(r.Form.Get("timestamp"), 10, 64) + now := f.now().Unix() + if ts < now-35*86400 || ts > now+300 { + f.fail(w, 400, "timestamp_too_far_in_past", "timestamp out of range") + return + } + ident := r.Form.Get("identifier") + if f.idents[ident] { + f.fail(w, 400, "duplicate_meter_event", "An event already exists with identifier "+ident) + return + } + f.idents[ident] = true + f.events = append(f.events, fakeEvent{EventName: name, Customer: cus, Value: val, Identifier: ident, Timestamp: ts}) + writeJSON(w, map[string]any{"object": "billing.meter_event", "identifier": ident}) + + case r.Method == "GET" && p == "/v1/customers/search": + if f.noSearch { + f.fail(w, 400, "", "The search feature is temporarily unavailable in your region.") + return + } + m := reSearch.FindStringSubmatch(r.Form.Get("query")) + if m == nil { + f.fail(w, 400, "parameter_invalid", "bad query") + return + } + var out []Customer + for id, c := range f.customers { + if f.searchable[id] && (f.searchAll || c.Metadata[m[1]] == strings.ReplaceAll(m[2], `\'`, "'")) { + out = append(out, *c) + } + } + writeJSON(w, map[string]any{"object": "search_result", "data": out, "has_more": false}) + + case r.Method == "GET" && p == "/v1/customers": + var out []Customer + for _, c := range f.customers { + if email := r.Form.Get("email"); email == "" || c.Email == email { + out = append(out, *c) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + writeJSON(w, page(r, out, func(c Customer) string { return c.ID })) + + case r.Method == "POST" && p == "/v1/customers": + c := &Customer{ID: f.id("cus"), Name: r.Form.Get("name"), Email: r.Form.Get("email"), Metadata: metadataOf(r)} + f.customers[c.ID] = c + writeJSON(w, c) + + case reCustomer.MatchString(p): + id := reCustomer.FindStringSubmatch(p)[1] + c, ok := f.customers[id] + if !ok { + f.fail(w, 404, "resource_missing", "No such customer: "+id) + return + } + if r.Method == "POST" { + c.Name = r.Form.Get("name") + for k, v := range metadataOf(r) { + c.Metadata[k] = v + } + } + writeJSON(w, c) + + case r.Method == "GET" && p == "/v1/prices": + writeJSON(w, page(r, f.prices, func(p Price) string { return p.ID })) + + case r.Method == "GET" && p == "/v1/subscriptions": + var out []Subscription + for _, s := range f.subs { + if s.Customer == r.Form.Get("customer") { + out = append(out, *s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + writeJSON(w, page(r, out, func(s Subscription) string { return s.ID })) + + case r.Method == "POST" && p == "/v1/subscriptions": + s := &Subscription{ID: f.id("sub"), Status: "active", Customer: r.Form.Get("customer"), Metadata: metadataOf(r)} + for i := 0; ; i++ { + pid := r.Form.Get(fmt.Sprintf("items[%d][price]", i)) + if pid == "" { + break + } + if r.Form.Get(fmt.Sprintf("items[%d][quantity]", i)) != "" { + f.fail(w, 400, "parameter_invalid", "quantity not allowed for metered prices") + return + } + f.addItem(s, pid) + } + f.subs[s.ID] = s + writeJSON(w, s) + + case r.Method == "POST" && p == "/v1/subscription_items": + s, ok := f.subs[r.Form.Get("subscription")] + if !ok { + f.fail(w, 400, "resource_missing", "no such subscription") + return + } + f.addItem(s, r.Form.Get("price")) + writeJSON(w, map[string]any{"object": "subscription_item"}) + + default: + f.fail(w, 404, "", "unhandled "+r.Method+" "+p) + } +} + +func (f *fakeStripe) addItem(s *Subscription, priceID string) { + for _, p := range f.prices { + if p.ID == priceID { + s.Items.Data = append(s.Items.Data, struct { + ID string `json:"id"` + Price Price `json:"price"` + }{ID: f.id("si"), Price: p}) + } + } +} + +func metadataOf(r *http.Request) map[string]string { + out := map[string]string{} + for k, v := range r.Form { + if strings.HasPrefix(k, "metadata[") && strings.HasSuffix(k, "]") { + out[strings.TrimSuffix(strings.TrimPrefix(k, "metadata["), "]")] = v[0] + } + } + return out +} + +func significantDigits(s string) int { + s = strings.TrimLeft(strings.Replace(strings.TrimPrefix(s, "-"), ".", "", 1), "0") + return len(s) +} + +func (f *fakeStripe) makeSearchable() { + f.mu.Lock() + defer f.mu.Unlock() + for id := range f.customers { + f.searchable[id] = true + } +} + +func (f *fakeStripe) count(prefix string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, r := range f.requests { + if strings.HasPrefix(r, prefix) { + n++ + } + } + return n +} diff --git a/internal/destinations/stripe/webhook.go b/internal/destinations/stripe/webhook.go new file mode 100644 index 0000000..0d0c1df --- /dev/null +++ b/internal/destinations/stripe/webhook.go @@ -0,0 +1,71 @@ +package stripe + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// DefaultTolerance matches Stripe's libraries. +const DefaultTolerance = 5 * time.Minute + +// VerifySignature checks a Stripe-Signature header ("t=...,v1=...") against +// the raw request body. Any v1 signature may match (Stripe sends several +// while a secret is being rolled); other schemes are ignored. +func VerifySignature(payload []byte, header, secret string, tolerance time.Duration, now time.Time) error { + if secret == "" { + return errors.New("stripe webhook secret not configured") + } + var ts int64 + var sigs []string + for _, part := range strings.Split(header, ",") { + k, v, ok := strings.Cut(strings.TrimSpace(part), "=") + if !ok { + continue + } + switch k { + case "t": + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("invalid timestamp in Stripe-Signature: %w", err) + } + ts = n + case "v1": + sigs = append(sigs, v) + } + } + if ts == 0 || len(sigs) == 0 { + return errors.New("Stripe-Signature missing t or v1") + } + if tolerance > 0 { + age := now.Sub(time.Unix(ts, 0)) + if age > tolerance || age < -tolerance { + return fmt.Errorf("Stripe-Signature timestamp outside tolerance (%s)", age.Round(time.Second)) + } + } + expected := sign(payload, secret, ts) + for _, s := range sigs { + if hmac.Equal([]byte(s), []byte(expected)) { + return nil + } + } + return errors.New("no matching Stripe signature") +} + +// SignatureHeader builds a Stripe-Signature header (tests and local tooling). +func SignatureHeader(payload []byte, secret string, t time.Time) string { + return fmt.Sprintf("t=%d,v1=%s", t.Unix(), sign(payload, secret, t.Unix())) +} + +func sign(payload []byte, secret string, ts int64) string { + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(strconv.FormatInt(ts, 10))) + mac.Write([]byte(".")) + mac.Write(payload) + return hex.EncodeToString(mac.Sum(nil)) +} diff --git a/internal/destinations/webhook/adapter.go b/internal/destinations/webhook/adapter.go new file mode 100644 index 0000000..68f12d5 --- /dev/null +++ b/internal/destinations/webhook/adapter.go @@ -0,0 +1,161 @@ +// Package webhook streams usage to any HTTP endpoint as CloudEvents 1.0 +// batches, signed with HMAC-SHA256. Use it to feed a data platform, a +// Kafka/HTTP bridge, or a homegrown rating engine alongside a billing backend. +package webhook + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// CloudEvent types emitted. +const ( + TypeUsage = "com.vcluster.vbilling.usage.v1" + TypeTenant = "com.vcluster.vbilling.tenant.v1" + TypeOffboarded = "com.vcluster.vbilling.tenant.offboarded.v1" + TypeCatalog = "com.vcluster.vbilling.catalog.v1" + + SignatureHeader = "X-VBilling-Signature" + DataSchema = "https://github.com/vClusterLabs-Experiments/vbilling/blob/main/docs/schema/usage-event.v1.json" + maxBatch = 500 +) + +func init() { + destinations.Register("webhook", func(cfg *config.Config) (destinations.Destination, error) { + if cfg.WebhookURL == "" { + return nil, errors.New("WEBHOOK_URL is required when the webhook adapter is enabled") + } + return New(cfg), nil + }) +} + +// CloudEvent is the structured-mode envelope. +type CloudEvent struct { + SpecVersion string `json:"specversion"` + ID string `json:"id"` + Source string `json:"source"` + Type string `json:"type"` + Subject string `json:"subject,omitempty"` + Time string `json:"time"` + DataContentType string `json:"datacontenttype"` + DataSchema string `json:"dataschema,omitempty"` + Data any `json:"data"` +} + +type Adapter struct { + url string + secret string + headers map[string]string + source string + http *http.Client + now func() time.Time +} + +func New(cfg *config.Config) *Adapter { + return &Adapter{ + url: cfg.WebhookURL, + secret: cfg.WebhookSecret, + headers: cfg.WebhookHeaders, + source: "vbilling/" + cfg.Region + "/" + cfg.ClusterName, + http: destinations.NewHTTPClient(), + now: time.Now, + } +} + +func (a *Adapter) Name() string { return "webhook" } + +// Bootstrap publishes the metric catalog so consumers can learn units and +// group keys without reading vBilling's docs. +func (a *Adapter) Bootstrap(ctx context.Context, metrics []usage.MetricDef) error { + return a.post(ctx, []CloudEvent{a.envelope(TypeCatalog, "catalog-"+strconv.FormatInt(a.now().Unix(), 10), "", a.now(), metrics)}) +} + +func (a *Adapter) EnsureTenant(ctx context.Context, t usage.Tenant) error { + return a.post(ctx, []CloudEvent{a.envelope(TypeTenant, "tenant-"+t.ID+"-"+strconv.FormatInt(a.now().Unix(), 10), t.ID, a.now(), t)}) +} + +func (a *Adapter) RemoveTenant(ctx context.Context, t usage.Tenant) error { + return a.post(ctx, []CloudEvent{a.envelope(TypeOffboarded, "offboard-"+t.ID+"-"+strconv.FormatInt(a.now().Unix(), 10), t.ID, a.now(), t)}) +} + +func (a *Adapter) SendEvents(ctx context.Context, events []usage.Event) error { + for i := 0; i < len(events); i += maxBatch { + end := i + maxBatch + if end > len(events) { + end = len(events) + } + batch := make([]CloudEvent, 0, end-i) + for j := i; j < end; j++ { + e := events[j] + ce := a.envelope(TypeUsage, e.ID, e.Tenant, e.WindowEnd, e) + ce.DataSchema = DataSchema + batch = append(batch, ce) + } + if err := a.post(ctx, batch); err != nil { + return err + } + } + return nil +} + +func (a *Adapter) envelope(typ, id, subject string, at time.Time, data any) CloudEvent { + return CloudEvent{ + SpecVersion: "1.0", ID: id, Source: a.source, Type: typ, Subject: subject, + Time: at.UTC().Format(time.RFC3339Nano), DataContentType: "application/json", Data: data, + } +} + +func (a *Adapter) post(ctx context.Context, batch []CloudEvent) error { + body, err := json.Marshal(batch) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/cloudevents-batch+json") + req.Header.Set("User-Agent", "vbilling/0.2") + for k, v := range a.headers { + req.Header.Set(k, v) + } + if a.secret != "" { + req.Header.Set(SignatureHeader, Sign(body, a.secret, a.now())) + } + resp, err := a.http.Do(req) + if err != nil { + return fmt.Errorf("webhook POST: %w", err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + return destinations.StatusError(http.MethodPost, a.url, resp.StatusCode, b) +} + +// Sign returns "t=,v1=", the +// same scheme as Stripe-Signature, so existing verification code works. +func Sign(body []byte, secret string, at time.Time) string { + ts := strconv.FormatInt(at.Unix(), 10) + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(ts + ".")) + mac.Write(body) + return "t=" + ts + ",v1=" + hex.EncodeToString(mac.Sum(nil)) +} + +var _ destinations.Destination = (*Adapter)(nil) diff --git a/internal/destinations/webhook/adapter_test.go b/internal/destinations/webhook/adapter_test.go new file mode 100644 index 0000000..35102fb --- /dev/null +++ b/internal/destinations/webhook/adapter_test.go @@ -0,0 +1,81 @@ +package webhook + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/config" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +func TestSendsSignedCloudEventsBatches(t *testing.T) { + now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + var got [][]CloudEvent + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if ct := r.Header.Get("Content-Type"); ct != "application/cloudevents-batch+json" { + t.Errorf("content type %q", ct) + } + if r.Header.Get("X-Tenant-Platform") != "dataplane" { + t.Error("custom header missing") + } + // The signature scheme is Stripe-compatible, so Stripe's verifier works. + if err := stripe.VerifySignature(body, r.Header.Get(SignatureHeader), "s3cret", 5*time.Minute, now); err != nil { + t.Errorf("signature: %v", err) + } + var batch []CloudEvent + json.Unmarshal(body, &batch) + got = append(got, batch) + })) + defer srv.Close() + + a := New(&config.Config{WebhookURL: srv.URL, WebhookSecret: "s3cret", Region: "ap-southeast-2", ClusterName: "syd-cp-1", + WebhookHeaders: map[string]string{"X-Tenant-Platform": "dataplane"}}) + a.now = func() time.Time { return now } + + var events []usage.Event + for i := 0; i < 1200; i++ { + ws := now.Add(-time.Duration(i+1) * time.Minute) + e := usage.Event{Tenant: "acme", Metric: usage.MetricGPUUtilization, Quantity: 1, WindowStart: ws, WindowEnd: ws.Add(time.Minute)} + e.Finalize() + events = append(events, e) + } + if err := a.SendEvents(context.Background(), events); err != nil { + t.Fatal(err) + } + if len(got) != 3 || len(got[0]) != 500 || len(got[2]) != 200 { + t.Fatalf("batches: %d", len(got)) + } + ce := got[0][0] + if ce.SpecVersion != "1.0" || ce.Type != TypeUsage || ce.ID != events[0].ID || ce.Subject != "acme" || + ce.Source != "vbilling/ap-southeast-2/syd-cp-1" || ce.DataSchema != DataSchema { + t.Fatalf("envelope: %+v", ce) + } + // Webhooks are a data feed: informational metrics are included. + if destinations.Accepts(a, usage.MetricGPUUtilization) != true { + t.Fatal("webhook should accept every metric") + } +} + +func TestErrorClassification(t *testing.T) { + status := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) })) + defer srv.Close() + a := New(&config.Config{WebhookURL: srv.URL}) + ev := usage.Event{Tenant: "a", Metric: usage.MetricGPUHours, Quantity: 1, WindowStart: time.Now().Add(-time.Minute), WindowEnd: time.Now()} + ev.Finalize() + for code, permanent := range map[int]bool{400: true, 422: true, 401: false, 404: false, 429: false, 503: false} { + status = code + err := a.SendEvents(context.Background(), []usage.Event{ev}) + if err == nil || destinations.IsPermanent(err) != permanent { + t.Errorf("HTTP %d: err=%v permanent=%v, want permanent=%v", code, err, destinations.IsPermanent(err), permanent) + } + } +} diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 06536bb..2252dd1 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -1,11 +1,16 @@ +// Package discovery finds tenant clusters running on this control plane +// cluster and the billing metadata attached to them. package discovery import ( "context" "fmt" "log" + "sort" + "strings" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" @@ -13,215 +18,273 @@ import ( "k8s.io/client-go/kubernetes" ) -// VCluster represents a discovered vCluster instance on the host cluster. -type VCluster struct { +// AnnotationPrefix namespaces vBilling's labels and annotations. Set them on +// the tenant cluster's StatefulSet/Deployment, its namespace, or its +// VirtualClusterInstance (later sources win): +// +// vbilling.vcluster.com/tenant billing customer ID (several clusters can share one) +// vbilling.vcluster.com/display-name customer name on invoices +// vbilling.vcluster.com/email billing email +// vbilling.vcluster.com/project project / cost attribution +// vbilling.vcluster.com/tenant-class public, enterprise, government, dev, ... +// vbilling.vcluster.com/plan plan code / rate card alias +// vbilling.vcluster.com/currency invoice currency +// vbilling.vcluster.com/stripe-customer-id pin an existing Stripe customer +// vbilling.vcluster.com/metronome-customer-id pin an existing Metronome customer +// vbilling.vcluster.com/exclude "true" to skip metering +const AnnotationPrefix = "vbilling.vcluster.com/" + +// TenantCluster is a discovered tenant cluster. +type TenantCluster struct { Name string Namespace string UID string CreatedAt time.Time - // Labels from the StatefulSet for additional metadata - Labels map[string]string + // Ready is true while the control plane has a ready replica. Sleeping or + // still-provisioning clusters are not charged instance hours. + Ready bool + Source string // statefulset | deployment | external + Project string + // Instance is the vCluster Platform instance name (the + // vcluster_platform_instance label of fleet observability metrics). + Instance string + // External tenant clusters run elsewhere (vCluster Standalone, other + // clusters) and are reached only through their own API. + External bool + DisplayName string + Labels map[string]string + Annotations map[string]string } -// ExternalID returns a stable identifier for the Tenant Cluster, used as the -// canonical tenant ID across billing adapters. -func (v *VCluster) ExternalID() string { - return fmt.Sprintf("vcluster-%s-%s", v.Namespace, v.Name) +// ExternalID is the stable tenant-cluster identifier (v0.1 compatible). +func (c *TenantCluster) ExternalID() string { + if c.External { + return "external-" + c.Name + } + return fmt.Sprintf("vcluster-%s-%s", c.Namespace, c.Name) +} + +// Meta returns a vBilling setting from annotations, then labels. +func (c *TenantCluster) Meta(key string) string { + if v := c.Annotations[AnnotationPrefix+key]; v != "" { + return v + } + return c.Labels[AnnotationPrefix+key] } -// DisplayName returns a human-readable name for the billing customer. -func (v *VCluster) DisplayName() string { - return fmt.Sprintf("vCluster: %s (%s)", v.Name, v.Namespace) +// TenantSource selects how tenant clusters map to billing customers. +type TenantSource string + +const ( + TenantPerCluster TenantSource = "cluster" // default: one customer per tenant cluster + TenantPerProject TenantSource = "project" // one customer per vCluster Platform project +) + +// TenantID resolves the billing customer. An explicit tenant label always wins. +func (c *TenantCluster) TenantID(src TenantSource) string { + if v := c.Meta("tenant"); v != "" { + return v + } + if src == TenantPerProject && c.Project != "" { + return "project-" + c.Project + } + return c.ExternalID() } -// Discoverer finds vCluster instances in a Kubernetes cluster. +// Discoverer finds tenant clusters. type Discoverer struct { - client kubernetes.Interface - dynamicClient dynamic.Interface // optional, for Platform API discovery - namespaces []string // empty = all namespaces + client kubernetes.Interface + dynamicClient dynamic.Interface // optional: vCluster Platform enrichment + namespaces []string + platformCluster string + external []TenantCluster } -func NewDiscoverer(client kubernetes.Interface, dynamicClient dynamic.Interface, namespaces []string) *Discoverer { - return &Discoverer{ - client: client, - dynamicClient: dynamicClient, - namespaces: namespaces, - } +// WithExternal adds tenant clusters configured outside this control plane +// cluster (see TENANT_CLUSTERS_FILE). +func (d *Discoverer) WithExternal(clusters []TenantCluster) *Discoverer { + d.external = clusters + return d +} + +func NewDiscoverer(client kubernetes.Interface, dynamicClient dynamic.Interface, namespaces []string, platformCluster string) *Discoverer { + return &Discoverer{client: client, dynamicClient: dynamicClient, namespaces: namespaces, platformCluster: platformCluster} } -// virtualClusterInstanceGVR is the GroupVersionResource for the vCluster Platform -// management API's VirtualClusterInstance custom resource. var virtualClusterInstanceGVR = schema.GroupVersionResource{ - Group: "management.loft.sh", - Version: "v1", - Resource: "virtualclusterinstances", + Group: "management.loft.sh", Version: "v1", Resource: "virtualclusterinstances", } -// Discover lists all vCluster instances across the cluster. -// It tries Platform API discovery first (if a dynamic client is available), -// then falls back to StatefulSet/Deployment scanning. -func (d *Discoverer) Discover(ctx context.Context) ([]VCluster, error) { - // Try Platform API first if dynamic client is available +// Discover returns the tenant clusters running here. Workloads are the +// source of truth for what runs; vCluster Platform instances, when the API +// is reachable, enrich them with project and display name. An error means +// "unknown", never "no clusters": callers must not offboard on it. +func (d *Discoverer) Discover(ctx context.Context) ([]TenantCluster, error) { + clusters, err := d.fromWorkloads(ctx) + if err != nil { + return nil, err + } + nsMeta, err := d.namespaceMeta(ctx, clusters) + if err != nil { + return nil, err + } + for i := range clusters { + c := &clusters[i] + if ns, ok := nsMeta[c.Namespace]; ok { + c.Labels = merge(ns.Labels, c.Labels) + c.Annotations = merge(ns.Annotations, c.Annotations) + } + } if d.dynamicClient != nil { - vclusters, err := d.DiscoverFromPlatform(ctx) - if err == nil && len(vclusters) > 0 { - log.Printf("[discovery] found %d vCluster(s) via Platform API", len(vclusters)) - return vclusters, nil + d.enrichFromPlatform(ctx, clusters) + } + clusters = append(clusters, d.external...) + out := clusters[:0] + for _, c := range clusters { + if c.Instance == "" { + c.Instance = c.Name } - if err != nil { - log.Printf("[discovery] Platform API not available, falling back to StatefulSet scanning: %v", err) - } else { - log.Printf("[discovery] Platform API returned 0 results, falling back to StatefulSet scanning") + if v := c.Meta("project"); v != "" { + c.Project = v // an explicit annotation beats the Platform project } + if v := c.Meta("display-name"); v != "" { + c.DisplayName = v + } + if c.Meta("exclude") == "true" { + continue + } + out = append(out, c) } - - // Fall back to StatefulSet/Deployment scanning - return d.discoverFromWorkloads(ctx) + sort.Slice(out, func(i, j int) bool { return out[i].ExternalID() < out[j].ExternalID() }) + return out, nil } -// DiscoverFromPlatform uses the Kubernetes dynamic client to list -// VirtualClusterInstance resources from the management.loft.sh API group. -// This is the preferred discovery method when the vCluster Platform is installed. -// It falls back gracefully if the CRD does not exist. -func (d *Discoverer) DiscoverFromPlatform(ctx context.Context) ([]VCluster, error) { - if d.dynamicClient == nil { - return nil, fmt.Errorf("dynamic client not configured") +func (d *Discoverer) scope() []string { + if len(d.namespaces) == 0 { + return []string{metav1.NamespaceAll} } + return d.namespaces +} - var vclusters []VCluster - - namespaces := d.namespaces - if len(namespaces) == 0 { - namespaces = []string{""} // empty string = all namespaces - } - - for _, ns := range namespaces { - var items []map[string]interface{} - - if ns == "" { - // Cluster-wide list - result, err := d.dynamicClient.Resource(virtualClusterInstanceGVR).List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, fmt.Errorf("list virtualclusterinstances (all namespaces): %w", err) - } - for _, item := range result.Items { - items = append(items, item.Object) - } - } else { - result, err := d.dynamicClient.Resource(virtualClusterInstanceGVR).Namespace(ns).List(ctx, metav1.ListOptions{}) - if err != nil { - log.Printf("[discovery] error listing virtualclusterinstances in namespace %q: %v", ns, err) +// fromWorkloads finds tenant clusters by their app=vcluster StatefulSets and +// Deployments. A failed list is an error: a partial view must never look +// like deleted clusters. +func (d *Discoverer) fromWorkloads(ctx context.Context) ([]TenantCluster, error) { + selector := labels.Set{"app": "vcluster"}.String() + seen := map[string]bool{} + var out []TenantCluster + for _, ns := range d.scope() { + sts, err := d.client.AppsV1().StatefulSets(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, fmt.Errorf("list statefulsets in %q: %w", ns, err) + } + for _, s := range sts.Items { + key := s.Namespace + "/" + s.Name + if seen[key] { continue } - for _, item := range result.Items { - items = append(items, item.Object) - } + seen[key] = true + out = append(out, TenantCluster{ + Name: s.Name, Namespace: s.Namespace, UID: string(s.UID), CreatedAt: s.CreationTimestamp.Time, + Ready: s.Status.ReadyReplicas > 0, Source: "statefulset", + Labels: copyMap(s.Labels), Annotations: copyMap(s.Annotations), + }) } - - for _, obj := range items { - name := nestedString(obj, "metadata", "name") - namespace := nestedString(obj, "metadata", "namespace") - uid := nestedString(obj, "metadata", "uid") - creationStr := nestedString(obj, "metadata", "creationTimestamp") - - createdAt := time.Time{} - if creationStr != "" { - if t, parseErr := time.Parse(time.RFC3339, creationStr); parseErr == nil { - createdAt = t - } - } - - // Extract labels from the resource - lbls := make(map[string]string) - if metadata, ok := obj["metadata"].(map[string]interface{}); ok { - if labelsRaw, ok := metadata["labels"].(map[string]interface{}); ok { - for k, v := range labelsRaw { - if s, ok := v.(string); ok { - lbls[k] = s - } - } - } + deps, err := d.client.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, fmt.Errorf("list deployments in %q: %w", ns, err) + } + for _, dep := range deps.Items { + key := dep.Namespace + "/" + dep.Name + if seen[key] { + continue } - - vclusters = append(vclusters, VCluster{ - Name: name, - Namespace: namespace, - UID: uid, - CreatedAt: createdAt, - Labels: lbls, + seen[key] = true + out = append(out, TenantCluster{ + Name: dep.Name, Namespace: dep.Namespace, UID: string(dep.UID), CreatedAt: dep.CreationTimestamp.Time, + Ready: dep.Status.ReadyReplicas > 0, Source: "deployment", + Labels: copyMap(dep.Labels), Annotations: copyMap(dep.Annotations), }) } } - - return vclusters, nil + return out, nil } -// discoverFromWorkloads finds vClusters by scanning for StatefulSets and -// Deployments with the app=vcluster label. This is the fallback discovery -// method when the Platform API is not available. -func (d *Discoverer) discoverFromWorkloads(ctx context.Context) ([]VCluster, error) { - selector := labels.Set{"app": "vcluster"}.String() - var vclusters []VCluster - - namespaces := d.namespaces - if len(namespaces) == 0 { - namespaces = []string{""} // empty string = all namespaces - } - - for _, ns := range namespaces { - // Check StatefulSets (default vCluster deployment) - stsList, err := d.client.AppsV1().StatefulSets(ns).List(ctx, metav1.ListOptions{ - LabelSelector: selector, - }) - if err != nil { - log.Printf("[discovery] error listing StatefulSets in namespace %q: %v", ns, err) +func (d *Discoverer) namespaceMeta(ctx context.Context, clusters []TenantCluster) (map[string]*corev1.Namespace, error) { + out := map[string]*corev1.Namespace{} + for _, c := range clusters { + if _, ok := out[c.Namespace]; ok { continue } - for _, sts := range stsList.Items { - vclusters = append(vclusters, VCluster{ - Name: sts.Name, - Namespace: sts.Namespace, - UID: string(sts.UID), - CreatedAt: sts.CreationTimestamp.Time, - Labels: sts.Labels, - }) + ns, err := d.client.CoreV1().Namespaces().Get(ctx, c.Namespace, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get namespace %s: %w", c.Namespace, err) } + out[c.Namespace] = ns + } + return out, nil +} - // Also check Deployments (some vCluster configurations use Deployments) - depList, err := d.client.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{ - LabelSelector: selector, - }) - if err != nil { - log.Printf("[discovery] error listing Deployments in namespace %q: %v", ns, err) +// enrichFromPlatform maps VirtualClusterInstances onto running workloads via +// spec.clusterRef.{namespace,virtualCluster}. The instance's own namespace +// is the project namespace (p-). Errors are non-fatal: the +// Platform API is optional and only adds metadata. +func (d *Discoverer) enrichFromPlatform(ctx context.Context, clusters []TenantCluster) { + list, err := d.dynamicClient.Resource(virtualClusterInstanceGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + return // Platform not installed or not reachable from here + } + byKey := map[string]int{} + for i, c := range clusters { + byKey[c.Namespace+"/"+c.Name] = i + } + enriched := 0 + for _, item := range list.Items { + obj := item.Object + cluster := nestedString(obj, "spec", "clusterRef", "cluster") + if d.platformCluster != "" && cluster != "" && cluster != d.platformCluster { + continue // runs on another control plane cluster + } + ns := nestedString(obj, "spec", "clusterRef", "namespace") + name := nestedString(obj, "spec", "clusterRef", "virtualCluster") + if name == "" { + name = item.GetName() + } + i, ok := byKey[ns+"/"+name] + if !ok { continue } - // Deduplicate: only add if not already found as StatefulSet - seen := make(map[string]bool) - for _, vc := range vclusters { - seen[vc.Namespace+"/"+vc.Name] = true + c := &clusters[i] + c.Instance = item.GetName() + if p := strings.TrimPrefix(item.GetNamespace(), "p-"); p != item.GetNamespace() { + c.Project = p } - for _, dep := range depList.Items { - key := dep.Namespace + "/" + dep.Name - if seen[key] { - continue - } - vclusters = append(vclusters, VCluster{ - Name: dep.Name, - Namespace: dep.Namespace, - UID: string(dep.UID), - CreatedAt: dep.CreationTimestamp.Time, - Labels: dep.Labels, - }) + if dn := nestedString(obj, "spec", "displayName"); dn != "" { + c.DisplayName = dn } + c.Labels = merge(c.Labels, item.GetLabels()) + c.Annotations = merge(c.Annotations, item.GetAnnotations()) + enriched++ } + if enriched > 0 { + log.Printf("[discovery] enriched %d tenant cluster(s) from vCluster Platform", enriched) + } +} - log.Printf("[discovery] found %d vCluster(s)", len(vclusters)) - return vclusters, nil +func merge(base, over map[string]string) map[string]string { + out := make(map[string]string, len(base)+len(over)) + for k, v := range base { + out[k] = v + } + for k, v := range over { + out[k] = v + } + return out } -// nestedString extracts a string value from a nested map using the given path -// of keys. Returns empty string if the path does not exist or the value is not -// a string. +func copyMap(m map[string]string) map[string]string { return merge(nil, m) } + +// nestedString extracts a string from a nested map, or "". func nestedString(obj map[string]interface{}, fields ...string) string { var current interface{} = obj for _, field := range fields { @@ -229,8 +292,7 @@ func nestedString(obj map[string]interface{}, fields ...string) string { if !ok { return "" } - current, ok = m[field] - if !ok { + if current, ok = m[field]; !ok { return "" } } diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go new file mode 100644 index 0000000..0390e13 --- /dev/null +++ b/internal/discovery/discovery_test.go @@ -0,0 +1,96 @@ +package discovery + +import ( + "context" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func sts(ns, name string, ready int32, ann map[string]string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, UID: types.UID("uid-" + name), Labels: map[string]string{"app": "vcluster"}, Annotations: ann}, + Status: appsv1.StatefulSetStatus{ReadyReplicas: ready}, + } +} + +func nsObj(name string, labels map[string]string) *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}} +} + +func TestDiscoverMergesMetadataAndReadiness(t *testing.T) { + kube := fake.NewSimpleClientset( + nsObj("team-a", map[string]string{AnnotationPrefix + "tenant": "acme", AnnotationPrefix + "tenant-class": "public"}), + nsObj("team-b", nil), + nsObj("excluded", nil), + sts("team-a", "train", 1, map[string]string{AnnotationPrefix + "display-name": "Acme AI"}), + sts("team-b", "sleepy", 0, nil), + sts("excluded", "internal", 1, map[string]string{AnnotationPrefix + "exclude": "true"}), + &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "postgres", Namespace: "team-a", Labels: map[string]string{"app": "postgres"}}}, + ) + d := NewDiscoverer(kube, nil, nil, "") + got, err := d.Discover(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("discovered %d clusters: %+v", len(got), got) + } + a, b := got[0], got[1] + if a.ExternalID() != "vcluster-team-a-train" || !a.Ready || a.TenantID(TenantPerCluster) != "acme" || a.DisplayName != "Acme AI" || a.Meta("tenant-class") != "public" { + t.Fatalf("team-a cluster: %+v", a) + } + if b.Ready || b.TenantID(TenantPerCluster) != "vcluster-team-b-sleepy" { + t.Fatalf("sleeping cluster must be discovered but not ready: %+v", b) + } +} + +func TestDiscoverFailsLoudlyInsteadOfReturningEmpty(t *testing.T) { + kube := fake.NewSimpleClientset() + kube.PrependReactor("list", "statefulsets", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("forbidden") + }) + if _, err := NewDiscoverer(kube, nil, nil, "").Discover(context.Background()); err == nil { + t.Fatal("a failed list must be an error, never an empty (all clusters deleted) result") + } +} + +func TestPlatformEnrichment(t *testing.T) { + kube := fake.NewSimpleClientset(nsObj("loft-research-v-train", nil), sts("loft-research-v-train", "train", 1, nil)) + vci := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "management.loft.sh/v1", "kind": "VirtualClusterInstance", + "metadata": map[string]interface{}{"name": "train", "namespace": "p-research", "annotations": map[string]interface{}{AnnotationPrefix + "plan": "gpu-pro"}}, + "spec": map[string]interface{}{"displayName": "Research Training", + "clusterRef": map[string]interface{}{"cluster": "syd-1", "namespace": "loft-research-v-train", "virtualCluster": "train"}}, + }} + other := vci.DeepCopy() + other.SetName("elsewhere") + other.Object["spec"].(map[string]interface{})["clusterRef"] = map[string]interface{}{"cluster": "mel-1", "namespace": "loft-research-v-train", "virtualCluster": "train"} + other.Object["spec"].(map[string]interface{})["displayName"] = "Wrong Region" + + scheme := runtime.NewScheme() + gvr := schema.GroupVersionResource{Group: "management.loft.sh", Version: "v1", Resource: "virtualclusterinstances"} + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{gvr: "VirtualClusterInstanceList"}, vci, other) + + got, err := NewDiscoverer(kube, dyn, nil, "syd-1").Discover(context.Background()) + if err != nil { + t.Fatal(err) + } + c := got[0] + if c.Project != "research" || c.DisplayName != "Research Training" || c.Meta("plan") != "gpu-pro" { + t.Fatalf("platform enrichment: %+v", c) + } + if c.TenantID(TenantPerProject) != "project-research" { + t.Fatalf("project tenant = %q", c.TenantID(TenantPerProject)) + } +} diff --git a/internal/enforcement/enforcer.go b/internal/enforcement/enforcer.go new file mode 100644 index 0000000..df477fb --- /dev/null +++ b/internal/enforcement/enforcer.go @@ -0,0 +1,378 @@ +// Package enforcement turns billing webhooks (Stripe dunning and alerts, +// Metronome spend and credit alerts) into a billing state per tenant and, +// depending on ENFORCEMENT_MODE, reflects it on the tenant's clusters: +// +// observe record and expose the state (API, metrics) only +// annotate also annotate the tenant cluster workload and label its namespace +// enforce also label suspended namespaces vbilling.vcluster.com/suspended=true, +// which the shipped ValidatingAdmissionPolicy uses to deny new pods +// +// Running workloads are never deleted: suspension stops growth, not service. +package enforcement + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/metronome" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" + "github.com/vclusterlabs-experiments/vbilling/internal/telemetry" +) + +// State is a tenant's billing standing. +type State string + +const ( + Active State = "active" + Warning State = "warning" // budget or credit alert + Delinquent State = "delinquent" // payment failed, retries pending + Suspended State = "suspended" // unpaid, canceled, or a rule says so +) + +func (s State) valid() bool { return s == Active || s == Warning || s == Delinquent || s == Suspended } + +// Labels and annotations written to tenant clusters. +const ( + LabelState = "vbilling.vcluster.com/billing-state" + LabelSuspended = "vbilling.vcluster.com/suspended" + AnnotationState = "vbilling.vcluster.com/billing-state" + AnnotationWhy = "vbilling.vcluster.com/billing-reason" + AnnotationAt = "vbilling.vcluster.com/billing-state-at" +) + +// Signal is a normalized billing event. +type Signal struct { + ID string `json:"id"` + Source string `json:"source"` + Type string `json:"type"` + CustomerID string `json:"customer_id"` + Tenant string `json:"tenant"` + State State `json:"state"` + Reason string `json:"reason"` + At time.Time `json:"at"` +} + +// TenantState is the current standing of one tenant. +type TenantState struct { + Tenant string `json:"tenant"` + State State `json:"state"` + Reason string `json:"reason"` + Source string `json:"source"` + EventType string `json:"event_type"` + Since time.Time `json:"since"` +} + +// Options configure the enforcer. +type Options struct { + Mode string // observe | annotate | enforce + StripeWebhookSecret string + MetronomeWebhookSecret string + Rules map[string]State // event type -> state overrides + Clusters func(tenant string) []discovery.TenantCluster + Resolvers map[string]destinations.CustomerResolver // "stripe", "metronome" + State *kv.Store + Telemetry *telemetry.Registry + Now func() time.Time +} + +// Enforcer is safe for concurrent use. +type Enforcer struct { + kube kubernetes.Interface + opts Options + seen *destinations.SentCache + + mu sync.Mutex + states map[string]TenantState +} + +func New(kube kubernetes.Interface, opts Options) *Enforcer { + if opts.Now == nil { + opts.Now = time.Now + } + if opts.State == nil { + opts.State = kv.Memory() + } + if opts.Telemetry == nil { + opts.Telemetry = telemetry.New() + } + e := &Enforcer{kube: kube, opts: opts, seen: destinations.NewSentCache(10_000), states: map[string]TenantState{}} + for tenant, raw := range opts.State.WithPrefix("state/") { + var ts TenantState + if json.Unmarshal([]byte(raw), &ts) == nil { + e.states[tenant] = ts + } + } + return e +} + +// ParseRules parses "event.type=state,..." (ENFORCEMENT_RULES). +func ParseRules(spec string) (map[string]State, error) { + out := map[string]State{} + for _, item := range strings.Split(spec, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + k, v, ok := strings.Cut(item, "=") + if !ok || !State(v).valid() { + return nil, fmt.Errorf("enforcement rule %q must be event.type=active|warning|delinquent|suspended", item) + } + out[strings.TrimSpace(k)] = State(strings.TrimSpace(v)) + } + return out, nil +} + +// Errors the webhook handlers map to HTTP status codes. +var ( + // ErrIgnored: verified, but carries no billing signal (reply 200). + ErrIgnored = errors.New("event type not relevant to billing state") + // ErrSignature: the webhook failed verification (reply 400). + ErrSignature = errors.New("webhook signature verification failed") + // ErrUnknownCustomer: not one of our tenants (reply 200 so the provider stops retrying). + ErrUnknownCustomer = errors.New("customer is not a vBilling tenant") + // ErrDuplicate: this provider event was already applied (reply 200). + ErrDuplicate = errors.New("event already applied") +) + +// HandleStripe verifies and applies a Stripe webhook. +func (e *Enforcer) HandleStripe(ctx context.Context, body []byte, sigHeader string) (Signal, error) { + if err := stripe.VerifySignature(body, sigHeader, e.opts.StripeWebhookSecret, stripe.DefaultTolerance, e.opts.Now()); err != nil { + return Signal{}, fmt.Errorf("%w: %v", ErrSignature, err) + } + var ev struct { + ID string `json:"id"` + Type string `json:"type"` + Data struct { + Object struct { + Customer string `json:"customer"` + Status string `json:"status"` + AttemptCount int `json:"attempt_count"` + } `json:"object"` + } `json:"data"` + } + if err := json.Unmarshal(body, &ev); err != nil { + return Signal{}, fmt.Errorf("decode: %w", err) + } + sig := Signal{ID: "stripe:" + ev.ID, Source: "stripe", Type: ev.Type, CustomerID: ev.Data.Object.Customer, At: e.opts.Now().UTC()} + switch ev.Type { + case "invoice.payment_failed": + sig.State, sig.Reason = Delinquent, fmt.Sprintf("invoice payment failed (attempt %d)", ev.Data.Object.AttemptCount) + case "invoice.paid", "invoice.payment_succeeded": + sig.State, sig.Reason = Active, "invoice paid" + case "customer.subscription.created", "customer.subscription.updated", "customer.subscription.resumed": + switch ev.Data.Object.Status { + case "past_due": + sig.State = Delinquent + case "unpaid", "canceled", "incomplete_expired": + sig.State = Suspended + case "active", "trialing": + sig.State = Active + } + sig.Reason = "subscription " + ev.Data.Object.Status + case "customer.subscription.deleted": + sig.State, sig.Reason = Suspended, "subscription deleted" + case "billing.alert.triggered": + sig.State, sig.Reason = Warning, "usage alert triggered" + } + err := e.apply(ctx, &sig) + return sig, err +} + +// HandleMetronome verifies and applies a Metronome webhook. +func (e *Enforcer) HandleMetronome(ctx context.Context, body []byte, date, signature string) (Signal, error) { + if err := metronome.VerifySignature(body, date, signature, e.opts.MetronomeWebhookSecret, metronome.DefaultTolerance, e.opts.Now()); err != nil { + return Signal{}, fmt.Errorf("%w: %v", ErrSignature, err) + } + var ev struct { + ID string `json:"id"` + Type string `json:"type"` + CustomerID string `json:"customer_id"` + Properties map[string]any `json:"properties"` + } + if err := json.Unmarshal(body, &ev); err != nil { + return Signal{}, fmt.Errorf("decode: %w", err) + } + cus := ev.CustomerID + if v, ok := ev.Properties["customer_id"].(string); ok && v != "" { + cus = v + } + sig := Signal{ID: "metronome:" + ev.ID, Source: "metronome", Type: ev.Type, CustomerID: cus, At: e.opts.Now().UTC()} + switch { + case strings.HasPrefix(ev.Type, "alerts."): + sig.State, sig.Reason = Warning, strings.ReplaceAll(strings.TrimPrefix(ev.Type, "alerts."), "_", " ") + if name, ok := ev.Properties["alert_name"].(string); ok && name != "" { + sig.Reason = name + } + case ev.Type == "payment_gate.payment_status": + switch ev.Properties["payment_status"] { + case "failed": + sig.State, sig.Reason = Delinquent, "threshold billing payment failed" + case "paid": + sig.State, sig.Reason = Active, "threshold billing payment succeeded" + } + } + err := e.apply(ctx, &sig) + return sig, err +} + +func (e *Enforcer) apply(ctx context.Context, sig *Signal) error { + if override, ok := e.opts.Rules[sig.Type]; ok { + sig.State = override + sig.Reason += " (rule: " + string(override) + ")" + } + if sig.State == "" { + return ErrIgnored + } + if e.seen.Has(sig.ID) { + return ErrDuplicate // providers retry and reorder; each event is applied once + } + resolver := e.opts.Resolvers[sig.Source] + if resolver == nil { + return fmt.Errorf("%w: no %s destination configured to resolve %s", ErrUnknownCustomer, sig.Source, sig.CustomerID) + } + tenant, ok := resolver.TenantForCustomer(ctx, sig.CustomerID) + if !ok { + return fmt.Errorf("%w: %s", ErrUnknownCustomer, sig.CustomerID) + } + sig.Tenant = tenant + if err := e.Set(ctx, tenant, sig.State, sig.Reason, sig.Source, sig.Type); err != nil { + return err + } + e.seen.Add(sig.ID) + return nil +} + +// Set records a tenant's state and reflects it on its tenant clusters. +// It is also used by the API for manual overrides. +func (e *Enforcer) Set(ctx context.Context, tenant string, state State, reason, source, eventType string) error { + if !state.valid() { + return fmt.Errorf("invalid state %q", state) + } + now := e.opts.Now().UTC() + e.mu.Lock() + prev := e.states[tenant] + ts := TenantState{Tenant: tenant, State: state, Reason: reason, Source: source, EventType: eventType, Since: now} + if prev.State == state { + ts.Since = prev.Since + } + e.states[tenant] = ts + e.mu.Unlock() + b, _ := json.Marshal(ts) + if err := e.opts.State.Set("state/"+tenant, string(b)); err != nil { + return err + } + if prev.State != state { + log.Printf("[enforcement] tenant %s: %s -> %s (%s via %s %s, mode=%s)", tenant, orActive(prev.State), state, reason, source, eventType, e.opts.Mode) + } + for _, s := range []State{Active, Warning, Delinquent, Suspended} { + v := 0.0 + if s == state { + v = 1 + } + e.opts.Telemetry.Set("vbilling_tenant_billing_state", "1 for the tenant's current billing state.", map[string]string{"tenant": tenant, "state": string(s)}, v) + } + if e.opts.Mode == "observe" || e.kube == nil || e.opts.Clusters == nil { + return nil + } + var errs []error + for _, cl := range e.opts.Clusters(tenant) { + if err := e.reflect(ctx, cl, ts); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", cl.ExternalID(), err)) + } + } + return errors.Join(errs...) +} + +func orActive(s State) State { + if s == "" { + return Active + } + return s +} + +// reflect writes the state onto one tenant cluster. +func (e *Enforcer) reflect(ctx context.Context, cl discovery.TenantCluster, ts TenantState) error { + ann := map[string]any{AnnotationState: string(ts.State), AnnotationWhy: ts.Reason, AnnotationAt: ts.Since.Format(time.RFC3339)} + patch, _ := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": ann}}) + var err error + switch cl.Source { + case "deployment": + _, err = e.kube.AppsV1().Deployments(cl.Namespace).Patch(ctx, cl.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + default: + _, err = e.kube.AppsV1().StatefulSets(cl.Namespace).Patch(ctx, cl.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + } + if err != nil { + return fmt.Errorf("annotate workload: %w", err) + } + + labels := map[string]any{LabelState: string(ts.State)} + if e.opts.Mode == "enforce" && ts.State == Suspended { + labels[LabelSuspended] = "true" + } else { + labels[LabelSuspended] = nil // merge patch: remove + } + nsPatch, _ := json.Marshal(map[string]any{"metadata": map[string]any{"labels": labels}}) + if _, err := e.kube.CoreV1().Namespaces().Patch(ctx, cl.Namespace, types.MergePatchType, nsPatch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("label namespace: %w", err) + } + + kind := "StatefulSet" + if cl.Source == "deployment" { + kind = "Deployment" + } + evType := corev1.EventTypeNormal + if ts.State != Active { + evType = corev1.EventTypeWarning + } + now := metav1.NewTime(e.opts.Now()) + _, err = e.kube.CoreV1().Events(cl.Namespace).Create(ctx, &corev1.Event{ + // Explicit name (not GenerateName) so the event is idempotent per state change. + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s.billing.%s.%x", cl.Name, ts.State, ts.Since.UnixNano())}, + InvolvedObject: corev1.ObjectReference{Kind: kind, APIVersion: "apps/v1", Namespace: cl.Namespace, Name: cl.Name}, + Reason: "BillingState" + strings.ToUpper(string(ts.State[:1])) + string(ts.State[1:]), + Message: fmt.Sprintf("Billing state %s: %s", ts.State, ts.Reason), + Type: evType, Source: corev1.EventSource{Component: "vbilling"}, + FirstTimestamp: now, LastTimestamp: now, Count: 1, + }, metav1.CreateOptions{}) + if err != nil { + log.Printf("[enforcement] event for %s: %v", cl.ExternalID(), err) // best effort + } + return nil +} + +// States returns all recorded tenant states. +func (e *Enforcer) States() []TenantState { + e.mu.Lock() + defer e.mu.Unlock() + out := make([]TenantState, 0, len(e.states)) + for _, s := range e.states { + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { return out[i].Tenant < out[j].Tenant }) + return out +} + +// StateOf returns a tenant's state (active when unknown). +func (e *Enforcer) StateOf(tenant string) TenantState { + e.mu.Lock() + defer e.mu.Unlock() + if s, ok := e.states[tenant]; ok { + return s + } + return TenantState{Tenant: tenant, State: Active} +} diff --git a/internal/enforcement/enforcer_test.go b/internal/enforcement/enforcer_test.go new file mode 100644 index 0000000..e7f3aaf --- /dev/null +++ b/internal/enforcement/enforcer_test.go @@ -0,0 +1,183 @@ +package enforcement + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/metronome" + "github.com/vclusterlabs-experiments/vbilling/internal/destinations/stripe" + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/kv" +) + +var now = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + +type resolver map[string]string + +func (r resolver) TenantForCustomer(_ context.Context, id string) (string, bool) { + t, ok := r[id] + return t, ok +} + +func setup(t *testing.T, mode string, rules map[string]State) (*Enforcer, *fake.Clientset, *kv.Store) { + t.Helper() + kube := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "team-a"}}, + &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "gpu", Namespace: "team-a"}}, + ) + state := kv.Memory() + e := New(kube, Options{ + Mode: mode, StripeWebhookSecret: "whsec_test", MetronomeWebhookSecret: "mtr_secret", Rules: rules, State: state, + Clusters: func(tenant string) []discovery.TenantCluster { + if tenant != "acme" { + return nil + } + return []discovery.TenantCluster{{Name: "gpu", Namespace: "team-a", Source: "statefulset"}} + }, + Resolvers: map[string]destinations.CustomerResolver{"stripe": resolver{"cus_acme": "acme"}, "metronome": resolver{"5f794d50-085a-4db6-8d15-286e518b7225": "acme"}}, + Now: func() time.Time { return now }, + }) + return e, kube, state +} + +func stripeEvent(t *testing.T, e *Enforcer, id, typ, status string) (Signal, error) { + body := []byte(fmt.Sprintf(`{"id":%q,"type":%q,"data":{"object":{"customer":"cus_acme","status":%q,"attempt_count":2}}}`, id, typ, status)) + return e.HandleStripe(context.Background(), body, stripe.SignatureHeader(body, "whsec_test", now)) +} + +func nsLabels(t *testing.T, kube *fake.Clientset) map[string]string { + ns, err := kube.CoreV1().Namespaces().Get(context.Background(), "team-a", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return ns.Labels +} + +func TestDunningLifecycleInEnforceMode(t *testing.T) { + e, kube, _ := setup(t, "enforce", nil) + + sig, err := stripeEvent(t, e, "evt_1", "invoice.payment_failed", "") + if err != nil || sig.State != Delinquent || sig.Tenant != "acme" { + t.Fatalf("payment_failed: %+v %v", sig, err) + } + sts, _ := kube.AppsV1().StatefulSets("team-a").Get(context.Background(), "gpu", metav1.GetOptions{}) + if sts.Annotations[AnnotationState] != "delinquent" || !strings.Contains(sts.Annotations[AnnotationWhy], "attempt 2") { + t.Fatalf("workload annotations: %v", sts.Annotations) + } + if l := nsLabels(t, kube); l[LabelState] != "delinquent" || l[LabelSuspended] != "" { + t.Fatalf("delinquent must not suspend: %v", l) + } + + if _, err := stripeEvent(t, e, "evt_2", "customer.subscription.updated", "unpaid"); err != nil { + t.Fatal(err) + } + if l := nsLabels(t, kube); l[LabelSuspended] != "true" || l[LabelState] != "suspended" { + t.Fatalf("unpaid must suspend in enforce mode: %v", l) + } + + if _, err := stripeEvent(t, e, "evt_3", "invoice.paid", ""); err != nil { + t.Fatal(err) + } + if l := nsLabels(t, kube); l[LabelSuspended] != "" || l[LabelState] != "active" { + t.Fatalf("paid invoice must lift suspension: %v", l) + } + events, _ := kube.CoreV1().Events("team-a").List(context.Background(), metav1.ListOptions{}) + if len(events.Items) != 3 { + t.Fatalf("kubernetes events: %d", len(events.Items)) + } +} + +func TestAnnotateModeNeverSuspendsAndObserveTouchesNothing(t *testing.T) { + e, kube, _ := setup(t, "annotate", nil) + if _, err := stripeEvent(t, e, "evt_1", "customer.subscription.deleted", ""); err != nil { + t.Fatal(err) + } + if l := nsLabels(t, kube); l[LabelState] != "suspended" || l[LabelSuspended] != "" { + t.Fatalf("annotate mode labels: %v", l) + } + + o, okube, _ := setup(t, "observe", nil) + if _, err := stripeEvent(t, o, "evt_1", "customer.subscription.deleted", ""); err != nil { + t.Fatal(err) + } + if l := nsLabels(t, okube); len(l) != 0 { + t.Fatalf("observe mode must not write to the cluster: %v", l) + } + if o.StateOf("acme").State != Suspended { + t.Fatal("observe mode must still record the state") + } +} + +func TestSignatureDedupeAndIrrelevantEvents(t *testing.T) { + e, _, _ := setup(t, "enforce", nil) + body := []byte(`{"id":"evt_x","type":"invoice.payment_failed","data":{"object":{"customer":"cus_acme"}}}`) + if _, err := e.HandleStripe(context.Background(), body, stripe.SignatureHeader(body, "wrong", now)); err == nil { + t.Fatal("bad signature accepted") + } + if _, err := stripeEvent(t, e, "evt_9", "customer.created", ""); !errors.Is(err, ErrIgnored) { + t.Fatalf("irrelevant event: %v", err) + } + if _, err := stripeEvent(t, e, "evt_1", "invoice.payment_failed", ""); err != nil { + t.Fatal(err) + } + if _, err := stripeEvent(t, e, "evt_2", "invoice.paid", ""); err != nil { + t.Fatal(err) + } + // A late redelivery of the old failure must not flip the tenant back. + if _, err := stripeEvent(t, e, "evt_1", "invoice.payment_failed", ""); !errors.Is(err, ErrDuplicate) { + t.Fatalf("redelivery: %v", err) + } + if got := e.StateOf("acme").State; got != Active { + t.Fatalf("redelivered event re-applied: %s", got) + } + body = []byte(`{"id":"evt_3","type":"invoice.payment_failed","data":{"object":{"customer":"cus_stranger"}}}`) + if _, err := e.HandleStripe(context.Background(), body, stripe.SignatureHeader(body, "whsec_test", now)); err == nil { + t.Fatal("unknown customer accepted") + } +} + +func TestMetronomeAlertsAndRules(t *testing.T) { + rules, err := ParseRules("alerts.spend_threshold_reached=suspended") + if err != nil { + t.Fatal(err) + } + e, kube, state := setup(t, "enforce", rules) + send := func(id, typ, props string) (Signal, error) { + body := []byte(fmt.Sprintf(`{"id":%q,"type":%q,"properties":{"customer_id":"5f794d50-085a-4db6-8d15-286e518b7225"%s}}`, id, typ, props)) + date := now.Format(http.TimeFormat) + return e.HandleMetronome(context.Background(), body, date, metronome.Sign(body, date, "mtr_secret")) + } + sig, err := send("a1", "alerts.low_remaining_contract_credit_balance_reached", `,"alert_name":"POC credits below 10%"`) + if err != nil || sig.State != Warning || sig.Reason != "POC credits below 10%" { + t.Fatalf("credit alert: %+v %v", sig, err) + } + sig, err = send("a2", "alerts.spend_threshold_reached", "") + if err != nil || sig.State != Suspended { + t.Fatalf("rule override: %+v %v", sig, err) + } + if l := nsLabels(t, kube); l[LabelSuspended] != "true" { + t.Fatalf("spend cap rule must suspend: %v", l) + } + // States survive restarts. + if raw, ok := state.Get("state/acme"); !ok || !strings.Contains(raw, "suspended") { + t.Fatalf("persisted state: %q", raw) + } + reloaded := New(kube, Options{Mode: "enforce", State: state}) + if reloaded.StateOf("acme").State != Suspended { + t.Fatal("state not restored from disk") + } + if _, err := ParseRules("x=bogus"); err == nil { + t.Fatal("invalid rule accepted") + } +} diff --git a/internal/kv/kv.go b/internal/kv/kv.go new file mode 100644 index 0000000..d5d8c7a --- /dev/null +++ b/internal/kv/kv.go @@ -0,0 +1,98 @@ +// Package kv is a tiny persistent string map adapters use to remember +// backend IDs (tenant -> Stripe customer, metric -> meter) across restarts. +package kv + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" +) + +// Store is safe for concurrent use. A Store with an empty path is memory-only. +type Store struct { + path string + mu sync.Mutex + data map[string]string +} + +// Open loads path if it exists. +func Open(path string) (*Store, error) { + s := &Store{path: path, data: map[string]string{}} + if path == "" { + return s, nil + } + b, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return s, os.MkdirAll(filepath.Dir(path), 0o750) + } + if err != nil { + return nil, err + } + if len(b) > 0 { + if err := json.Unmarshal(b, &s.data); err != nil { + return nil, err + } + } + return s, nil +} + +// Memory returns a memory-only store. +func Memory() *Store { s, _ := Open(""); return s } + +func (s *Store) Get(k string) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.data[k] + return v, ok +} + +func (s *Store) Set(k, v string) error { + s.mu.Lock() + defer s.mu.Unlock() + if old, ok := s.data[k]; ok && old == v { + return nil + } + s.data[k] = v + return s.flushLocked() +} + +func (s *Store) Delete(k string) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.data[k]; !ok { + return nil + } + delete(s.data, k) + return s.flushLocked() +} + +// WithPrefix returns all entries whose key starts with prefix, prefix trimmed. +func (s *Store) WithPrefix(prefix string) map[string]string { + s.mu.Lock() + defer s.mu.Unlock() + out := map[string]string{} + for k, v := range s.data { + if strings.HasPrefix(k, prefix) { + out[strings.TrimPrefix(k, prefix)] = v + } + } + return out +} + +func (s *Store) flushLocked() error { + if s.path == "" { + return nil + } + b, err := json.MarshalIndent(s.data, "", " ") + if err != nil { + return err + } + tmp := s.path + ".tmp" + if err := os.WriteFile(tmp, b, 0o640); err != nil { + return err + } + return os.Rename(tmp, s.path) +} diff --git a/internal/metrics/collector.go b/internal/metrics/collector.go index afbafbf..5b8ac0f 100644 --- a/internal/metrics/collector.go +++ b/internal/metrics/collector.go @@ -1,847 +1,758 @@ +// Package metrics turns the state of tenant clusters into usage events. +// +// Allocation metrics (GPUs, dedicated nodes, storage, load balancers, +// control plane hours, requests-based CPU/memory) are computed from object +// lifetimes overlapping each window, to the second. That is what makes +// backfill after a restart possible and keeps short jobs accurate. +// Consumption metrics (metrics-server CPU/memory, Prometheus egress and +// DCGM utilization) are sampled for the latest window only. package metrics import ( "context" - "encoding/json" "fmt" - "io" "log" - "math" - "net/http" - "os" - "strconv" - "strings" + "sort" "time" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" metricsclient "k8s.io/metrics/pkg/client/clientset/versioned" -) - -// VClusterMetrics contains all billable metrics for a single vCluster namespace. -type VClusterMetrics struct { - Namespace string - Timestamp time.Time - - // Core resources (from metrics-server) - CPUCores float64 // current CPU usage in cores - MemoryBytes float64 // current memory usage in bytes - - // Storage (from PVCs) - StorageBytes float64 // total requested PVC storage in bytes - - // GPU allocation (from pod resource requests) - GPUs []GPUAllocation - - // GPU utilization (from DCGM via Prometheus, optional) - GPUUtilization []GPUUtilizationMetric - - // Network (from Prometheus or kubelet, optional) - NetworkTxBytes float64 // egress bytes since last collection - NetworkRxBytes float64 // ingress bytes since last collection - // LoadBalancer services - LoadBalancerCount int + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) - // Node cost attribution - NodeCostMultiplier float64 // weighted average: 1.0 = on-demand, <1.0 = has spot nodes +const gib = 1024 * 1024 * 1024 + +// Options configure metering behavior. +type Options struct { + Region string + RegionFromNode bool + Basis string // usage | requests | max + MeterControlPlane bool + MeterByNamespace bool + GPUResources []string + SKULabel string + CapacityTypeLabel string + UnhealthyTaints []string + DedicatedNodeLabel string + PrometheusURL string + PromHeaders map[string]string // extra request headers (tokens, X-Scope-OrgID) + PromTokenFile string // bearer token file, re-read per query + EgressQuery string + GPUUtilQuery string // "none" disables GPU utilization + GPUCountQuery string + PrometheusPreset string // "vcluster-platform": fleet observability label selectors + PromMetrics []PromMetric // operator-defined metrics from PromQL + // Tenant clusters with their own nodes: "node" bills each private node + // whole (default), "usage" bills the pods running on them instead. + PrivateNodeBilling string + TenantExcludeNamespaces []string // usage mode: namespaces never billed (default kube-system) + DRAGPUDrivers []string // DRA drivers whose devices are GPUs (default gpu.nvidia.com, gpu.amd.com) + Now func() time.Time +} + +// Collector gathers billable state from the Kubernetes API. +type Collector struct { + kube kubernetes.Interface + metrics metricsclient.Interface // nil disables usage-based CPU/memory + prom *prometheusClient + graveyard *Graveyard // deletions seen between collections; nil = none + tenants TenantAPI // tenant clusters' own APIs; nil = control plane cluster only + dyn dynamic.Interface // DRA ResourceClaims and ResourceSlices; nil = none + opts Options - // Private/dedicated nodes allocated to this vCluster - PrivateNodes []PrivateNode + excludedTenantNS map[string]bool } -// GPUAllocation tracks GPU resources requested by pods. -type GPUAllocation struct { - GPUType string // e.g. "NVIDIA-A100-SXM4-80GB" from node label - Count int64 // number of GPUs allocated - NodeName string -} +// UseGraveyard makes the collector bill pods and nodes deleted between +// collections (see Graveyard). +func (c *Collector) UseGraveyard(g *Graveyard) { c.graveyard = g } -// GPUUtilizationMetric tracks actual GPU utilization from DCGM exporter. -type GPUUtilizationMetric struct { - GPUType string - GPUUUID string - Utilization float64 // 0-100 percent - MemoryUsedMB float64 - MemoryTotalMB float64 - PodName string - PodNamespace string -} +// UseDynamic enables DRA (ResourceClaim) GPU metering on the control plane cluster. +func (c *Collector) UseDynamic(d dynamic.Interface) { c.dyn = d } -// PrivateNode represents a dedicated node allocated to a vCluster tenant. -// In private node mode, the entire node is billed to the tenant. -type PrivateNode struct { - NodeName string - CPUCores int64 // total CPU capacity - MemoryBytes int64 // total memory capacity - GPUCount int64 // total GPUs on this node - GPUType string // GPU model from labels - StorageBytes int64 // total ephemeral storage capacity - IsSpot bool // spot/preemptible node - InstanceType string // cloud instance type (e.g. p4d.24xlarge) -} +// baseFn builds an event template for a tenant cluster and node. +type baseFn func(metric string, n *nodeInfo) usage.Event -// Collector gathers resource metrics from the Kubernetes cluster. -type Collector struct { - kubeClient kubernetes.Interface - metricsClient metricsclient.Interface - promClient *prometheusClient // nil if Prometheus not configured - spotDiscount float64 // percentage discount for spot nodes +// podEntry is a pod to meter; deletedAt is set for pods seen deleted. +type podEntry struct { + pod *corev1.Pod + deletedAt time.Time } -func NewCollector(kubeClient kubernetes.Interface, metricsClient metricsclient.Interface, prometheusURL string, spotDiscount float64) *Collector { - var prom *prometheusClient - if prometheusURL != "" { - prom = newPrometheusClient(prometheusURL) - log.Printf("[metrics] Prometheus integration enabled: %s", prometheusURL) - } else { - log.Printf("[metrics] Prometheus not configured - DCGM and network metrics unavailable") +// withDeleted adds pods deleted since the last collection to a live list. +func withDeleted(live []corev1.Pod, gone []deleted[*corev1.Pod]) []podEntry { + out := make([]podEntry, 0, len(live)+len(gone)) + seen := make(map[string]bool, len(live)) + for i := range live { + out = append(out, podEntry{pod: &live[i]}) + seen[string(live[i].UID)] = true } - - return &Collector{ - kubeClient: kubeClient, - metricsClient: metricsClient, - promClient: prom, - spotDiscount: spotDiscount, + for _, d := range gone { + if !seen[string(d.obj.UID)] { + out = append(out, podEntry{pod: d.obj, deletedAt: d.at}) + } } + return out } -// Collect gathers all billable metrics for a vCluster namespace. -func (c *Collector) Collect(ctx context.Context, namespace string) (*VClusterMetrics, error) { - m := &VClusterMetrics{ - Namespace: namespace, - Timestamp: time.Now(), - NodeCostMultiplier: 1.0, +func entryLabels(pods []podEntry) []map[string]string { + out := make([]map[string]string, len(pods)) + for i := range pods { + out[i] = pods[i].pod.Labels } + return out +} - // Collect all metrics concurrently using goroutines would be nice, - // but for clarity and debuggability we do them sequentially. - // Each method logs its own errors and returns partial results. - - c.collectCPUMemory(ctx, namespace, m) - c.collectStorage(ctx, namespace, m) - c.collectGPUAllocation(ctx, namespace, m) - c.collectLoadBalancers(ctx, namespace, m) - c.collectNodeCostAttribution(ctx, namespace, m) - - // Optional Prometheus-based metrics - if c.promClient != nil { - c.collectDCGMMetrics(ctx, namespace, m) - c.collectNetworkMetrics(ctx, namespace, m) +func NewCollector(kube kubernetes.Interface, mc metricsclient.Interface, opts Options) *Collector { + if opts.Now == nil { + opts.Now = time.Now } - - // Collect private/dedicated nodes for this vCluster - c.collectPrivateNodes(ctx, namespace, m) - - // If private nodes were found, also fetch their actual CPU/memory usage - // from metrics-server so both shared-namespace pods AND private node - // usage are captured in the billing totals. - if m.HasPrivateNodes() { - c.collectPrivateNodeUsage(ctx, namespace, m) + if opts.Basis == "" { + opts.Basis = "usage" } - - return m, nil -} - -// TotalGPUCount returns the total number of GPUs allocated across all types. -func (m *VClusterMetrics) TotalGPUCount() int64 { - var total int64 - for _, g := range m.GPUs { - total += g.Count + egress, util, count := defaultEgressQuery, defaultGPUUtilQuery, defaultGPUCountQuery + if opts.PrometheusPreset == PresetVClusterPlatform { + egress, util, count = platformQueries.egress, platformQueries.util, platformQueries.count } - return total -} - -// GPUCountByType returns GPU counts grouped by GPU model. -func (m *VClusterMetrics) GPUCountByType() map[string]int64 { - result := make(map[string]int64) - for _, g := range m.GPUs { - result[g.GPUType] += g.Count + if opts.EgressQuery == "" { + opts.EgressQuery = egress + } + // A custom utilization query usually means a different exporter, so the + // DCGM device count only rides along with the default query. + if opts.GPUUtilQuery == "" { + opts.GPUUtilQuery = util + if opts.GPUCountQuery == "" { + opts.GPUCountQuery = count + } } - return result + if opts.PrivateNodeBilling == "" { + opts.PrivateNodeBilling = "node" + } + if opts.TenantExcludeNamespaces == nil { + opts.TenantExcludeNamespaces = []string{"kube-system"} + } + if opts.DRAGPUDrivers == nil { + opts.DRAGPUDrivers = DefaultDRAGPUDrivers + } + c := &Collector{kube: kube, metrics: mc, opts: opts, excludedTenantNS: map[string]bool{}} + for _, ns := range opts.TenantExcludeNamespaces { + c.excludedTenantNS[ns] = true + } + if opts.PrometheusURL != "" { + c.prom = newPrometheusClient(opts.PrometheusURL, opts.PromHeaders, opts.PromTokenFile) + } + return c } -// MemoryGB returns memory usage in GB. -func (m *VClusterMetrics) MemoryGB() float64 { - return m.MemoryBytes / (1024 * 1024 * 1024) +// Window is a closed metering window. Latest marks the window being +// collected live (consumption metrics are only sampled for it). +type Window struct { + Start, End time.Time + Latest bool } -// StorageGB returns storage in GB. -func (m *VClusterMetrics) StorageGB() float64 { - return m.StorageBytes / (1024 * 1024 * 1024) +// Target is a tenant cluster with its resolved billing identity. +type Target struct { + Cluster discovery.TenantCluster + Tenant string + Project string + TenantClass string } -// NetworkEgressGB returns egress traffic in GB. -func (m *VClusterMetrics) NetworkEgressGB() float64 { - return m.NetworkTxBytes / (1024 * 1024 * 1024) +// Stats summarize one collection run. +type Stats struct { + Pods int + GPUPods int + NodesDown int + Dedicated int + PromFailures int + PrivateNodes int + // TenantAPIFailed lists tenant clusters (external IDs) whose own API + // could not be read; nothing from it was metered for these windows. + TenantAPIFailed []string } -// HasPrivateNodes returns true if this vCluster has dedicated nodes allocated. -func (m *VClusterMetrics) HasPrivateNodes() bool { - return len(m.PrivateNodes) > 0 +// accumulator merges observations with the same identity within one +// window into one event. +type accumulator struct { + window Window + events map[string]*usage.Event + order []string } -// PrivateNodeCount returns the number of private/dedicated nodes. -func (m *VClusterMetrics) PrivateNodeCount() int { - return len(m.PrivateNodes) +func newAccumulator(w Window) *accumulator { + return &accumulator{window: w, events: map[string]*usage.Event{}} } -// PrivateNodeGPUsByType returns GPU counts on private nodes grouped by GPU model. -func (m *VClusterMetrics) PrivateNodeGPUsByType() map[string]int64 { - result := make(map[string]int64) - for _, n := range m.PrivateNodes { - if n.GPUCount > 0 && n.GPUType != "" { - result[n.GPUType] += n.GPUCount +func (a *accumulator) add(tmpl usage.Event, qty float64, props map[string]any) { + if qty <= 0 { + return + } + tmpl.ID = "" + tmpl.WindowStart, tmpl.WindowEnd = a.window.Start.UTC(), a.window.End.UTC() + key := usage.DeriveID(&tmpl) + ev, ok := a.events[key] + if !ok { + e := tmpl + e.Properties = map[string]any{} + a.events[key] = &e + a.order = append(a.order, key) + ev = &e + } + ev.Quantity += qty + for k, v := range props { + switch n := v.(type) { + case int: + prev, _ := ev.Properties[k].(int) + ev.Properties[k] = prev + n + case float64: + prev, _ := ev.Properties[k].(float64) + ev.Properties[k] = prev + n + default: + ev.Properties[k] = v } } - return result } -// PrivateNodeTotalCPU returns the total CPU cores across all private nodes. -func (m *VClusterMetrics) PrivateNodeTotalCPU() int64 { - var total int64 - for _, n := range m.PrivateNodes { - total += n.CPUCores +func (a *accumulator) list() []usage.Event { + out := make([]usage.Event, 0, len(a.order)) + for _, k := range a.order { + e := *a.events[k] + if len(e.Properties) == 0 { + e.Properties = nil + } + out = append(out, e) } - return total + return out } -// PrivateNodeTotalMemoryGB returns total memory in GB across all private nodes. -func (m *VClusterMetrics) PrivateNodeTotalMemoryGB() float64 { - var total int64 - for _, n := range m.PrivateNodes { - total += n.MemoryBytes - } - return float64(total) / (1024 * 1024 * 1024) +// Collect meters every target over every window. The result maps each +// window end to its events, ready to commit window by window. +func (c *Collector) Collect(ctx context.Context, targets []Target, windows []Window) (map[time.Time][]usage.Event, Stats, error) { + return c.collect(ctx, targets, windows, false) } -// --- CPU and Memory from metrics-server --- - -func (c *Collector) collectCPUMemory(ctx context.Context, namespace string, m *VClusterMetrics) { - podMetrics, err := c.metricsClient.MetricsV1beta1().PodMetricses(namespace).List(ctx, metav1.ListOptions{}) +func (c *Collector) collect(ctx context.Context, targets []Target, windows []Window, onlyTenantAPI bool) (map[time.Time][]usage.Event, Stats, error) { + var st Stats + out := map[time.Time][]usage.Event{} + if len(windows) == 0 { + return out, st, nil + } + nodeList, err := c.kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - log.Printf("[metrics] warning: cannot get pod metrics for %s: %v", namespace, err) - return + return nil, st, fmt.Errorf("list nodes: %w", err) + } + c.graveyard.SeenNodes(nodeList.Items, c.opts.Now()) + nodes := map[string]*nodeInfo{} + for i := range nodeList.Items { + ni := c.parseNode(&nodeList.Items[i]) + nodes[ni.name] = ni + if ni.down { + st.NodesDown++ + } } - - for _, pod := range podMetrics.Items { - for _, container := range pod.Containers { - cpu := container.Usage.Cpu() - mem := container.Usage.Memory() - if cpu != nil { - m.CPUCores += float64(cpu.MilliValue()) / 1000.0 - } - if mem != nil { - m.MemoryBytes += float64(mem.Value()) - } + for _, d := range c.graveyard.Nodes() { + if _, live := nodes[d.obj.Name]; !live { + ni := c.parseNode(d.obj) + ni.deletedAt = d.at + nodes[ni.name] = ni } } - log.Printf("[metrics] %s: CPU=%.3f cores, Memory=%.2f GB", namespace, m.CPUCores, m.MemoryGB()) -} - -// --- Storage from PVCs --- -func (c *Collector) collectStorage(ctx context.Context, namespace string, m *VClusterMetrics) { - pvcs, err := c.kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - log.Printf("[metrics] warning: cannot list PVCs in %s: %v", namespace, err) - return + dra := c.draFor(ctx, c.dyn, "") + applyDRANodes(nodes, dra) + accs := map[time.Time]*accumulator{} + for _, w := range windows { + accs[w.End] = newAccumulator(w) } - - for _, pvc := range pvcs.Items { - if pvc.Status.Phase != corev1.ClaimBound { - continue + now := c.opts.Now().UTC() + if r, ok := c.tenants.(interface { + Retain([]discovery.TenantCluster) + }); ok && !onlyTenantAPI { + clusters := make([]discovery.TenantCluster, len(targets)) + for i := range targets { + clusters[i] = targets[i].Cluster } - storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage] - m.StorageBytes += float64(storage.Value()) - } - log.Printf("[metrics] %s: Storage=%.2f GB (%d PVCs)", namespace, m.StorageGB(), len(pvcs.Items)) -} - -// --- GPU allocation from pod resource requests --- - -func (c *Collector) collectGPUAllocation(ctx context.Context, namespace string, m *VClusterMetrics) { - pods, err := c.kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - FieldSelector: "status.phase=Running", - }) - if err != nil { - log.Printf("[metrics] warning: cannot list pods in %s: %v", namespace, err) - return + r.Retain(clusters) } - - // Cache node GPU types - nodeGPUType := make(map[string]string) - - for _, pod := range pods.Items { - gpuCount := podGPUCount(&pod) - if gpuCount == 0 { - continue + for _, t := range targets { + // External tenant clusters have nothing on this control plane cluster. + if !onlyTenantAPI && !t.Cluster.External { + if err := c.collectTarget(ctx, t, nodes, dra, windows, accs, &st); err != nil { + return nil, st, fmt.Errorf("meter %s: %w", t.Cluster.ExternalID(), err) + } } - - // Determine GPU type from the node this pod runs on - nodeName := pod.Spec.NodeName - gpuType, ok := nodeGPUType[nodeName] - if !ok { - gpuType = c.getNodeGPUType(ctx, nodeName) - nodeGPUType[nodeName] = gpuType + if c.tenants != nil { + c.collectTenant(ctx, t, nodes, windows, accs, &st) } - - m.GPUs = append(m.GPUs, GPUAllocation{ - GPUType: gpuType, - Count: gpuCount, - NodeName: nodeName, - }) - } - - if total := m.TotalGPUCount(); total > 0 { - log.Printf("[metrics] %s: GPUs=%d allocated (%v)", namespace, total, m.GPUCountByType()) } -} - -// podGPUCount returns the total nvidia.com/gpu requests across all containers. -func podGPUCount(pod *corev1.Pod) int64 { - var total int64 - gpuResource := corev1.ResourceName("nvidia.com/gpu") - - for _, c := range pod.Spec.Containers { - if qty, ok := c.Resources.Requests[gpuResource]; ok { - total += qty.Value() - } - if qty, ok := c.Resources.Limits[gpuResource]; ok && total == 0 { - total += qty.Value() + for _, w := range windows { + var evs []usage.Event + for _, ev := range accs[w.End].list() { + ev.Source = "collector" + ev.RecordedAt = now + if !w.Latest { + if ev.Properties == nil { + ev.Properties = map[string]any{} + } + ev.Properties["backfilled"] = true + } + ev.Finalize() + // Never commit an event the backends would reject: a bug here must + // surface loudly rather than as dead letters downstream. + if err := ev.Validate(); err != nil { + return nil, st, fmt.Errorf("invalid event for %s/%s: %w", ev.Tenant, ev.Metric, err) + } + evs = append(evs, ev) } + sort.Slice(evs, func(i, j int) bool { return evs[i].ID < evs[j].ID }) + out[w.End] = evs } - return total + return out, st, nil } -// getNodeGPUType reads the GPU model from node labels. -func (c *Collector) getNodeGPUType(ctx context.Context, nodeName string) string { - if nodeName == "" { - return "unknown" - } +func (c *Collector) collectTarget(ctx context.Context, t Target, nodes map[string]*nodeInfo, dra *draIndex, windows []Window, accs map[time.Time]*accumulator, st *Stats) error { + cl := t.Cluster + ns := cl.Namespace + extID := cl.ExternalID() - node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + base := c.baseFor(t) + + podList, err := c.kube.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{}) if err != nil { - log.Printf("[metrics] warning: cannot get node %s: %v", nodeName, err) - return "unknown" + return fmt.Errorf("list pods: %w", err) } + c.graveyard.SeenPods(podList.Items, c.opts.Now()) + pods := withDeleted(podList.Items, c.graveyard.Pods(ns)) + owned := ownedFilter(entryLabels(pods), cl.Name, c.opts.MeterControlPlane) - // Check common GPU label conventions - gpuLabels := []string{ - "nvidia.com/gpu.product", // NVIDIA GPU Operator - "nvidia.com/gpu.machine", // alternative - "accelerator", // GKE - "cloud.google.com/gke-accelerator", // GKE specific - "k8s.amazonaws.com/accelerator", // EKS - "node.kubernetes.io/instance-type", // fallback: instance type + // Dedicated nodes: billed whole, so pods on them are not billed again. + dedicated := map[string]*nodeInfo{} + for name, n := range nodes { + if c.dedicatedTo(n, cl.Name, ns, extID) { + dedicated[name] = n + } } + st.Dedicated += len(dedicated) - for _, label := range gpuLabels { - if v, ok := node.Labels[label]; ok && v != "" { - return sanitizeGPUType(v) - } + var usageByPod map[string][2]float64 + latest := windows[len(windows)-1] + if latest.Latest && c.opts.Basis != "requests" && c.metrics != nil { + usageByPod = podUsage(ctx, c.metrics, ns) } - return "unknown" -} + virtualNS := func(p *corev1.Pod) string { return p.Labels[LabelVirtualNamespace] } + c.meterPods(pods, func(p *corev1.Pod) bool { return owned(p.Labels) }, virtualNS, + nodes, dedicated, dra, windows, accs, st, base, usageByPod, "shared") -// sanitizeGPUType normalizes GPU type strings for consistent billing. -func sanitizeGPUType(raw string) string { - // Remove spaces and convert to uppercase for consistency - s := strings.TrimSpace(raw) - s = strings.ReplaceAll(s, " ", "-") - return s -} + // Dedicated nodes: full capacity, per node. + for name, n := range dedicated { + if n.labels[LabelBillable] == "false" { + continue // the tenant's own hardware: neither the node nor its pods are billed + } + c.meterWholeNode(name, n, windows, accs, base, "dedicated_node") + } -// --- LoadBalancer services --- + // Storage: bound PVCs synced from the tenant cluster, per storage class. + pvcs, err := c.kube.CoreV1().PersistentVolumeClaims(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list pvcs: %w", err) + } + meterVolumes(pvcs.Items, ownedFilter(pvcLabels(pvcs.Items), cl.Name, c.opts.MeterControlPlane), windows, accs, base) -func (c *Collector) collectLoadBalancers(ctx context.Context, namespace string, m *VClusterMetrics) { - services, err := c.kubeClient.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{}) + // Load balancers with an assigned address. + svcs, err := c.kube.CoreV1().Services(ns).List(ctx, metav1.ListOptions{}) if err != nil { - log.Printf("[metrics] warning: cannot list services in %s: %v", namespace, err) - return + return fmt.Errorf("list services: %w", err) } + meterLoadBalancers(svcs.Items, func(svc *corev1.Service) bool { + return !isControlPlane(svc.Labels, cl.Name) || c.opts.MeterControlPlane + }, windows, accs, base) - for _, svc := range services.Items { - if svc.Spec.Type == corev1.ServiceTypeLoadBalancer { - m.LoadBalancerCount++ + // Control plane hours, only while ready (no charge while provisioning or asleep). + if cl.Ready { + for _, w := range windows { + ov := usage.Overlap(cl.CreatedAt, time.Time{}, w.Start, w.End) + accs[w.End].add(base(usage.MetricInstanceHours, nil), ov.Hours(), nil) } } - if m.LoadBalancerCount > 0 { - log.Printf("[metrics] %s: LoadBalancers=%d", namespace, m.LoadBalancerCount) + if c.prom != nil && latest.Latest { + c.collectPrometheus(ctx, t, latest, accs[latest.End], base, st) } + return nil } -// --- Node cost attribution (spot vs on-demand) --- - -func (c *Collector) collectNodeCostAttribution(ctx context.Context, namespace string, m *VClusterMetrics) { - pods, err := c.kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - FieldSelector: "status.phase=Running", - }) +func (c *Collector) collectPrometheus(ctx context.Context, t Target, w Window, acc *accumulator, base baseFn, st *Stats) { + size := w.End.Sub(w.Start) + vars := varsFor(t, size) + id := t.Cluster.ExternalID() + res, err := c.prom.Query(ctx, expand(c.opts.EgressQuery, vars), w.End) if err != nil { - return + st.PromFailures++ + log.Printf("[metrics] %s: egress query failed: %v", id, err) + } else { + var bytes float64 + for _, r := range res { + bytes += r.Value + } + acc.add(base(usage.MetricNetworkEgressGB, nil), bytes/gib, nil) } - if len(pods.Items) == 0 { - return + if c.opts.GPUUtilQuery != "none" { + c.collectGPUUtilization(ctx, vars, id, w, acc, base, st) } - // Cache node spot status - nodeSpot := make(map[string]bool) - var spotPods, totalPods int - - for _, pod := range pods.Items { - nodeName := pod.Spec.NodeName - if nodeName == "" { + for _, m := range c.opts.PromMetrics { + res, err := c.prom.Query(ctx, expand(m.Query, vars), w.End) + if err != nil { + st.PromFailures++ + log.Printf("[metrics] %s: %s query failed: %v", id, m.Code, err) continue } - - isSpot, ok := nodeSpot[nodeName] - if !ok { - isSpot = c.isSpotNode(ctx, nodeName) - nodeSpot[nodeName] = isSpot - } - - totalPods++ - if isSpot { - spotPods++ - } - } - - if totalPods > 0 { - spotFraction := float64(spotPods) / float64(totalPods) - // Cost multiplier: spot pods get discounted, on-demand pods stay at 1.0 - discount := (c.spotDiscount / 100.0) * spotFraction - m.NodeCostMultiplier = 1.0 - discount - if spotPods > 0 { - log.Printf("[metrics] %s: %d/%d pods on spot nodes, cost multiplier=%.2f", - namespace, spotPods, totalPods, m.NodeCostMultiplier) + for _, r := range res { + ev := base(m.Code, nil) + for _, l := range m.Dimensions { + if v := r.Labels[l]; v != "" { + ev.Dimensions[dimKey(l)] = v + } + } + if m.SKU != "" { + ev.SKU = r.Labels[m.SKU] + } + acc.add(ev, r.Value, nil) } } } -// isSpotNode checks if a node is a spot/preemptible instance. -func (c *Collector) isSpotNode(ctx context.Context, nodeName string) bool { - node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) +func (c *Collector) collectGPUUtilization(ctx context.Context, vars queryVars, id string, w Window, acc *accumulator, base baseFn, st *Stats) { + util, err := c.prom.Query(ctx, expand(c.opts.GPUUtilQuery, vars), w.End) if err != nil { - return false + st.PromFailures++ + log.Printf("[metrics] %s: GPU utilization query failed: %v", id, err) + return } - - spotLabels := []string{ - "kubernetes.io/lifecycle", // common: "spot" or "preemptible" - "node.kubernetes.io/lifecycle", // alternative - "cloud.google.com/gke-preemptible", // GKE preemptible - "cloud.google.com/gke-spot", // GKE spot - "eks.amazonaws.com/capacityType", // EKS: "SPOT" or "ON_DEMAND" - "karpenter.sh/capacity-type", // Karpenter: "spot" or "on-demand" - "node.kubernetes.io/instance-type", // check below + countBy := map[string]float64{} + if c.opts.GPUCountQuery != "" && len(util) > 0 { + counts, _ := c.prom.Query(ctx, expand(c.opts.GPUCountQuery, vars), w.End) + for _, r := range counts { + countBy[promGPUModel(r.Labels)] = r.Value + } } + for _, r := range util { + model := promGPUModel(r.Labels) + ev := base(usage.MetricGPUUtilization, nil) + ev.Dimensions[usage.DimGPUType] = model + props := map[string]any{"avg_utilization_pct": r.Value} + if n, ok := countBy[model]; ok { + props["gpu_devices"] = n + } + acc.add(ev, r.Value*w.End.Sub(w.Start).Hours(), props) + } +} - spotValues := map[string]bool{ - "spot": true, - "preemptible": true, - "SPOT": true, - "true": true, +// promGPUModel reads the GPU model from a utilization result: DCGM's +// modelName, or gpu_type/model from a custom GPU_UTIL_QUERY. +func promGPUModel(labels map[string]string) string { + if v := firstLabel(labels, "modelName", "gpu_type", "model"); v != "" { + return normalizeGPUType(v) } + return "unknown" +} - for _, label := range spotLabels { - if v, ok := node.Labels[label]; ok { - if spotValues[v] { - return true +// podUsage returns current CPU cores and memory bytes per pod name. +func podUsage(ctx context.Context, mc metricsclient.Interface, ns string) map[string][2]float64 { + out := map[string][2]float64{} + list, err := mc.MetricsV1beta1().PodMetricses(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + log.Printf("[metrics] %s: metrics-server unavailable, usage-based CPU/memory skipped: %v", ns, err) + return out + } + for _, pm := range list.Items { + var cpu, mem float64 + for _, ctr := range pm.Containers { + if q := ctr.Usage.Cpu(); q != nil { + cpu += float64(q.MilliValue()) / 1000 + } + if q := ctr.Usage.Memory(); q != nil { + mem += float64(q.Value()) } } + out[pm.Namespace+"/"+pm.Name] = [2]float64{cpu, mem} } - - return false + return out } -// isSpotNodeFromObj checks if a node object is a spot/preemptible instance -// without fetching the node from the API (used when we already have the object). -func isSpotNodeFromObj(node *corev1.Node) bool { - spotLabels := []string{ - "kubernetes.io/lifecycle", - "node.kubernetes.io/lifecycle", - "cloud.google.com/gke-preemptible", - "cloud.google.com/gke-spot", - "eks.amazonaws.com/capacityType", - "karpenter.sh/capacity-type", - } - - spotValues := map[string]bool{ - "spot": true, - "preemptible": true, - "SPOT": true, - "true": true, - } - - for _, label := range spotLabels { - if v, ok := node.Labels[label]; ok { - if spotValues[v] { - return true - } - } +func podLabels(pods []corev1.Pod) []map[string]string { + out := make([]map[string]string, len(pods)) + for i := range pods { + out[i] = pods[i].Labels } + return out +} - return false +func pvcLabels(pvcs []corev1.PersistentVolumeClaim) []map[string]string { + out := make([]map[string]string, len(pvcs)) + for i := range pvcs { + out[i] = pvcs[i].Labels + } + return out } -// --- Private/dedicated node collection --- - -// collectPrivateNodes finds nodes dedicated to this vCluster and records their -// full capacity for billing. In private node mode, the entire node is billed -// to the tenant regardless of actual utilization. -func (c *Collector) collectPrivateNodes(ctx context.Context, namespace string, m *VClusterMetrics) { - gpuResource := corev1.ResourceName("nvidia.com/gpu") - ephemeralResource := corev1.ResourceName("ephemeral-storage") - - // Search for private nodes using multiple strategies. - // The label value could be the vCluster name, the namespace, or a custom pattern. - // We try all reasonable combinations. - selectors := []string{ - fmt.Sprintf("vcluster.loft.sh/managed-by=%s", namespace), - fmt.Sprintf("vcluster.loft.sh/cluster=%s", namespace), - } - - // Also try with the vCluster name derived from the namespace - // (strip common prefixes like "vcluster-", "vc-") - vclusterName := namespace - for _, prefix := range []string{"vcluster-", "vc-"} { - if strings.HasPrefix(namespace, prefix) { - vclusterName = strings.TrimPrefix(namespace, prefix) +// ownedFilter decides which host objects belong to a tenant cluster. When +// the syncer's managed-by label is present in the namespace it is used +// strictly (several tenant clusters can share a namespace); otherwise every +// non-control-plane object counts, as in v0.1. +func ownedFilter(all []map[string]string, clusterName string, includeControlPlane bool) func(map[string]string) bool { + labeled := false + for _, l := range all { + if l[LabelManagedBy] != "" { + labeled = true break } } - if vclusterName != namespace { - selectors = append(selectors, - fmt.Sprintf("vcluster.loft.sh/managed-by=%s", vclusterName), - fmt.Sprintf("vcluster.loft.sh/cluster=%s", vclusterName), - ) - } - - // Strategy 3: custom label from environment variable. - // Format: VCLUSTER_NODE_LABEL== - // The value pattern may contain %s which is replaced with the vCluster name. - if customLabel := os.Getenv("VCLUSTER_NODE_LABEL"); customLabel != "" { - parts := strings.SplitN(customLabel, "=", 2) - if len(parts) == 2 { - key := parts[0] - valPattern := parts[1] - val := strings.ReplaceAll(valPattern, "%s", vclusterName) - selectors = append(selectors, fmt.Sprintf("%s=%s", key, val)) + return func(l map[string]string) bool { + if isControlPlane(l, clusterName) { + return includeControlPlane + } + if labeled { + return l[LabelManagedBy] == clusterName } + return true } +} - seen := make(map[string]bool) - - for _, sel := range selectors { - nodes, err := c.kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ - LabelSelector: sel, - }) - if err != nil { - log.Printf("[metrics] warning: cannot list nodes with selector %q: %v", sel, err) +// meterPods meters GPU, CPU and memory time of pods outside the skipped +// (whole-node billed) nodes. mode is the billing_mode dimension. +func (c *Collector) meterPods(pods []podEntry, include func(*corev1.Pod) bool, nsOf func(*corev1.Pod) string, + nodes, skip map[string]*nodeInfo, dra *draIndex, windows []Window, accs map[time.Time]*accumulator, st *Stats, + base baseFn, usageByPod map[string][2]float64, mode string) { + for _, e := range pods { + p := e.pod + if !include(p) { continue } - - for _, node := range nodes.Items { - if seen[node.Name] { - continue - } - seen[node.Name] = true - - pn := PrivateNode{ - NodeName: node.Name, - } - - // CPU capacity - if cpu, ok := node.Status.Capacity[corev1.ResourceCPU]; ok { - pn.CPUCores = cpu.Value() - } - - // Memory capacity - if mem, ok := node.Status.Capacity[corev1.ResourceMemory]; ok { - pn.MemoryBytes = mem.Value() + start, end, started := podLifetime(p) + if !started { + continue + } + // A pod deleted while running stops being billed when it was deleted. + if !e.deletedAt.IsZero() && (end.IsZero() || end.After(e.deletedAt)) { + end = e.deletedAt + } + st.Pods++ + n := nodes[p.Spec.NodeName] + if _, ok := skip[p.Spec.NodeName]; ok { + continue + } + capType := usage.CapacityOnDemand + gpuType := "unknown" + if n != nil { + capType, gpuType = n.capacityType, n.gpuType + } + vns := nsOf(p) + withNS := func(e usage.Event) usage.Event { + if c.opts.MeterByNamespace && vns != "" { + e.Dimensions[usage.DimNamespace] = vns } - - // GPU count - if gpu, ok := node.Status.Capacity[gpuResource]; ok { - pn.GPUCount = gpu.Value() + return e + } + allocs := append(c.gpuAllocs(p, n), dra.allocs(p.UID)...) + if gpuType == "unknown" && dra != nil && dra.gpuType[p.UID] != "" { + gpuType = dra.gpuType[p.UID] + } + if len(allocs) > 0 { + st.GPUPods++ + } + reqCPU, reqMem := podRequests(p) + for _, w := range windows { + acc := accs[w.End] + billable, down := n.split(start, end, w.Start, w.End) + for _, a := range allocs { + sku := gpuType + if n != nil && n.sku != "" { + sku = n.sku + } + if a.profile != "full" { + sku += "-" + a.profile + } + ev := withNS(base(usage.MetricGPUHours, n)) + ev.SKU = sku + ev.Dimensions[usage.DimGPUType] = gpuType + ev.Dimensions[usage.DimGPUProfile] = a.profile + ev.Dimensions[usage.DimCapacityType] = capType + ev.Dimensions[usage.DimBillingMode] = mode + gpuEq := a.count * a.fraction + acc.add(ev, gpuEq*billable.Hours(), map[string]any{"gpu_devices": a.count, "pods": 1}) + if down > 0 { + dt := base(usage.MetricGPUDowntimeHours, n) + dt.SKU = sku + dt.Dimensions[usage.DimGPUType] = gpuType + dt.Dimensions[usage.DimNode] = p.Spec.NodeName + acc.add(dt, gpuEq*down.Hours(), map[string]any{"reason": "node not ready or unhealthy"}) + } } - // GPU type from label (check multiple label conventions) - for _, gpuLabel := range []string{ - "nvidia.com/gpu.product", - "nvidia.com/gpu.machine", - "cloud.google.com/gke-accelerator", - "k8s.amazonaws.com/accelerator", - } { - if gpuType, ok := node.Labels[gpuLabel]; ok && gpuType != "" { - pn.GPUType = sanitizeGPUType(gpuType) + var cpu, mem float64 + switch c.opts.Basis { + case "requests": + cpu, mem = reqCPU, reqMem + case "usage", "max": + if !w.Latest { + if c.opts.Basis == "max" { + cpu, mem = reqCPU, reqMem // usage is unknown for past windows + } break } + u := usageByPod[p.Namespace+"/"+p.Name] + cpu, mem = u[0], u[1] + if c.opts.Basis == "max" { + cpu, mem = max(cpu, reqCPU), max(mem, reqMem) + } } - - // Ephemeral storage capacity - if stor, ok := node.Status.Capacity[ephemeralResource]; ok { - pn.StorageBytes = stor.Value() - } - - // Instance type from label - if instType, ok := node.Labels["node.kubernetes.io/instance-type"]; ok { - pn.InstanceType = instType + for _, m := range []struct { + metric string + qty float64 + }{{usage.MetricCPUCoreHours, cpu}, {usage.MetricMemoryGBHours, mem / gib}} { + if m.qty <= 0 { + continue + } + ev := withNS(base(m.metric, n)) + ev.Dimensions[usage.DimCapacityType] = capType + ev.Dimensions[usage.DimBillingMode] = mode + acc.add(ev, m.qty*billable.Hours(), map[string]any{"basis": c.opts.Basis}) } - - // Spot status - pn.IsSpot = isSpotNodeFromObj(&node) - - m.PrivateNodes = append(m.PrivateNodes, pn) } } +} - if len(m.PrivateNodes) > 0 { - var totalCPU int64 - var totalGPU int64 - for _, pn := range m.PrivateNodes { - totalCPU += pn.CPUCores - totalGPU += pn.GPUCount +// meterWholeNode bills a node's full capacity (node-hours, whole GPUs, CPU +// and memory) to one tenant cluster: dedicated nodes of the control plane +// cluster and private nodes of the tenant cluster itself. +func (c *Collector) meterWholeNode(name string, n *nodeInfo, windows []Window, accs map[time.Time]*accumulator, base baseFn, mode string) { + for _, w := range windows { + acc := accs[w.End] + billable, down := n.split(n.created, n.deletedAt, w.Start, w.End) + h := billable.Hours() + sku := nodeSKU(n) + nodeEv := base(usage.MetricPrivateNodeHours, n) + nodeEv.ResourceID = name + nodeEv.SKU = sku + nodeEv.Dimensions[usage.DimCapacityType] = n.capacityType + nodeEv.Dimensions[usage.DimBillingMode] = mode + nodeEv.Dimensions[usage.DimNode] = name + if n.instanceType != "" { + nodeEv.Dimensions[usage.DimInstanceType] = n.instanceType } - log.Printf("[metrics] %s: found %d private node(s): totalCPU=%d cores, totalMemory=%.2f GB, totalGPUs=%d", - namespace, len(m.PrivateNodes), totalCPU, m.PrivateNodeTotalMemoryGB(), totalGPU) - - for _, pn := range m.PrivateNodes { - spotStr := "on-demand" - if pn.IsSpot { - spotStr = "spot" + acc.add(nodeEv, h, nil) + if gpus := n.physicalGPUs(); gpus > 0 { + gev := base(usage.MetricGPUHours, n) + gev.ResourceID = name + gev.SKU = n.gpuType + if n.sku != "" { + gev.SKU = n.sku + } + gev.Dimensions[usage.DimGPUType] = n.gpuType + gev.Dimensions[usage.DimGPUProfile] = "full" + gev.Dimensions[usage.DimCapacityType] = n.capacityType + gev.Dimensions[usage.DimBillingMode] = mode + gev.Dimensions[usage.DimNode] = name + acc.add(gev, gpus*h, map[string]any{"gpu_devices": float64(n.gpuDevices()), "physical_gpus": gpus}) + if down > 0 { + dt := base(usage.MetricGPUDowntimeHours, n) + dt.ResourceID = name + dt.SKU = gev.SKU + dt.Dimensions[usage.DimGPUType] = n.gpuType + dt.Dimensions[usage.DimNode] = name + acc.add(dt, gpus*down.Hours(), map[string]any{"reason": "node not ready or unhealthy"}) } - log.Printf("[metrics] node=%s type=%s cpu=%d mem=%dMi gpus=%d(%s) %s", - pn.NodeName, pn.InstanceType, pn.CPUCores, - pn.MemoryBytes/(1024*1024), pn.GPUCount, pn.GPUType, spotStr) + } else if lost := n.expectedGPUs(); lost > 0 && (h > 0 || down > 0) { + // Installed but not allocatable: GPU downtime, never usage. + dt := base(usage.MetricGPUDowntimeHours, n) + dt.ResourceID = name + dt.SKU = n.gpuType + if n.sku != "" { + dt.SKU = n.sku + } + dt.Dimensions[usage.DimGPUType] = n.gpuType + dt.Dimensions[usage.DimNode] = name + reason := "GPUs installed but not allocatable" + if h == 0 { + reason = "node not ready or unhealthy" + } + acc.add(dt, lost*(h+down.Hours()), map[string]any{"reason": reason}) + } + for _, m := range []struct { + metric string + qty float64 + }{{usage.MetricCPUCoreHours, n.cpuCapacity}, {usage.MetricMemoryGBHours, n.memCapacity / gib}} { + ev := base(m.metric, n) + ev.ResourceID = name + ev.Dimensions[usage.DimCapacityType] = n.capacityType + ev.Dimensions[usage.DimBillingMode] = mode + ev.Dimensions[usage.DimNode] = name + acc.add(ev, m.qty*h, nil) } } } -// collectPrivateNodeUsage queries metrics-server for actual CPU/memory usage -// on private nodes and adds that usage to the billing totals. This ensures -// that workloads running directly on dedicated nodes (outside the vCluster -// namespace) are also captured. -func (c *Collector) collectPrivateNodeUsage(ctx context.Context, namespace string, m *VClusterMetrics) { - var addedCPU float64 - var addedMem float64 - - for _, pn := range m.PrivateNodes { - nodeMetrics, err := c.metricsClient.MetricsV1beta1().NodeMetricses().Get(ctx, pn.NodeName, metav1.GetOptions{}) - if err != nil { - log.Printf("[metrics] warning: cannot get node metrics for private node %s: %v", pn.NodeName, err) +// meterVolumes bills bound volumes per storage class. +func meterVolumes(pvcs []corev1.PersistentVolumeClaim, include func(map[string]string) bool, windows []Window, accs map[time.Time]*accumulator, base baseFn) { + for _, pvc := range pvcs { + if pvc.Status.Phase != corev1.ClaimBound || !include(pvc.Labels) { continue } - - cpu := nodeMetrics.Usage.Cpu() - mem := nodeMetrics.Usage.Memory() - - if cpu != nil { - cpuVal := float64(cpu.MilliValue()) / 1000.0 - m.CPUCores += cpuVal - addedCPU += cpuVal + size := pvc.Status.Capacity[corev1.ResourceStorage] + if size.IsZero() { + size = pvc.Spec.Resources.Requests[corev1.ResourceStorage] } - if mem != nil { - memVal := float64(mem.Value()) - m.MemoryBytes += memVal - addedMem += memVal + class := "default" + if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != "" { + class = *pvc.Spec.StorageClassName + } + for _, w := range windows { + ov := usage.Overlap(pvc.CreationTimestamp.Time, time.Time{}, w.Start, w.End) + ev := base(usage.MetricStorageGBHours, nil) + ev.SKU = class + accs[w.End].add(ev, float64(size.Value())/gib*ov.Hours(), map[string]any{"volumes": 1}) } - } - - if addedCPU > 0 || addedMem > 0 { - log.Printf("[metrics] %s: added private node usage: CPU=+%.3f cores, Memory=+%.2f GB", - namespace, addedCPU, addedMem/(1024*1024*1024)) } } -// --- DCGM GPU utilization from Prometheus --- - -func (c *Collector) collectDCGMMetrics(ctx context.Context, namespace string, m *VClusterMetrics) { - if c.promClient == nil { - return - } - - // Query DCGM GPU utilization for pods in this namespace - // DCGM_FI_DEV_GPU_UTIL{namespace=""} gives GPU utilization 0-100 - query := fmt.Sprintf(`DCGM_FI_DEV_GPU_UTIL{namespace="%s"}`, namespace) - results, err := c.promClient.Query(ctx, query) - if err != nil { - log.Printf("[metrics] warning: DCGM query failed for %s: %v", namespace, err) - return - } - - for _, r := range results { - util, _ := strconv.ParseFloat(r.Value, 64) - - // Also get memory usage for this GPU - var memUsed, memTotal float64 - memQuery := fmt.Sprintf(`DCGM_FI_DEV_FB_USED{namespace="%s",gpu="%s"}`, namespace, r.Labels["gpu"]) - memResults, err := c.promClient.Query(ctx, memQuery) - if err == nil && len(memResults) > 0 { - memUsed, _ = strconv.ParseFloat(memResults[0].Value, 64) - } - memTotalQuery := fmt.Sprintf(`DCGM_FI_DEV_FB_FREE{namespace="%s",gpu="%s"} + DCGM_FI_DEV_FB_USED{namespace="%s",gpu="%s"}`, - namespace, r.Labels["gpu"], namespace, r.Labels["gpu"]) - memTotalResults, err := c.promClient.Query(ctx, memTotalQuery) - if err == nil && len(memTotalResults) > 0 { - memTotal, _ = strconv.ParseFloat(memTotalResults[0].Value, 64) +// meterLoadBalancers bills LoadBalancer services that have an address. +func meterLoadBalancers(svcs []corev1.Service, include func(*corev1.Service) bool, windows []Window, accs map[time.Time]*accumulator, base baseFn) { + for i := range svcs { + svc := &svcs[i] + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer || len(svc.Status.LoadBalancer.Ingress) == 0 || !include(svc) { + continue } - - m.GPUUtilization = append(m.GPUUtilization, GPUUtilizationMetric{ - GPUType: r.Labels["modelName"], - GPUUUID: r.Labels["UUID"], - Utilization: util, - MemoryUsedMB: memUsed, - MemoryTotalMB: memTotal, - PodName: r.Labels["pod"], - PodNamespace: r.Labels["namespace"], - }) - } - - if len(m.GPUUtilization) > 0 { - avgUtil := 0.0 - for _, g := range m.GPUUtilization { - avgUtil += g.Utilization + for _, w := range windows { + ov := usage.Overlap(svc.CreationTimestamp.Time, time.Time{}, w.Start, w.End) + accs[w.End].add(base(usage.MetricLBHours, nil), ov.Hours(), map[string]any{"load_balancers": 1}) } - avgUtil /= float64(len(m.GPUUtilization)) - log.Printf("[metrics] %s: DCGM GPU utilization: %d GPUs, avg=%.1f%%", - namespace, len(m.GPUUtilization), avgUtil) - } -} - -// --- Network egress from Prometheus --- - -func (c *Collector) collectNetworkMetrics(ctx context.Context, namespace string, m *VClusterMetrics) { - if c.promClient == nil { - return - } - - // Query total network transmit bytes (egress) over the last collection interval - // Using rate over 5m to get bytes/sec, then we'll multiply by our interval - txQuery := fmt.Sprintf( - `sum(rate(container_network_transmit_bytes_total{namespace="%s"}[5m])) * 300`, - namespace, - ) - txResults, err := c.promClient.Query(ctx, txQuery) - if err != nil { - log.Printf("[metrics] warning: network tx query failed for %s: %v", namespace, err) - return - } - if len(txResults) > 0 { - m.NetworkTxBytes, _ = strconv.ParseFloat(txResults[0].Value, 64) - } - - // Also get rx for completeness (not billed by default but tracked) - rxQuery := fmt.Sprintf( - `sum(rate(container_network_receive_bytes_total{namespace="%s"}[5m])) * 300`, - namespace, - ) - rxResults, err := c.promClient.Query(ctx, rxQuery) - if err == nil && len(rxResults) > 0 { - m.NetworkRxBytes, _ = strconv.ParseFloat(rxResults[0].Value, 64) - } - - if m.NetworkTxBytes > 0 { - log.Printf("[metrics] %s: Network egress=%.2f MB", - namespace, m.NetworkTxBytes/(1024*1024)) } } -// --- Prometheus client for DCGM and network metrics --- - -type prometheusClient struct { - baseURL string - httpClient *http.Client -} - -func newPrometheusClient(baseURL string) *prometheusClient { - return &prometheusClient{ - baseURL: strings.TrimRight(baseURL, "/"), - httpClient: &http.Client{ - Timeout: 10 * time.Second, - }, - } -} - -type promQueryResponse struct { - Status string `json:"status"` - Data struct { - ResultType string `json:"resultType"` - Result []struct { - Metric map[string]string `json:"metric"` - Value []interface{} `json:"value"` - } `json:"result"` - } `json:"data"` -} - -type promResult struct { - Labels map[string]string - Value string -} - -func (p *prometheusClient) Query(ctx context.Context, query string) ([]promResult, error) { - url := fmt.Sprintf("%s/api/v1/query?query=%s", p.baseURL, query) - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - resp, err := p.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("prometheus query: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var promResp promQueryResponse - if err := json.Unmarshal(body, &promResp); err != nil { - return nil, fmt.Errorf("decode prometheus response: %w", err) - } - - if promResp.Status != "success" { - return nil, fmt.Errorf("prometheus query failed: %s", string(body)) - } - - var results []promResult - for _, r := range promResp.Data.Result { - val := "" - if len(r.Value) >= 2 { - val = fmt.Sprintf("%v", r.Value[1]) +// baseFor returns the event template builder for a tenant cluster. +func (c *Collector) baseFor(t Target) baseFn { + extID := t.Cluster.ExternalID() + return func(metric string, n *nodeInfo) usage.Event { + e := usage.Event{ + Tenant: t.Tenant, Metric: metric, Project: t.Project, ResourceID: extID, + Region: n.regionOr(c.opts.Region, c.opts.RegionFromNode), + Dimensions: map[string]string{usage.DimTenantCluster: extID}, } - results = append(results, promResult{ - Labels: r.Metric, - Value: val, - }) - } - - return results, nil -} - -// --- Helpers --- - -// round rounds a float to n decimal places. -func round(val float64, precision int) float64 { - ratio := math.Pow(10, float64(precision)) - return math.Round(val*ratio) / ratio -} - -// parseQuantity safely parses a Kubernetes resource quantity. -func parseQuantity(s string) float64 { - q, err := resource.ParseQuantity(s) - if err != nil { - return 0 + if t.TenantClass != "" { + e.Dimensions[usage.DimTenantClass] = t.TenantClass + } + if n != nil && n.zone != "" { + e.Dimensions[usage.DimZone] = n.zone + } + return e } - return float64(q.Value()) } - -// Ensure metricsv1beta1 is used (compile check) -var _ *metricsv1beta1.PodMetrics diff --git a/internal/metrics/collector_test.go b/internal/metrics/collector_test.go new file mode 100644 index 0000000..fd79f05 --- /dev/null +++ b/internal/metrics/collector_test.go @@ -0,0 +1,585 @@ +package metrics + +import ( + "context" + "encoding/json" + "math" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" + metricsfake "k8s.io/metrics/pkg/client/clientset/versioned/fake" + + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +var ws = time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) +var we = ws.Add(time.Minute) + +const ns = "team-a" + +func node(name string, labels map[string]string, gpus int64, ready bool, mods ...func(*corev1.Node)) *corev1.Node { + n := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels, CreationTimestamp: metav1.NewTime(ws.Add(-24 * time.Hour))}, + Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("64"), + corev1.ResourceMemory: resource.MustParse("512Gi"), + }, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } + if gpus > 0 { + n.Status.Capacity["nvidia.com/gpu"] = *resource.NewQuantity(gpus, resource.DecimalSI) + } + if !ready { + n.Status.Conditions[0].Status = corev1.ConditionFalse + } + for _, m := range mods { + m(n) + } + return n +} + +type podOpt func(*corev1.Pod) + +func pod(name, nodeName string, opts ...podOpt) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{LabelManagedBy: "gpu", LabelVirtualNamespace: "training"}}, + Spec: corev1.PodSpec{NodeName: nodeName, Containers: []corev1.Container{{Name: "main", Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{}, Limits: corev1.ResourceList{}}}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{{ + Name: "main", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(ws.Add(-time.Hour))}}}}}, + } + for _, o := range opts { + o(p) + } + return p +} + +func withGPU(res string, n int64) podOpt { + return func(p *corev1.Pod) { + p.Spec.Containers[0].Resources.Limits[corev1.ResourceName(res)] = *resource.NewQuantity(n, resource.DecimalSI) + } +} + +func withRequests(cpu, mem string) podOpt { + return func(p *corev1.Pod) { + p.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse(cpu) + p.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse(mem) + } +} + +func startedAt(t time.Time) podOpt { + return func(p *corev1.Pod) { + p.Status.ContainerStatuses[0].State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(t)}} + } +} + +func finished(start, end time.Time) podOpt { + return func(p *corev1.Pod) { + p.Status.Phase = corev1.PodSucceeded + p.Status.ContainerStatuses[0].State = corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + StartedAt: metav1.NewTime(start), FinishedAt: metav1.NewTime(end)}} + } +} + +func labels(kv ...string) podOpt { + return func(p *corev1.Pod) { + for i := 0; i < len(kv); i += 2 { + p.Labels[kv[i]] = kv[i+1] + } + } +} + +func target(name string, ready bool) Target { + return Target{Cluster: discovery.TenantCluster{Name: name, Namespace: ns, Ready: ready, CreatedAt: ws.Add(-48 * time.Hour)}, Tenant: "acme", Project: "research"} +} + +func collect(t *testing.T, opts Options, objs []runtime.Object, windows []Window, targets ...Target) map[time.Time][]usage.Event { + t.Helper() + if opts.Region == "" { + opts.Region = "ap-southeast-2" + } + if opts.GPUResources == nil { + opts.GPUResources = []string{"nvidia.com/gpu"} + } + if opts.Basis == "" { + opts.Basis = "requests" + } + opts.Now = func() time.Time { return we.Add(2 * time.Second) } + c := NewCollector(fake.NewSimpleClientset(objs...), nil, opts) + if len(targets) == 0 { + targets = []Target{target("gpu", true)} + } + out, _, err := c.Collect(context.Background(), targets, windows) + if err != nil { + t.Fatal(err) + } + return out +} + +func latest() []Window { return []Window{{Start: ws, End: we, Latest: true}} } + +// find returns the summed quantity of events matching metric and the +// given dimension/field filters. +func find(evs []usage.Event, metric string, filters ...string) (float64, int) { + var sum float64 + n := 0 +outer: + for _, e := range evs { + if e.Metric != metric { + continue + } + for i := 0; i < len(filters); i += 2 { + k, v := filters[i], filters[i+1] + got := e.Dim(k) + switch k { + case "sku": + got = e.SKU + case "resource_id": + got = e.ResourceID + } + if got != v { + continue outer + } + } + sum += e.Quantity + n++ + } + return sum, n +} + +func near(a, b float64) bool { return math.Abs(a-b) < 1e-8 } + +func TestGPUSecondsPrecisionAndLifecycle(t *testing.T) { + h100 := node("h100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA H100 80GB HBM3", "topology.kubernetes.io/zone": "syd-1a"}, 8, true) + objs := []runtime.Object{h100, + pod("full-window", "h100-1", withGPU("nvidia.com/gpu", 2)), + pod("started-mid", "h100-1", withGPU("nvidia.com/gpu", 1), startedAt(ws.Add(20*time.Second))), + pod("done-mid", "h100-1", withGPU("nvidia.com/gpu", 1), finished(ws.Add(-time.Hour), ws.Add(30*time.Second))), + pod("done-before", "h100-1", withGPU("nvidia.com/gpu", 4), finished(ws.Add(-time.Hour), ws.Add(-time.Second))), + pod("pulling-image", "h100-1", withGPU("nvidia.com/gpu", 8), func(p *corev1.Pod) { + p.Status.ContainerStatuses[0].State = corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ContainerCreating"}} + }), + } + evs := collect(t, Options{}, objs, latest())[we] + got, n := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-H100-80GB-HBM3", usage.DimBillingMode, "shared") + want := (2*60 + 1*40 + 1*30) / 3600.0 // GPU-seconds -> hours; nothing for unstarted or finished pods + if n != 1 || !near(got, usage.Round(want)) { + t.Fatalf("GPU hours = %v (%d events), want %v", got, n, want) + } + for _, e := range evs { + if e.Metric == usage.MetricGPUHours { + if e.Region != "ap-southeast-2" || e.Dim(usage.DimZone) != "syd-1a" || e.Dim(usage.DimCapacityType) != usage.CapacityOnDemand || + e.Tenant != "acme" || e.Project != "research" || e.Dim(usage.DimTenantCluster) != "vcluster-team-a-gpu" { + t.Fatalf("GPU event attribution: %+v", e) + } + } + } +} + +func TestFractionalGPUs(t *testing.T) { + a100 := node("a100-mig", map[string]string{"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-80GB"}, 0, true) + shared := node("l40s-ts", map[string]string{"nvidia.com/gpu.product": "NVIDIA-L40S", "nvidia.com/gpu.replicas": "4", "nvidia.com/gpu.sharing-strategy": "time-slicing"}, 4, true) + objs := []runtime.Object{a100, shared, + pod("mig", "a100-mig", withGPU("nvidia.com/mig-1g.10gb", 2)), + pod("big-mig", "a100-mig", withGPU("nvidia.com/mig-3g.40gb", 1)), + pod("ts", "l40s-ts", withGPU("nvidia.com/gpu", 1)), + } + evs := collect(t, Options{}, objs, latest())[we] + if got, _ := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-A100-SXM4-80GB-mig-1g.10gb", usage.DimGPUProfile, "mig-1g.10gb"); !near(got, usage.Round(2*(1.0/7)/60)) { + t.Errorf("1g MIG GPU-equivalent hours = %v", got) + } + if got, _ := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-A100-SXM4-80GB-mig-3g.40gb"); !near(got, usage.Round((3.0/7)/60)) { + t.Errorf("3g MIG GPU-equivalent hours = %v", got) + } + if got, _ := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-L40S-timeslice-4", usage.DimGPUProfile, "timeslice-4"); !near(got, usage.Round(0.25/60)) { + t.Errorf("time-sliced GPU hours = %v", got) + } +} + +func TestUnhealthyNodeTimeIsCreditedNotBilled(t *testing.T) { + down := node("h100-down", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, false, func(n *corev1.Node) { + n.Status.Conditions[0].LastTransitionTime = metav1.NewTime(ws.Add(15 * time.Second)) + }) + tainted := node("h100-xid", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true, func(n *corev1.Node) { + n.Spec.Taints = []corev1.Taint{{Key: "nvidia.com/gpu.unhealthy", Effect: corev1.TaintEffectNoSchedule}} // no timeAdded: whole window + }) + objs := []runtime.Object{down, tainted, + pod("a", "h100-down", withGPU("nvidia.com/gpu", 4)), + pod("b", "h100-xid", withGPU("nvidia.com/gpu", 2)), + } + opts := Options{UnhealthyTaints: []string{"node.kubernetes.io/not-ready", "nvidia.com/gpu.unhealthy"}} + evs := collect(t, opts, objs, latest())[we] + if got, _ := find(evs, usage.MetricGPUHours); !near(got, usage.Round(4*15/3600.0)) { + t.Errorf("billable GPU hours = %v, want only the 15s before the node went NotReady", got) + } + dt, _ := find(evs, usage.MetricGPUDowntimeHours) + if !near(dt, usage.Round(4*45/3600.0)+usage.Round(2*60/3600.0)) { + t.Errorf("downtime GPU hours = %v", dt) + } +} + +func TestControlPlaneAndOtherTenantClustersExcluded(t *testing.T) { + n := node("cpu-1", nil, 0, true) + objs := []runtime.Object{n, + pod("vcluster-0", "cpu-1", withRequests("2", "4Gi"), labels("app", "vcluster", "release", "gpu"), func(p *corev1.Pod) { delete(p.Labels, LabelManagedBy) }), + pod("mine", "cpu-1", withRequests("1", "2Gi")), + pod("neighbour", "cpu-1", withRequests("8", "16Gi"), labels(LabelManagedBy, "other")), + } + evs := collect(t, Options{}, objs, latest())[we] + if got, _ := find(evs, usage.MetricCPUCoreHours); !near(got, usage.Round(1.0/60)) { + t.Fatalf("CPU core-hours = %v, want only the tenant's own synced pod", got) + } + if got, _ := find(evs, usage.MetricMemoryGBHours); !near(got, usage.Round(2.0/60)) { + t.Fatalf("memory GiB-hours = %v", got) + } + evs = collect(t, Options{MeterControlPlane: true}, objs, latest())[we] + if got, _ := find(evs, usage.MetricCPUCoreHours); !near(got, usage.Round(3.0/60)) { + t.Fatalf("with METER_CONTROL_PLANE CPU core-hours = %v", got) + } +} + +func TestDedicatedNodesBilledWholeWithoutDoubleCounting(t *testing.T) { + dedicated := node("bm-01", map[string]string{"vcluster.loft.sh/managed-by": ns, "nvidia.com/gpu.product": "NVIDIA-H100-80GB-HBM3", + "node.kubernetes.io/instance-type": "bm.gpu.h100.8", "karpenter.sh/capacity-type": "reserved"}, 8, true) + objs := []runtime.Object{dedicated, + pod("on-dedicated", "bm-01", withGPU("nvidia.com/gpu", 8), withRequests("32", "256Gi")), + } + evs := collect(t, Options{}, objs, latest())[we] + if got, n := find(evs, usage.MetricPrivateNodeHours, "resource_id", "bm-01", "sku", "bm.gpu.h100.8", usage.DimCapacityType, usage.CapacityReserved); n != 1 || !near(got, usage.Round(1.0/60)) { + t.Fatalf("node hours = %v (%d)", got, n) + } + if got, _ := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "dedicated_node"); !near(got, usage.Round(8.0/60)) { + t.Fatalf("dedicated GPU hours = %v", got) + } + if _, n := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "shared"); n != 0 { + t.Fatal("pod on the tenant's dedicated node was billed again as shared usage") + } + if got, _ := find(evs, usage.MetricCPUCoreHours, usage.DimBillingMode, "dedicated_node"); !near(got, usage.Round(64.0/60)) { + t.Fatalf("dedicated CPU capacity hours = %v", got) + } +} + +func TestStorageLoadBalancersAndControlPlaneHours(t *testing.T) { + fast := "fast-nvme" + objs := []runtime.Object{ + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "data", Namespace: ns, Labels: map[string]string{LabelManagedBy: "gpu"}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &fast}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound, Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("100Gi")}}}, + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "pending", Namespace: ns, Labels: map[string]string{LabelManagedBy: "gpu"}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}}, + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "data-gpu-0", Namespace: ns, Labels: map[string]string{"app": "vcluster", "release": "gpu"}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound, Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("5Gi")}}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "ingress", Namespace: ns, Labels: map[string]string{LabelManagedBy: "gpu"}}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, Status: corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.10"}}}}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "no-ip-yet", Namespace: ns, Labels: map[string]string{LabelManagedBy: "gpu"}}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}}, + } + evs := collect(t, Options{}, objs, latest())[we] + if got, _ := find(evs, usage.MetricStorageGBHours, "sku", "fast-nvme"); !near(got, usage.Round(100.0/60)) { + t.Errorf("storage GiB-hours = %v", got) + } + if _, n := find(evs, usage.MetricStorageGBHours); n != 1 { + t.Errorf("storage events = %d (pending and control-plane PVCs must be skipped)", n) + } + if got, _ := find(evs, usage.MetricLBHours); !near(got, usage.Round(1.0/60)) { + t.Errorf("LB hours = %v (only LBs with an address bill)", got) + } + if got, _ := find(evs, usage.MetricInstanceHours); !near(got, usage.Round(1.0/60)) { + t.Errorf("instance hours = %v", got) + } + evs = collect(t, Options{}, objs, latest(), target("gpu", false))[we] + if _, n := find(evs, usage.MetricInstanceHours); n != 0 { + t.Error("a tenant cluster that is not ready (provisioning or asleep) must not bill instance hours") + } +} + +func TestSpotCapacityAndNamespaceAttribution(t *testing.T) { + spot := node("spot-1", map[string]string{"cloud.google.com/gke-spot": "true", "nvidia.com/gpu.product": "NVIDIA-L4"}, 1, true) + pre := node("dev-1", map[string]string{"vbilling.vcluster.com/capacity-type": "preemptible", "nvidia.com/gpu.product": "NVIDIA-L4"}, 1, true) + objs := []runtime.Object{spot, pre, + pod("a", "spot-1", withGPU("nvidia.com/gpu", 1)), + pod("b", "dev-1", withGPU("nvidia.com/gpu", 1), labels(LabelVirtualNamespace, "inference")), + } + evs := collect(t, Options{MeterByNamespace: true, CapacityTypeLabel: "vbilling.vcluster.com/capacity-type"}, objs, latest())[we] + if _, n := find(evs, usage.MetricGPUHours, usage.DimCapacityType, usage.CapacitySpot, usage.DimNamespace, "training"); n != 1 { + t.Error("spot GPU event with virtual namespace missing") + } + if _, n := find(evs, usage.MetricGPUHours, usage.DimCapacityType, usage.CapacityPreemptible, usage.DimNamespace, "inference"); n != 1 { + t.Error("preemptible GPU event missing") + } +} + +func TestBackfillAndDeterminism(t *testing.T) { + n := node("h100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true) + pm := &metricsv1beta1.PodMetrics{ObjectMeta: metav1.ObjectMeta{Name: "train", Namespace: ns}, + Containers: []metricsv1beta1.ContainerMetrics{{Name: "main", Usage: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3"), corev1.ResourceMemory: resource.MustParse("8Gi")}}}} + objs := []runtime.Object{n, pod("train", "h100-1", withGPU("nvidia.com/gpu", 1), withRequests("1", "1Gi"))} + windows := []Window{{Start: ws.Add(-2 * time.Minute), End: ws.Add(-time.Minute)}, {Start: ws.Add(-time.Minute), End: ws}, {Start: ws, End: we, Latest: true}} + + run := func() map[time.Time][]usage.Event { + mc := metricsfake.NewSimpleClientset() + // The tracker would guess "podmetricses"; the client lists "pods". + if err := mc.Tracker().Create(schema.GroupVersionResource{Group: "metrics.k8s.io", Version: "v1beta1", Resource: "pods"}, pm, ns); err != nil { + t.Fatal(err) + } + c := NewCollector(fake.NewSimpleClientset(objs...), mc, Options{Region: "r", GPUResources: []string{"nvidia.com/gpu"}, Basis: "usage", + Now: func() time.Time { return we.Add(time.Second) }}) + out, _, err := c.Collect(context.Background(), []Target{target("gpu", true)}, windows) + if err != nil { + t.Fatal(err) + } + return out + } + out := run() + for i, w := range windows { + evs := out[w.End] + if got, _ := find(evs, usage.MetricGPUHours); !near(got, usage.Round(1.0/60)) { + t.Errorf("window %d GPU hours = %v", i, got) + } + _, cpuEvents := find(evs, usage.MetricCPUCoreHours) + if w.Latest && cpuEvents != 1 { + t.Error("usage-based CPU missing from the live window") + } + if !w.Latest && cpuEvents != 0 { + t.Error("usage-based CPU must not be invented for backfilled windows") + } + for _, e := range evs { + if !w.Latest && e.Properties["backfilled"] != true { + t.Errorf("backfilled event not flagged: %+v", e) + } + } + } + if got, _ := find(out[we], usage.MetricCPUCoreHours); !near(got, usage.Round(3.0/60)) { + t.Errorf("usage CPU core-hours = %v, want metrics-server value", got) + } + // Every event is stamped with its own window and IDs never repeat across windows. + seen := map[string]bool{} + for _, w := range windows { + for _, e := range out[w.End] { + if !e.WindowStart.Equal(w.Start) || !e.WindowEnd.Equal(w.End) { + t.Fatalf("event %s/%s has window [%s, %s), want [%s, %s)", e.Tenant, e.Metric, e.WindowStart, e.WindowEnd, w.Start, w.End) + } + if seen[e.ID] { + t.Fatalf("event ID %s repeats across windows", e.ID) + } + seen[e.ID] = true + } + } + again := run() + for _, w := range windows { + a, b := out[w.End], again[w.End] + if len(a) != len(b) { + t.Fatal("non-deterministic event count") + } + for i := range a { + if a[i].ID != b[i].ID { + t.Fatalf("event IDs differ between identical runs: %s vs %s", a[i].ID, b[i].ID) + } + } + } +} + +func TestPrometheusEgressAndUtilizationAtWindowEnd(t *testing.T) { + var queries []string + prom := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("query") + queries = append(queries, q) + if r.URL.Query().Get("time") != strconv.FormatInt(we.Unix(), 10) { + t.Errorf("query not evaluated at window end: %s", r.URL.RawQuery) + } + val := "0" + labels := map[string]string{} + switch { + case strings.Contains(q, "container_network_transmit_bytes_total"): + val = strconv.Itoa(3 * gib) + case strings.Contains(q, "avg_over_time(DCGM_FI_DEV_GPU_UTIL"): + val, labels = "75", map[string]string{"modelName": "NVIDIA H100 80GB HBM3"} + case strings.Contains(q, "count by"): + val, labels = "2", map[string]string{"modelName": "NVIDIA H100 80GB HBM3"} + } + json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": map[string]any{"resultType": "vector", + "result": []map[string]any{{"metric": labels, "value": []any{float64(we.Unix()), val}}}}}) + })) + defer prom.Close() + evs := collect(t, Options{PrometheusURL: prom.URL}, nil, latest())[we] + if got, _ := find(evs, usage.MetricNetworkEgressGB); !near(got, 3) { + t.Errorf("egress GiB = %v", got) + } + if got, _ := find(evs, usage.MetricGPUUtilization, usage.DimGPUType, "NVIDIA-H100-80GB-HBM3"); !near(got, usage.Round(75.0/60)) { + t.Errorf("utilization hours = %v", got) + } + if !strings.Contains(queries[0], `increase(container_network_transmit_bytes_total{namespace="team-a"}[60s])`) { + t.Errorf("egress query = %s", queries[0]) + } +} + +func TestMigFraction(t *testing.T) { + cases := []struct { + profile, gpu string + want float64 + }{{"1g.10gb", "NVIDIA-H100", 1.0 / 7}, {"7g.80gb", "NVIDIA-H100", 1}, {"1g.6gb", "NVIDIA-A30", 0.25}, {"2g.20gb+me", "NVIDIA-A100", 2.0 / 7}, {"weird", "x", 1}} + for _, c := range cases { + if got := migFraction(c.profile, c.gpu); !near(got, c.want) { + t.Errorf("migFraction(%s,%s) = %v, want %v", c.profile, c.gpu, got, c.want) + } + } +} + +func TestMIGSingleStrategyGKEPartitionsAndSharedProducts(t *testing.T) { + single := node("a100-single", map[string]string{"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-40GB-MIG-1g.5gb", "nvidia.com/mig.strategy": "single"}, 56, true) + gke := node("gke-a100", map[string]string{"cloud.google.com/gke-accelerator": "nvidia-tesla-a100", "cloud.google.com/gke-gpu-partition-size": "3g.20gb"}, 16, true) + shared := node("t4-shared", map[string]string{"nvidia.com/gpu.product": "Tesla-T4-SHARED", "nvidia.com/gpu.replicas": "4", "nvidia.com/gpu.sharing-strategy": "time-slicing"}, 16, true) + objs := []runtime.Object{single, gke, shared, + pod("slices", "a100-single", withGPU("nvidia.com/gpu", 2)), + pod("part", "gke-a100", withGPU("nvidia.com/gpu", 1)), + pod("ts", "t4-shared", withGPU("nvidia.com/gpu", 1)), + } + evs := collect(t, Options{}, objs, latest())[we] + if got, n := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-A100-SXM4-40GB-mig-1g.5gb", usage.DimGPUType, "NVIDIA-A100-SXM4-40GB", usage.DimGPUProfile, "mig-1g.5gb"); n != 1 || !near(got, usage.Round(2*(1.0/7)/60)) { + t.Errorf("MIG single strategy: GPU-equivalent hours = %v (%d events), want two 1/7 slices", got, n) + } + if got, _ := find(evs, usage.MetricGPUHours, "sku", "nvidia-tesla-a100-mig-3g.20gb", usage.DimGPUProfile, "mig-3g.20gb"); !near(got, usage.Round((3.0/7)/60)) { + t.Errorf("GKE GPU partition: GPU-equivalent hours = %v", got) + } + if got, _ := find(evs, usage.MetricGPUHours, "sku", "Tesla-T4-timeslice-4", usage.DimGPUType, "Tesla-T4"); !near(got, usage.Round(0.25/60)) { + t.Errorf("-SHARED product label: time-sliced GPU hours = %v", got) + } +} + +func TestDedicatedNodesBilledInWholeGPUs(t *testing.T) { + dedicated := func(extra map[string]string) map[string]string { + l := map[string]string{"vcluster.loft.sh/managed-by": ns} + for k, v := range extra { + l[k] = v + } + return l + } + capacity := func(res string, n int64) func(*corev1.Node) { + return func(nd *corev1.Node) { + nd.Status.Capacity[corev1.ResourceName(res)] = *resource.NewQuantity(n, resource.DecimalSI) + } + } + cases := []struct { + name string + node *corev1.Node + want float64 + }{ + {"time-sliced, 8 GPUs x 4 replicas", node("ts", dedicated(map[string]string{"nvidia.com/gpu.product": "NVIDIA-L40S-SHARED", "nvidia.com/gpu.replicas": "4"}), 32, true), 8}, + {"MIG single, 8 GPUs x 7 slices", node("mig", dedicated(map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100-80GB-HBM3-MIG-1g.10gb"}), 56, true), 8}, + {"MIG mixed, 6 whole + 2 carved", node("mixed", dedicated(map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100-80GB-HBM3"}), 6, true, + capacity("nvidia.com/mig-3g.40gb", 4), capacity("nvidia.com/mig-1g.10gb", 2)), 8}, + {"renamed shared resource", node("renamed", dedicated(map[string]string{"nvidia.com/gpu.product": "NVIDIA-L4", "nvidia.com/gpu.replicas": "2"}), 0, true, + capacity("nvidia.com/gpu.shared", 8)), 4}, + } + for _, c := range cases { + evs := collect(t, Options{}, []runtime.Object{c.node}, latest())[we] + if got, n := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "dedicated_node"); n != 1 || !near(got, usage.Round(c.want/60)) { + t.Errorf("%s: dedicated GPU hours = %v (%d events), want %v", c.name, got, n, usage.Round(c.want/60)) + } + } +} + +// promServer answers instant queries through respond and records them. +func promServer(t *testing.T, check func(*http.Request) bool, respond func(q string) (string, map[string]string)) (*httptest.Server, *[]string) { + var queries []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if check != nil && !check(r) { + w.WriteHeader(http.StatusUnauthorized) + return + } + q := r.URL.Query().Get("query") + queries = append(queries, q) + val, labels := respond(q) + result := []map[string]any{} + if val != "" { + result = append(result, map[string]any{"metric": labels, "value": []any{float64(we.Unix()), val}}) + } + json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": map[string]any{"resultType": "vector", "result": result}}) + })) + t.Cleanup(srv.Close) + return srv, &queries +} + +func TestPrometheusAuthAndExportedNamespaceLabels(t *testing.T) { + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte("s3cret\n"), 0o600); err != nil { + t.Fatal(err) + } + authed := func(r *http.Request) bool { + return r.Header.Get("Authorization") == "Bearer s3cret" && r.Header.Get("X-Scope-OrgID") == "tenant-1" + } + // kube-prometheus-stack renames the exporter's namespace label. + prom, queries := promServer(t, authed, func(q string) (string, map[string]string) { + switch { + case strings.Contains(q, "avg_over_time(DCGM_FI_DEV_GPU_UTIL") && strings.Contains(q, `exported_namespace="team-a"`): + return "50", map[string]string{"modelName": "NVIDIA H100 80GB HBM3"} + case strings.Contains(q, "count by (modelName)"): + return "4", map[string]string{"modelName": "NVIDIA H100 80GB HBM3"} + } + return "", nil + }) + opts := Options{PrometheusURL: prom.URL, PromHeaders: map[string]string{"X-Scope-OrgID": "tenant-1"}, PromTokenFile: tokenFile} + evs := collect(t, opts, nil, latest())[we] + if len(*queries) == 0 { + t.Fatal("no authenticated queries reached Prometheus") + } + got, n := find(evs, usage.MetricGPUUtilization, usage.DimGPUType, "NVIDIA-H100-80GB-HBM3") + if n != 1 || !near(got, usage.Round(50.0/60)) { + t.Fatalf("utilization hours = %v (%d events)", got, n) + } + for _, e := range evs { + if e.Metric == usage.MetricGPUUtilization && e.Properties["gpu_devices"] != 4.0 { + t.Errorf("gpu_devices = %v", e.Properties["gpu_devices"]) + } + } + + bad := newPrometheusClient(prom.URL, nil, "") + if _, err := bad.Query(context.Background(), "up", we); err == nil || !strings.Contains(err.Error(), "HTTP 401") { + t.Errorf("unauthenticated query error = %v", err) + } + basic, _ := promServer(t, func(r *http.Request) bool { u, p, ok := r.BasicAuth(); return ok && u == "vb" && p == "pw" }, func(string) (string, map[string]string) { return "1", nil }) + withCreds := strings.Replace(basic.URL, "http://", "http://vb:pw@", 1) + if _, err := newPrometheusClient(withCreds, nil, "").Query(context.Background(), "up", we); err != nil { + t.Errorf("basic auth from the URL: %v", err) + } +} + +func TestCustomAndDisabledGPUUtilQuery(t *testing.T) { + prom, queries := promServer(t, nil, func(q string) (string, map[string]string) { + if strings.Contains(q, "amd_gpu_busy") { + return "30", map[string]string{"gpu_type": "AMD Instinct MI300X"} + } + return "", nil + }) + custom := `avg by (gpu_type) (avg_over_time(amd_gpu_busy{namespace="{{namespace}}"}[{{window}}]))` + evs := collect(t, Options{PrometheusURL: prom.URL, GPUUtilQuery: custom}, nil, latest())[we] + if got, _ := find(evs, usage.MetricGPUUtilization, usage.DimGPUType, "AMD-Instinct-MI300X"); !near(got, usage.Round(30.0/60)) { + t.Errorf("custom utilization hours = %v", got) + } + for _, q := range *queries { + if strings.Contains(q, "DCGM_FI_DEV_GPU_UTIL") { + t.Errorf("default DCGM query ran alongside a custom one: %s", q) + } + } + *queries = nil + collect(t, Options{PrometheusURL: prom.URL, GPUUtilQuery: "none"}, nil, latest()) + if len(*queries) != 1 || !strings.Contains((*queries)[0], "container_network_transmit_bytes_total") { + t.Errorf("with GPU_UTIL_QUERY=none only egress should run, got %q", *queries) + } +} diff --git a/internal/metrics/dra.go b/internal/metrics/dra.go new file mode 100644 index 0000000..a7a6d3d --- /dev/null +++ b/internal/metrics/dra.go @@ -0,0 +1,185 @@ +package metrics + +import ( + "context" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" +) + +// Dynamic Resource Allocation (resource.k8s.io): GPUs allocated through +// ResourceClaims never appear in a container's resource limits. A claim's +// status names the allocated devices and reservedFor names the pods using +// them; ResourceSlices describe each device (model, MIG profile) and which +// node it is on. Read through the dynamic client so every API version from +// v1beta1 (Kubernetes 1.32) to v1 works. + +var draVersions = []string{"v1", "v1beta2", "v1beta1"} + +// DefaultDRAGPUDrivers are the DRA drivers whose devices are GPUs. +var DefaultDRAGPUDrivers = []string{"gpu.nvidia.com", "gpu.amd.com"} + +type draDevice struct { + gpuType string + profile string // MIG profile, "" for a whole GPU + node string +} + +// draIndex is one collection's view of DRA GPU allocations. +type draIndex struct { + byPod map[types.UID][]gpuAlloc + gpuType map[types.UID]string + nodeGPUs map[string]float64 // whole GPUs a node offers through DRA + nodeType map[string]string +} + +func (x *draIndex) allocs(uid types.UID) []gpuAlloc { + if x == nil { + return nil + } + return x.byPod[uid] +} + +// draFor builds the index for a namespace ("" = all), or nil when the +// cluster has no DRA API or no client is configured. +func (c *Collector) draFor(ctx context.Context, dyn dynamic.Interface, namespace string) *draIndex { + if dyn == nil { + return nil + } + slices, version := listDRA(ctx, dyn, "resourceslices", "") + if version == "" { + return nil + } + gpuDrivers := map[string]bool{} + for _, d := range c.opts.DRAGPUDrivers { + gpuDrivers[d] = true + } + devices := map[string]draDevice{} + x := &draIndex{byPod: map[types.UID][]gpuAlloc{}, gpuType: map[types.UID]string{}, nodeGPUs: map[string]float64{}, nodeType: map[string]string{}} + for _, s := range slices { + driver, _, _ := unstructured.NestedString(s.Object, "spec", "driver") + if !gpuDrivers[driver] { + continue + } + pool, _, _ := unstructured.NestedString(s.Object, "spec", "pool", "name") + node, _, _ := unstructured.NestedString(s.Object, "spec", "nodeName") + list, _, _ := unstructured.NestedSlice(s.Object, "spec", "devices") + for _, raw := range list { + dev, ok := raw.(map[string]any) + if !ok { + continue + } + name, _ := dev["name"].(string) + attrs := deviceAttributes(dev) + d := draDevice{gpuType: normalizeGPUType(attrs["productName"]), node: node} + if strings.EqualFold(attrs["type"], "mig") { + d.profile = attrs["profile"] + } else if node != "" { + x.nodeGPUs[node]++ + if d.gpuType != "" { + x.nodeType[node] = d.gpuType + } + } + devices[driver+"/"+pool+"/"+name] = d + } + } + claims, _ := listDRAVersion(ctx, dyn, "resourceclaims", namespace, version) + for _, cl := range claims { + results, _, _ := unstructured.NestedSlice(cl.Object, "status", "allocation", "devices", "results") + consumers, _, _ := unstructured.NestedSlice(cl.Object, "status", "reservedFor") + var pods []types.UID + for _, raw := range consumers { + if r, ok := raw.(map[string]any); ok && r["resource"] == "pods" { + if uid, ok := r["uid"].(string); ok { + pods = append(pods, types.UID(uid)) + } + } + } + if len(pods) == 0 { + continue + } + share := 1 / float64(len(pods)) // a claim shared by several pods splits its devices + for _, raw := range results { + r, ok := raw.(map[string]any) + if !ok { + continue + } + driver, _ := r["driver"].(string) + if !gpuDrivers[driver] { + continue + } + pool, _ := r["pool"].(string) + name, _ := r["device"].(string) + d := devices[driver+"/"+pool+"/"+name] + a := gpuAlloc{count: share, fraction: 1, profile: "full"} + if d.profile != "" { + a.fraction, a.profile = migFraction(d.profile, d.gpuType), "mig-"+d.profile + } + for _, uid := range pods { + x.byPod[uid] = append(x.byPod[uid], a) + if d.gpuType != "" { + x.gpuType[uid] = d.gpuType + } + } + } + } + return x +} + +// deviceAttributes flattens a device's attributes (v1beta1 nests them under +// "basic"; later versions do not) to name -> value, dropping the domain +// prefix of qualified names ("gpu.nvidia.com/productName"). +func deviceAttributes(dev map[string]any) map[string]string { + attrs, ok := dev["attributes"].(map[string]any) + if !ok { + if basic, ok := dev["basic"].(map[string]any); ok { + attrs, _ = basic["attributes"].(map[string]any) + } + } + out := map[string]string{} + for k, raw := range attrs { + if i := strings.LastIndex(k, "/"); i >= 0 { + k = k[i+1:] + } + v, ok := raw.(map[string]any) + if !ok { + continue + } + for _, field := range []string{"string", "version"} { + if s, ok := v[field].(string); ok { + out[k] = s + } + } + } + return out +} + +func listDRA(ctx context.Context, dyn dynamic.Interface, resource, namespace string) ([]unstructured.Unstructured, string) { + for _, v := range draVersions { + if items, ok := listDRAVersion(ctx, dyn, resource, namespace, v); ok { + return items, v + } + } + return nil, "" +} + +func listDRAVersion(ctx context.Context, dyn dynamic.Interface, resource, namespace, version string) ([]unstructured.Unstructured, bool) { + gvr := schema.GroupVersionResource{Group: "resource.k8s.io", Version: version, Resource: resource} + var ( + list *unstructured.UnstructuredList + err error + ) + if namespace == "" { + list, err = dyn.Resource(gvr).List(ctx, metav1.ListOptions{}) + } else { + list, err = dyn.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{}) + } + if err != nil { + return nil, false + } + return list.Items, true +} diff --git a/internal/metrics/dra_test.go b/internal/metrics/dra_test.go new file mode 100644 index 0000000..bfe8767 --- /dev/null +++ b/internal/metrics/dra_test.go @@ -0,0 +1,81 @@ +package metrics + +import ( + "context" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +func draObj(kind, ns, name string, spec, status map[string]any) *unstructured.Unstructured { + o := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "resource.k8s.io/v1", "kind": kind, + "metadata": map[string]any{"name": name}, "spec": spec}} + if ns != "" { + o.SetNamespace(ns) + } + if status != nil { + o.Object["status"] = status + } + return o +} + +func gpuDevice(name, typ, profile string) map[string]any { + attrs := map[string]any{"type": map[string]any{"string": typ}, "productName": map[string]any{"string": "NVIDIA H200"}} + if profile != "" { + attrs["gpu.nvidia.com/profile"] = map[string]any{"string": profile} // qualified name + } + return map[string]any{"name": name, "attributes": attrs} +} + +func claim(name, device, podUID string) *unstructured.Unstructured { + return draObj("ResourceClaim", ns, name, map[string]any{}, map[string]any{ + "allocation": map[string]any{"devices": map[string]any{"results": []any{map[string]any{"request": "gpu", "driver": "gpu.nvidia.com", "pool": "dra-node-1", "device": device}}}}, + "reservedFor": []any{map[string]any{"resource": "pods", "name": name, "uid": podUID}}, + }) +} + +func TestDRAClaimedGPUs(t *testing.T) { + slice := draObj("ResourceSlice", "", "dra-node-1-gpus", map[string]any{"driver": "gpu.nvidia.com", "nodeName": "dra-node-1", + "pool": map[string]any{"name": "dra-node-1", "generation": int64(1), "resourceSliceCount": int64(1)}, + "devices": []any{gpuDevice("gpu-0", "gpu", ""), gpuDevice("gpu-1", "gpu", ""), gpuDevice("mig-1g", "mig", "1g.18gb")}}, nil) + dedicatedSlice := draObj("ResourceSlice", "", "dra-ded-gpus", map[string]any{"driver": "gpu.nvidia.com", "nodeName": "dra-ded", + "pool": map[string]any{"name": "dra-ded", "generation": int64(1), "resourceSliceCount": int64(1)}, + "devices": []any{gpuDevice("gpu-0", "gpu", ""), gpuDevice("gpu-1", "gpu", "")}}, nil) + gvrs := map[schema.GroupVersionResource]string{ + {Group: "resource.k8s.io", Version: "v1", Resource: "resourceclaims"}: "ResourceClaimList", + {Group: "resource.k8s.io", Version: "v1", Resource: "resourceslices"}: "ResourceSliceList", + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrs, + slice, dedicatedSlice, claim("trainer", "gpu-0", "uid-trainer"), claim("small", "mig-1g", "uid-small")) + + trainer := pod("trainer", "dra-node-1") + trainer.UID = types.UID("uid-trainer") + small := pod("small", "dra-node-1") + small.UID = types.UID("uid-small") + host := []runtime.Object{node("dra-node-1", nil, 0, true), node("dra-ded", map[string]string{"vcluster.loft.sh/managed-by": ns}, 0, true), trainer, small} + opts := Options{GPUResources: []string{"nvidia.com/gpu"}, Basis: "requests", Region: "r", Now: func() time.Time { return we }} + c := NewCollector(fake.NewSimpleClientset(host...), nil, opts) + c.UseDynamic(dyn) + out, _, err := c.Collect(context.Background(), []Target{target("gpu", true)}, latest()) + if err != nil { + t.Fatal(err) + } + evs := out[we] + if got, _ := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-H200", usage.DimBillingMode, "shared"); !near(got, usage.Round(1.0/60)) { + t.Errorf("DRA whole-GPU hours = %v", got) + } + if got, _ := find(evs, usage.MetricGPUHours, "sku", "NVIDIA-H200-mig-1g.18gb"); !near(got, usage.Round((1.0/7)/60)) { + t.Errorf("DRA MIG hours = %v", got) + } + if got, _ := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "dedicated_node", "sku", "NVIDIA-H200"); !near(got, usage.Round(2.0/60)) { + t.Errorf("dedicated node with DRA-only GPUs: GPU hours = %v, want its 2 GPUs", got) + } +} diff --git a/internal/metrics/graveyard.go b/internal/metrics/graveyard.go new file mode 100644 index 0000000..3fdbcea --- /dev/null +++ b/internal/metrics/graveyard.go @@ -0,0 +1,193 @@ +package metrics + +import ( + "context" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +// Graveyard keeps pods and nodes deleted between two collections. Windows +// are metered from a list taken when the window closes, so without it a pod +// deleted before then (a short job, a pod evicted with its node, an Auto +// Node scaled down) would lose its last partial window, or all of its life. +// Deleted objects are kept for the retention period, then dropped. Deletions +// while vBilling itself is down are not seen. +// +// A deletion noticed only after a watch reconnects (the informer reports it +// with an unknown final state) is dated to the last collection that still +// listed the object, never later: an outage can under-bill, not over-bill. +type Graveyard struct { + mu sync.Mutex + retention time.Duration + now func() time.Time + pods map[types.UID]deleted[*corev1.Pod] + nodes map[string]deleted[*corev1.Node] + seenPods map[types.UID]time.Time + seenNodes map[string]time.Time +} + +type deleted[T any] struct { + obj T + at time.Time +} + +func NewGraveyard(retention time.Duration, now func() time.Time) *Graveyard { + if now == nil { + now = time.Now + } + return &Graveyard{retention: retention, now: now, + pods: map[types.UID]deleted[*corev1.Pod]{}, nodes: map[string]deleted[*corev1.Node]{}, + seenPods: map[types.UID]time.Time{}, seenNodes: map[string]time.Time{}} +} + +// Watch records pod and node deletions seen by informers on client until ctx +// ends. podSelector narrows the pods watched (for example the label vCluster +// puts on synced pods); watchPods=false watches nodes only. It returns once +// the informers have synced. +func (g *Graveyard) Watch(ctx context.Context, client kubernetes.Interface, podSelector string, watchPods, watchNodes bool) { + var synced []cache.InformerSynced + if watchPods { + podFactory := informers.NewSharedInformerFactoryWithOptions(client, 0, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { o.LabelSelector = podSelector })) + inf := podFactory.Core().V1().Pods().Informer() + _, _ = inf.AddEventHandler(cache.ResourceEventHandlerFuncs{DeleteFunc: func(obj any) { + obj, uncertain := finalState(obj) + if p, ok := obj.(*corev1.Pod); ok { + g.recordPod(p, g.now(), uncertain) + } + }}) + synced = append(synced, inf.HasSynced) + podFactory.Start(ctx.Done()) + } + if watchNodes { + factory := informers.NewSharedInformerFactory(client, 0) + inf := factory.Core().V1().Nodes().Informer() + _, _ = inf.AddEventHandler(cache.ResourceEventHandlerFuncs{DeleteFunc: func(obj any) { + obj, uncertain := finalState(obj) + if n, ok := obj.(*corev1.Node); ok { + g.recordNode(n, g.now(), uncertain) + } + }}) + synced = append(synced, inf.HasSynced) + factory.Start(ctx.Done()) + } + cache.WaitForCacheSync(ctx.Done(), synced...) +} + +func finalState(obj any) (any, bool) { + if tomb, ok := obj.(cache.DeletedFinalStateUnknown); ok { + return tomb.Obj, true + } + return obj, false +} + +// RecordPod remembers a deleted pod and when it was deleted. +func (g *Graveyard) RecordPod(p *corev1.Pod, at time.Time) { g.recordPod(p, at, false) } + +// RecordNode remembers a deleted node and when it was deleted. +func (g *Graveyard) RecordNode(n *corev1.Node, at time.Time) { g.recordNode(n, at, false) } + +func (g *Graveyard) recordPod(p *corev1.Pod, at time.Time, uncertain bool) { + g.mu.Lock() + defer g.mu.Unlock() + if seen, ok := g.seenPods[p.UID]; uncertain && ok && seen.Before(at) { + at = seen + } + g.pods[p.UID] = deleted[*corev1.Pod]{obj: p, at: at} +} + +func (g *Graveyard) recordNode(n *corev1.Node, at time.Time, uncertain bool) { + g.mu.Lock() + defer g.mu.Unlock() + if seen, ok := g.seenNodes[n.Name]; uncertain && ok && seen.Before(at) { + at = seen + } + g.nodes[n.Name] = deleted[*corev1.Node]{obj: n, at: at} +} + +// SeenNodes records that a collection listed these nodes at time at. +func (g *Graveyard) SeenNodes(nodes []corev1.Node, at time.Time) { + if g == nil { + return + } + g.mu.Lock() + defer g.mu.Unlock() + for i := range nodes { + g.seenNodes[nodes[i].Name] = at + } +} + +// SeenPods records that a collection listed these pods at time at. +func (g *Graveyard) SeenPods(pods []corev1.Pod, at time.Time) { + if g == nil { + return + } + g.mu.Lock() + defer g.mu.Unlock() + for i := range pods { + g.seenPods[pods[i].UID] = at + } +} + +// Pods returns deleted pods of a namespace ("" = all) still in retention. +func (g *Graveyard) Pods(namespace string) []deleted[*corev1.Pod] { + if g == nil { + return nil + } + g.mu.Lock() + defer g.mu.Unlock() + g.pruneLocked() + var out []deleted[*corev1.Pod] + for _, d := range g.pods { + if namespace == "" || d.obj.Namespace == namespace { + out = append(out, d) + } + } + return out +} + +// Nodes returns deleted nodes still in retention. +func (g *Graveyard) Nodes() []deleted[*corev1.Node] { + if g == nil { + return nil + } + g.mu.Lock() + defer g.mu.Unlock() + g.pruneLocked() + out := make([]deleted[*corev1.Node], 0, len(g.nodes)) + for _, d := range g.nodes { + out = append(out, d) + } + return out +} + +func (g *Graveyard) pruneLocked() { + cutoff := g.now().Add(-g.retention) + for k, at := range g.seenPods { + if at.Before(cutoff) { + delete(g.seenPods, k) + } + } + for k, at := range g.seenNodes { + if at.Before(cutoff) { + delete(g.seenNodes, k) + } + } + for k, d := range g.pods { + if d.at.Before(cutoff) { + delete(g.pods, k) + } + } + for k, d := range g.nodes { + if d.at.Before(cutoff) { + delete(g.nodes, k) + } + } +} diff --git a/internal/metrics/nodes.go b/internal/metrics/nodes.go new file mode 100644 index 0000000..1762002 --- /dev/null +++ b/internal/metrics/nodes.go @@ -0,0 +1,334 @@ +package metrics + +import ( + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// nodeInfo is everything metering needs to know about a node. +type nodeInfo struct { + name string + created time.Time + gpuType string + zone string + region string + capacityType string + instanceType string + sku string // value of SKU_LABEL, if configured and present + gpuCapacity int64 // devices of GPU_RESOURCES (nvidia.com/gpu, amd.com/gpu) + cpuCapacity float64 + memCapacity float64 // bytes + gpuReplicas int // time-slicing replicas per physical GPU (1 = not shared) + labels map[string]string + deletedAt time.Time // set for nodes seen deleted (Auto Nodes scale-down); zero = present + draGPUs float64 // whole GPUs the node offers through DRA ResourceSlices + + // MIG and sharing. migProfile is set when every advertised GPU device is + // a MIG slice (NVIDIA "single" strategy, GKE GPU partitions). Under the + // "mixed" strategy slices are their own resources, summed in migMixed as + // whole-GPU equivalents. sharedCapacity counts nvidia.com/gpu.shared. + migProfile string + migMixed float64 + migDevices int64 + sharedCapacity int64 + + // Fair metering: a node that is not Ready, or carries an unhealthy + // taint, is down. downSince is when that started (zero = unknown, so the + // whole window counts as down: the tenant gets the benefit of the doubt). + down bool + downSince time.Time +} + +var gpuProductLabels = []string{ + "nvidia.com/gpu.product", + "nvidia.com/gpu.machine", + "amd.com/gpu.product-name", + "cloud.google.com/gke-accelerator", + "k8s.amazonaws.com/accelerator", + "accelerator", +} + +func (c *Collector) parseNode(n *corev1.Node) *nodeInfo { + ni := &nodeInfo{ + name: n.Name, + created: n.CreationTimestamp.Time, + gpuType: "unknown", + zone: firstLabel(n.Labels, "topology.kubernetes.io/zone", "failure-domain.beta.kubernetes.io/zone"), + region: firstLabel(n.Labels, "topology.kubernetes.io/region", "failure-domain.beta.kubernetes.io/region"), + instanceType: firstLabel(n.Labels, "node.kubernetes.io/instance-type", "beta.kubernetes.io/instance-type"), + capacityType: capacityType(n.Labels, c.opts.CapacityTypeLabel), + gpuReplicas: 1, + labels: n.Labels, + } + if v := firstLabel(n.Labels, gpuProductLabels...); v != "" { + ni.gpuType = normalizeGPUType(v) + } + // GPU Feature Discovery suffixes the product for sharing ("-SHARED") and + // for MIG single strategy ("-MIG-1g.10gb"); bill against the base GPU. + ni.gpuType = strings.TrimSuffix(ni.gpuType, "-SHARED") + if gpu, profile, ok := strings.Cut(ni.gpuType, "-MIG-"); ok && gpu != "" && profile != "" { + ni.gpuType, ni.migProfile = gpu, profile + } else if p := n.Labels["cloud.google.com/gke-gpu-partition-size"]; p != "" { + ni.migProfile = p + } + if c.opts.SKULabel != "" { + ni.sku = n.Labels[c.opts.SKULabel] + } + for _, res := range c.opts.GPUResources { + if q, ok := n.Status.Capacity[corev1.ResourceName(res)]; ok { + ni.gpuCapacity += q.Value() + } + } + for name, q := range n.Status.Capacity { + switch res := string(name); { + case res == "nvidia.com/gpu.shared": + ni.sharedCapacity += q.Value() + case strings.HasPrefix(res, "nvidia.com/mig-"): + ni.migDevices += q.Value() + ni.migMixed += float64(q.Value()) * migFraction(strings.TrimPrefix(res, "nvidia.com/mig-"), ni.gpuType) + } + } + if q, ok := n.Status.Capacity[corev1.ResourceCPU]; ok { + ni.cpuCapacity = float64(q.MilliValue()) / 1000 + } + if q, ok := n.Status.Capacity[corev1.ResourceMemory]; ok { + ni.memCapacity = float64(q.Value()) + } + if r, err := strconv.Atoi(n.Labels["nvidia.com/gpu.replicas"]); err == nil && r > 1 && + n.Labels["nvidia.com/gpu.sharing-strategy"] != "none" { + ni.gpuReplicas = r + } + // GKE shares GPUs natively (time-sharing or MPS) and labels the node itself. + if s := n.Labels["cloud.google.com/gke-gpu-sharing-strategy"]; s == "time-sharing" || s == "mps" { + if r, err := strconv.Atoi(n.Labels["cloud.google.com/gke-max-shared-clients-per-gpu"]); err == nil && r > 1 { + ni.gpuReplicas = r + } + } + + for _, cond := range n.Status.Conditions { + if cond.Type == corev1.NodeReady && cond.Status != corev1.ConditionTrue { + ni.markDown(cond.LastTransitionTime.Time) + } + } + for _, t := range n.Spec.Taints { + for _, key := range c.opts.UnhealthyTaints { + if t.Key == key { + var since time.Time + if t.TimeAdded != nil { + since = t.TimeAdded.Time + } + ni.markDown(since) + } + } + } + return ni +} + +// deviceShare is the part of a physical GPU that one advertised GPU device +// is on this node: a MIG slice, a time-slicing replica, or both. +func (n *nodeInfo) deviceShare() float64 { + share := 1.0 + if n.migProfile != "" { + share = migFraction(n.migProfile, n.gpuType) + } + if n.gpuReplicas > 1 { + share /= float64(n.gpuReplicas) + } + return share +} + +// expectedGPUs is how many physical GPUs the node's GPU Feature Discovery +// labels say it has (0 without such labels). While a driver or device plugin +// restarts (after a reboot, a preemption or a MIG change) a node advertises +// no GPUs although they are installed. +func (n *nodeInfo) expectedGPUs() float64 { + c, err := strconv.ParseFloat(n.labels["nvidia.com/gpu.count"], 64) + if err != nil || c <= 0 { + return 0 + } + if n.migProfile != "" { // under the single strategy the count is MIG devices + c *= migFraction(n.migProfile, n.gpuType) + } + return usage.Round(c) +} + +// deviceProfile names how one advertised GPU device is carved up. +func (n *nodeInfo) deviceProfile() string { + var parts []string + if n.migProfile != "" { + parts = append(parts, "mig-"+n.migProfile) + } + if n.gpuReplicas > 1 { + parts = append(parts, "timeslice-"+strconv.Itoa(n.gpuReplicas)) + } + if len(parts) == 0 { + return "full" + } + return strings.Join(parts, "-") +} + +// physicalGPUs converts everything the node advertises (whole GPUs, MIG +// slices, time-slicing replicas) to whole-GPU equivalents, the quantity a +// dedicated node is billed for. Slices that are not partitioned out are not +// advertised, so a partly carved GPU never counts as more than one. +func (n *nodeInfo) physicalGPUs() float64 { + shared := 0.0 + if n.sharedCapacity > 0 { + shared = float64(n.sharedCapacity) / float64(max(n.gpuReplicas, 1)) + } + return usage.Round(float64(n.gpuCapacity)*n.deviceShare() + shared + n.migMixed + n.draGPUs) +} + +// applyDRANodes adds the GPUs nodes offer through DRA, which no extended +// resource reports. +func applyDRANodes(nodes map[string]*nodeInfo, dra *draIndex) { + if dra == nil { + return + } + for name, n := range nodes { + n.draGPUs = dra.nodeGPUs[name] + if n.gpuType == "unknown" && dra.nodeType[name] != "" { + n.gpuType = dra.nodeType[name] + } + } +} + +// gpuDevices is the number of GPU devices the node advertises. +func (n *nodeInfo) gpuDevices() int64 { return n.gpuCapacity + n.sharedCapacity + n.migDevices } + +func (n *nodeInfo) markDown(since time.Time) { + if !n.down { + n.down, n.downSince = true, since + return + } + // Keep the earliest known start; zero (unknown) dominates. + if since.IsZero() || (!n.downSince.IsZero() && since.Before(n.downSince)) { + n.downSince = since + } +} + +// split divides the part of [start, end) inside [ws, we) into billable and +// downtime durations for this node. +func (n *nodeInfo) split(start, end, ws, we time.Time) (billable, down time.Duration) { + total := usage.Overlap(start, end, ws, we) + if total == 0 || n == nil || !n.down { + return total, 0 + } + from := start + if !n.downSince.IsZero() && n.downSince.After(from) { + from = n.downSince + } + down = usage.Overlap(from, end, ws, we) + return total - down, down +} + +func (n *nodeInfo) regionOr(fallback string, fromNode bool) string { + if fromNode && n != nil && n.region != "" { + return n.region + } + return fallback +} + +// capacityType normalizes the many spot/preemptible label conventions. +func capacityType(labels map[string]string, customKey string) string { + if v := labels[customKey]; v != "" && customKey != "" { + return normalizeCapacity(v) + } + for _, key := range []string{ + "karpenter.sh/capacity-type", + "eks.amazonaws.com/capacityType", + "kubernetes.io/lifecycle", + "node.kubernetes.io/lifecycle", + } { + if v := labels[key]; v != "" { + return normalizeCapacity(v) + } + } + if labels["cloud.google.com/gke-spot"] == "true" { + return usage.CapacitySpot + } + if labels["cloud.google.com/gke-preemptible"] == "true" { + return usage.CapacityPreemptible + } + return usage.CapacityOnDemand +} + +func normalizeCapacity(v string) string { + switch strings.ToLower(strings.ReplaceAll(v, "_", "-")) { + case "spot": + return usage.CapacitySpot + case "preemptible", "preemptable", "reclaimable": + return usage.CapacityPreemptible + case "reserved", "committed", "dedicated": + return usage.CapacityReserved + default: + return usage.CapacityOnDemand + } +} + +func normalizeGPUType(raw string) string { + return strings.ReplaceAll(strings.TrimSpace(raw), " ", "-") +} + +func firstLabel(labels map[string]string, keys ...string) string { + for _, k := range keys { + if v := labels[k]; v != "" { + return v + } + } + return "" +} + +// dedicatedTo reports whether a node is dedicated to the tenant cluster. +// Conventions (any match): vbilling.vcluster.com/tenant-cluster=, vcluster.loft.sh/managed-by or vcluster.loft.sh/cluster = +// , or the VCLUSTER_NODE_LABEL pattern ("key=%s"). +func (c *Collector) dedicatedTo(n *nodeInfo, name, namespace, externalID string) bool { + l := n.labels + if v := l["vbilling.vcluster.com/tenant-cluster"]; v != "" && (v == externalID || v == name) { + return true + } + candidates := []string{namespace, name} + for _, prefix := range []string{"vcluster-", "vc-"} { // v0.1 compatibility + if strings.HasPrefix(namespace, prefix) { + candidates = append(candidates, strings.TrimPrefix(namespace, prefix)) + } + } + for _, key := range []string{"vcluster.loft.sh/managed-by", "vcluster.loft.sh/cluster"} { + v := l[key] + for _, cand := range candidates { + if v != "" && v == cand { + return true + } + } + } + if c.opts.DedicatedNodeLabel != "" { + if key, pattern, ok := strings.Cut(c.opts.DedicatedNodeLabel, "="); ok { + for _, cand := range candidates { + if l[key] != "" && l[key] == strings.ReplaceAll(pattern, "%s", cand) { + return true + } + } + } + } + return false +} + +// nodeSKU is the SKU for whole-node billing. +func nodeSKU(n *nodeInfo) string { + switch { + case n.sku != "": + return n.sku + case n.instanceType != "": + return n.instanceType + case n.physicalGPUs() > 0: + return strconv.FormatFloat(n.physicalGPUs(), 'f', -1, 64) + "x" + n.gpuType + default: + return "cpu-node" + } +} diff --git a/internal/metrics/pods.go b/internal/metrics/pods.go new file mode 100644 index 0000000..0d5f160 --- /dev/null +++ b/internal/metrics/pods.go @@ -0,0 +1,150 @@ +package metrics + +import ( + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" +) + +// Labels vCluster's syncer puts on objects it creates in the host namespace. +const ( + LabelManagedBy = "vcluster.loft.sh/managed-by" + LabelVirtualNamespace = "vcluster.loft.sh/namespace" +) + +// gpuAlloc is one container's accelerator request. +type gpuAlloc struct { + count float64 // devices requested + fraction float64 // GPU-equivalent per device (1, 1/7 for a 1g MIG slice, 1/replicas when time-sliced) + profile string // full | mig- | timeslice- +} + +// podLifetime returns the billable interval of a pod: from the moment its +// first container started (image pulls and scheduling are not billed) to +// the moment its last container finished. A zero end means still running; +// started=false means nothing has run yet. +func podLifetime(p *corev1.Pod) (start, end time.Time, started bool) { + running := false + for _, cs := range p.Status.ContainerStatuses { + var s time.Time + switch { + case cs.State.Running != nil: + s = cs.State.Running.StartedAt.Time + running = true + case cs.State.Terminated != nil: + s = cs.State.Terminated.StartedAt.Time + if f := cs.State.Terminated.FinishedAt.Time; f.After(end) { + end = f + } + } + // A restarted container reports its latest start; the previous run + // is in LastTerminationState. + if lt := cs.LastTerminationState.Terminated; lt != nil && !lt.StartedAt.IsZero() && (s.IsZero() || lt.StartedAt.Time.Before(s)) { + s = lt.StartedAt.Time + } + if !s.IsZero() && (start.IsZero() || s.Before(start)) { + start = s + } + } + if start.IsZero() { + return time.Time{}, time.Time{}, false + } + if running || (p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed) { + end = time.Time{} + } + return start, end, true +} + +// gpuAllocs extracts accelerator requests from a pod. Device plugins +// require requests == limits, so whichever is set counts. +func (c *Collector) gpuAllocs(p *corev1.Pod, n *nodeInfo) []gpuAlloc { + var out []gpuAlloc + full := map[string]bool{} + for _, r := range c.opts.GPUResources { + full[r] = true + } + for _, ctr := range p.Spec.Containers { + seen := map[corev1.ResourceName]bool{} + for _, list := range []corev1.ResourceList{ctr.Resources.Requests, ctr.Resources.Limits} { + for name, q := range list { + if seen[name] { + continue + } + seen[name] = true + count := float64(q.Value()) + if count <= 0 { + continue + } + res := string(name) + switch { + case full[res]: + // On MIG single-strategy or time-sliced nodes one device + // is a slice or replica of a GPU, not a whole GPU. + a := gpuAlloc{count: count, fraction: 1, profile: "full"} + if n != nil { + a.fraction, a.profile = n.deviceShare(), n.deviceProfile() + } + out = append(out, a) + case res == "nvidia.com/gpu.shared": + replicas := 1 + if n != nil && n.gpuReplicas > 1 { + replicas = n.gpuReplicas + } + out = append(out, gpuAlloc{count: count, fraction: 1 / float64(replicas), profile: "timeslice-" + strconv.Itoa(replicas)}) + case strings.HasPrefix(res, "nvidia.com/mig-"): + profile := strings.TrimPrefix(res, "nvidia.com/mig-") + gpuType := "" + if n != nil { + gpuType = n.gpuType + } + out = append(out, gpuAlloc{count: count, fraction: migFraction(profile, gpuType), profile: "mig-" + profile}) + } + } + } + } + return out +} + +// migFraction converts a MIG profile ("1g.10gb", "3g.40gb", "1g.10gb+me") +// to the share of a physical GPU's compute slices. A30 GPUs have 4 slices; +// A100/H100/H200/B200-class GPUs have 7. +func migFraction(profile, gpuType string) float64 { + g := strings.IndexByte(profile, 'g') + if g <= 0 { + return 1 + } + slices, err := strconv.Atoi(profile[:g]) + if err != nil || slices <= 0 { + return 1 + } + total := 7.0 + if strings.Contains(strings.ToUpper(gpuType), "A30") { + total = 4 + } + if f := float64(slices) / total; f < 1 { + return f + } + return 1 +} + +// podRequests sums CPU (cores) and memory (bytes) requests of a pod's +// regular containers. +func podRequests(p *corev1.Pod) (cpu, mem float64) { + for _, ctr := range p.Spec.Containers { + if q, ok := ctr.Resources.Requests[corev1.ResourceCPU]; ok { + cpu += float64(q.MilliValue()) / 1000 + } + if q, ok := ctr.Resources.Requests[corev1.ResourceMemory]; ok { + mem += float64(q.Value()) + } + } + return cpu, mem +} + +// isControlPlane reports whether a host object belongs to the tenant +// cluster's own control plane (its StatefulSet pods, data PVCs, API service). +func isControlPlane(labels map[string]string, clusterName string) bool { + return labels["app"] == "vcluster" && (labels["release"] == clusterName || labels["release"] == "") +} diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go new file mode 100644 index 0000000..22239a7 --- /dev/null +++ b/internal/metrics/prometheus.go @@ -0,0 +1,269 @@ +package metrics + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "sigs.k8s.io/yaml" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// prometheusClient runs instant queries. Queries are evaluated at the +// window end, so results line up with metering windows. Any server with the +// Prometheus HTTP API works (Prometheus, Thanos, Mimir, VictoriaMetrics). +// Basic auth can be given in the URL (https://user:pass@host); headers and a +// bearer token file cover token-based and multi-tenant endpoints. +type prometheusClient struct { + baseURL string + httpClient *http.Client + headers map[string]string + tokenFile string // re-read on every query so rotated tokens keep working +} + +func newPrometheusClient(baseURL string, headers map[string]string, tokenFile string) *prometheusClient { + return &prometheusClient{ + baseURL: strings.TrimRight(baseURL, "/"), + httpClient: &http.Client{Timeout: 10 * time.Second}, + headers: headers, + tokenFile: tokenFile, + } +} + +type promResult struct { + Labels map[string]string + Value float64 +} + +func (p *prometheusClient) Query(ctx context.Context, query string, at time.Time) ([]promResult, error) { + params := url.Values{} + params.Set("query", query) + params.Set("time", strconv.FormatInt(at.Unix(), 10)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/api/v1/query?"+params.Encode(), nil) + if err != nil { + return nil, err + } + for k, v := range p.headers { + req.Header.Set(k, v) + } + if p.tokenFile != "" && req.Header.Get("Authorization") == "" { + token, err := os.ReadFile(p.tokenFile) + if err != nil { + return nil, fmt.Errorf("read prometheus bearer token: %w", err) + } + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(token))) + } + resp, err := p.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("prometheus query: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("prometheus query: HTTP %d (check PROMETHEUS_HEADERS, PROMETHEUS_BEARER_TOKEN_FILE or the URL credentials)", resp.StatusCode) + } + var pr struct { + Status string `json:"status"` + Error string `json:"error"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(body, &pr); err != nil { + return nil, fmt.Errorf("decode prometheus response (HTTP %d): %w", resp.StatusCode, err) + } + if pr.Status != "success" { + return nil, fmt.Errorf("prometheus query failed: %s", pr.Error) + } + out := make([]promResult, 0, len(pr.Data.Result)) + for _, r := range pr.Data.Result { + if len(r.Value) < 2 { + continue + } + s, _ := r.Value[1].(string) + v, err := strconv.ParseFloat(s, 64) + if err != nil { + continue + } + out = append(out, promResult{Labels: r.Metric, Value: v}) + } + return out, nil +} + +// queryVars fill the placeholders of query templates: {{namespace}} (the +// tenant cluster's namespace on the control plane cluster), {{vcluster}} +// (its name), {{instance}} (its vCluster Platform instance), {{project}}, +// {{tenant}} (the billing customer), {{tenant_cluster}} (its external ID), +// {{window}} (the window length, e.g. 60s) and {{window_seconds}} (e.g. 60, +// to integrate a gauge over the window). +type queryVars struct { + Namespace, VCluster, Instance, Project, Tenant, TenantCluster string + Window time.Duration +} + +func varsFor(t Target, window time.Duration) queryVars { + instance := t.Cluster.Instance + if instance == "" { + instance = t.Cluster.Name + } + return queryVars{Namespace: t.Cluster.Namespace, VCluster: t.Cluster.Name, Instance: instance, + Project: t.Project, Tenant: t.Tenant, TenantCluster: t.Cluster.ExternalID(), Window: window} +} + +// expand fills a query template. Values are escaped for PromQL string literals. +func expand(tmpl string, v queryVars) string { + q := func(s string) string { return strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) } + return strings.NewReplacer( + "{{namespace}}", q(v.Namespace), + "{{vcluster}}", q(v.VCluster), + "{{instance}}", q(v.Instance), + "{{project}}", q(v.Project), + "{{tenant}}", q(v.Tenant), + "{{tenant_cluster}}", q(v.TenantCluster), + "{{window}}", strconv.Itoa(int(v.Window.Seconds()))+"s", + "{{window_seconds}}", strconv.Itoa(int(v.Window.Seconds())), + ).Replace(tmpl) +} + +// Default queries. Egress counts transmitted bytes of the tenant cluster's +// pods; set EGRESS_QUERY to count only internet-bound traffic (for example +// from Cilium/Hubble or your fabric's flow metrics). +// +// GPU utilization comes from the DCGM exporter, which labels each GPU with +// the workload's namespace. Prometheus keeps that label as `namespace` when +// it honors the exporter's labels, and renames it to `exported_namespace` +// when it does not (the kube-prometheus-stack default), so the defaults +// match either. GPU_UTIL_QUERY and GPU_COUNT_QUERY replace them. +const defaultEgressQuery = `sum(increase(container_network_transmit_bytes_total{namespace="{{namespace}}"}[{{window}}]))` + +var ( + defaultGPUUtilQuery = dcgmUtilQuery(`namespace="{{namespace}}"`, `exported_namespace="{{namespace}}"`) + defaultGPUCountQuery = dcgmCountQuery(`namespace="{{namespace}}"`, `exported_namespace="{{namespace}}"`) +) + +// dcgmUtilQuery averages DCGM GPU utilization (percent) per model for any of +// the selectors. DCGM reports no DCGM_FI_DEV_GPU_UTIL for MIG instances, so +// GPUs in MIG mode use each instance's graphics engine activity +// (DCGM_FI_PROF_GR_ENGINE_ACTIVE, 0-1) instead. +func dcgmUtilQuery(selectors ...string) string { + var parts []string + for _, s := range selectors { + parts = append(parts, `avg_over_time(DCGM_FI_DEV_GPU_UTIL{`+s+`}[{{window}}])`) + } + for _, s := range selectors { + parts = append(parts, `100 * avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{`+s+`,GPU_I_ID!=""}[{{window}}])`) + } + return `avg by (modelName) (` + strings.Join(parts, " or ") + `)` +} + +// dcgmCountQuery counts physical GPUs per model. The MIG instances of a GPU +// are separate series that share its UUID, so they count once. +func dcgmCountQuery(selectors ...string) string { + var parts []string + for _, s := range selectors { + parts = append(parts, `DCGM_FI_DEV_GPU_UTIL{`+s+`}`) + } + for _, s := range selectors { + parts = append(parts, `DCGM_FI_PROF_GR_ENGINE_ACTIVE{`+s+`,GPU_I_ID!=""}`) + } + return `count by (modelName) (group by (modelName, UUID, gpu, Hostname, hostname) (` + strings.Join(parts, " or ") + `))` +} + +// PresetVClusterPlatform reads vCluster Platform fleet observability: tenant +// clusters' OpenTelemetry collectors push metrics through the Platform's +// write gateway, which stamps every series with tamper-proof +// vcluster_platform_* labels from the collector's access key. Metrics from +// private nodes (DCGM, kubelet/cAdvisor) arrive the same way. +const PresetVClusterPlatform = "vcluster-platform" + +const platformSelector = `vcluster_platform_instance="{{instance}}",vcluster_platform_project="{{project}}"` + +var platformQueries = struct{ egress, util, count string }{ + egress: `sum(increase(container_network_transmit_bytes_total{` + platformSelector + `}[{{window}}]))`, + util: dcgmUtilQuery(platformSelector), + count: dcgmCountQuery(platformSelector), +} + +// PromMetric is a usage metric defined by a PromQL query, evaluated for each +// tenant cluster at every window end (PROMETHEUS_METRICS_FILE). Billable +// ones are created in the billing backends like built-in metrics. +type PromMetric struct { + Code string `json:"code"` + Name string `json:"name,omitempty"` + Unit string `json:"unit,omitempty"` + Description string `json:"description,omitempty"` + Billable bool `json:"billable"` + Query string `json:"query"` + Dimensions []string `json:"dimensions,omitempty"` // result labels kept as event dimensions (and pricing group keys) + SKU string `json:"sku,omitempty"` // result label used as the event SKU +} + +// LoadPromMetrics reads PROMETHEUS_METRICS_FILE and registers its metrics. +func LoadPromMetrics(path string) ([]PromMetric, error) { + if path == "" { + return nil, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var out []PromMetric + if err := yaml.UnmarshalStrict(data, &out); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + for _, m := range out { + if m.Query == "" { + return nil, fmt.Errorf("%s: metric %q has no query", path, m.Code) + } + var keys []string + for _, d := range m.Dimensions { + keys = append(keys, dimKey(d)) + } + if err := usage.RegisterCustom(usage.MetricDef{Code: m.Code, Name: m.Name, Unit: m.Unit, Description: m.Description, + Billable: m.Billable, GroupKeys: keys}); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + } + return out, nil +} + +// dimKey turns a Prometheus label (modelName, model-name) into a dimension +// key (model_name). +func dimKey(label string) string { + var b strings.Builder + for i, r := range label { + switch { + case r >= 'A' && r <= 'Z': + if i > 0 { + b.WriteByte('_') + } + b.WriteRune(r + ('a' - 'A')) + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + k := strings.Trim(b.String(), "_") + if k == "" || k[0] < 'a' || k[0] > 'z' { + k = "l_" + k + } + if len(k) > 64 { + k = k[:64] + } + return k +} diff --git a/internal/metrics/prometheus_test.go b/internal/metrics/prometheus_test.go new file mode 100644 index 0000000..4d16a10 --- /dev/null +++ b/internal/metrics/prometheus_test.go @@ -0,0 +1,88 @@ +package metrics + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +func TestQueryPlaceholdersAreFilledAndEscaped(t *testing.T) { + v := queryVars{Namespace: "team-a", VCluster: "gpu", Instance: "gpu-prod", Project: "research", Tenant: `ac"me`, TenantCluster: "vcluster-team-a-gpu", Window: time.Minute} + got := expand(`x{a="{{namespace}}",b="{{vcluster}}",c="{{instance}}",d="{{project}}",e="{{tenant}}",f="{{tenant_cluster}}"}[{{window}}] * {{window_seconds}}`, v) + want := `x{a="team-a",b="gpu",c="gpu-prod",d="research",e="ac\"me",f="vcluster-team-a-gpu"}[60s] * 60` + if got != want { + t.Errorf("expand = %s\nwant %s", got, want) + } +} + +func TestPlatformPresetQueriesFleetObservabilityLabels(t *testing.T) { + prom, queries := promServer(t, nil, func(q string) (string, map[string]string) { + if strings.Contains(q, "DCGM_FI_DEV_GPU_UTIL") && strings.Contains(q, `vcluster_platform_instance="gpu"`) && strings.Contains(q, "avg_over_time") { + return "40", map[string]string{"modelName": "NVIDIA A100-SXM4-40GB"} + } + return "", nil + }) + evs := collect(t, Options{PrometheusURL: prom.URL, PrometheusPreset: PresetVClusterPlatform}, nil, latest())[we] + if got, _ := find(evs, usage.MetricGPUUtilization, usage.DimGPUType, "NVIDIA-A100-SXM4-40GB"); !near(got, usage.Round(40.0/60)) { + t.Errorf("utilization from fleet observability = %v", got) + } + for _, q := range *queries { + if !strings.Contains(q, `vcluster_platform_instance="gpu",vcluster_platform_project="research"`) { + t.Errorf("preset query without the tamper-proof tenant selector: %s", q) + } + } +} + +func TestPrometheusDefinedBillableMetrics(t *testing.T) { + path := filepath.Join(t.TempDir(), "metrics.yaml") + body := `- code: test_output_tokens + name: Output tokens + unit: tokens + billable: true + dimensions: [model_name, modelTier] + sku: model_name + query: sum by (model_name, modelTier) (increase(vllm:generation_tokens_total{namespace="{{namespace}}"}[{{window}}])) +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + pm, err := LoadPromMetrics(path) + if err != nil { + t.Fatal(err) + } + def, ok := usage.Lookup("test_output_tokens") + if !ok || !def.Billable || strings.Join(def.GroupKeys, ",") != "model_name,model_tier" { + t.Fatalf("registered metric = %+v", def) + } + prom, queries := promServer(t, nil, func(q string) (string, map[string]string) { + if strings.Contains(q, "vllm:generation_tokens_total") { + return "125000", map[string]string{"model_name": "llama-3.1-70b", "modelTier": "premium"} + } + return "", nil + }) + evs := collect(t, Options{PrometheusURL: prom.URL, PromMetrics: pm, GPUUtilQuery: "none"}, []runtime.Object{}, latest())[we] + got, n := find(evs, "test_output_tokens", "sku", "llama-3.1-70b", "model_name", "llama-3.1-70b", "model_tier", "premium") + if n != 1 || got != 125000 { + t.Fatalf("token events: %v (%d)", got, n) + } + if !strings.Contains(strings.Join(*queries, "\n"), `vllm:generation_tokens_total{namespace="team-a"}[60s]`) { + t.Errorf("queries = %q", *queries) + } + if _, err := LoadPromMetrics(path); err == nil { + t.Error("registering the same metric twice must fail") + } +} + +func TestDimKey(t *testing.T) { + for in, want := range map[string]string{"modelName": "model_name", "model-name": "model_name", "GPU": "g_p_u", "9lives": "l_9lives"} { + if got := dimKey(in); got != want { + t.Errorf("dimKey(%s) = %s, want %s", in, got, want) + } + } +} diff --git a/internal/metrics/tenant.go b/internal/metrics/tenant.go new file mode 100644 index 0000000..d721cec --- /dev/null +++ b/internal/metrics/tenant.go @@ -0,0 +1,171 @@ +package metrics + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + metricsclient "k8s.io/metrics/pkg/client/clientset/versioned" + + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// Tenant clusters with their own nodes (vCluster Private Nodes, Auto Nodes, +// Standalone) run workloads the control plane cluster never sees: nothing is +// synced to it. Those tenant clusters are metered through their own API. + +// ErrNoTenantAPI means no credentials are configured for a tenant cluster's +// own API; it is then metered from the control plane cluster only. +var ErrNoTenantAPI = errors.New("no tenant API credentials") + +// LabelFakeNode marks the placeholder nodes vCluster creates for shared +// tenant clusters; LabelBillable=false exempts a node (for example hardware +// the tenant owns) from whole-node billing. +const ( + LabelFakeNode = "vcluster.loft.sh/fake-node" + LabelBillable = "vbilling.vcluster.com/billable" +) + +// TenantConn is a connection to a tenant cluster's own API server. +type TenantConn struct { + Kube kubernetes.Interface + Metrics metricsclient.Interface // optional, usage-based CPU/memory in usage mode + Dynamic dynamic.Interface // optional, DRA ResourceClaims + Graveyard *Graveyard // optional, nodes and pods deleted between collections + // External tenant clusters (Standalone, or anything outside this control + // plane cluster) share no nodes with it, so no node is excluded. + External bool +} + +// TenantAPI resolves connections to tenant clusters' own API servers. +type TenantAPI interface { + Conn(ctx context.Context, cl discovery.TenantCluster) (*TenantConn, error) +} + +// UseTenantAPI meters tenant clusters through their own API as well. +func (c *Collector) UseTenantAPI(t TenantAPI) { c.tenants = t } + +// CollectTenantAPI meters only what tenant APIs see, for windows whose +// collection could not reach a tenant API. Its events go to the ledger as +// late events, deduplicated by ID. +func (c *Collector) CollectTenantAPI(ctx context.Context, targets []Target, windows []Window) (map[time.Time][]usage.Event, Stats, error) { + return c.collect(ctx, targets, windows, true) +} + +// collectTenant runs the tenant-API part for one target. All-or-nothing: on +// error nothing is added, and the target is reported in Stats so its +// windows can be filled once the API is reachable again. +func (c *Collector) collectTenant(ctx context.Context, t Target, hostNodes map[string]*nodeInfo, windows []Window, accs map[time.Time]*accumulator, st *Stats) { + conn, err := c.tenants.Conn(ctx, t.Cluster) + if errors.Is(err, ErrNoTenantAPI) { + return + } + tmp := map[time.Time]*accumulator{} + for _, w := range windows { + tmp[w.End] = newAccumulator(w) + } + if err == nil { + err = c.collectTenantAPI(ctx, t, conn, hostNodes, windows, tmp, st) + } + if err != nil { + log.Printf("[metrics] %s: tenant API unavailable, its windows will be filled later: %v", t.Cluster.ExternalID(), err) + st.TenantAPIFailed = append(st.TenantAPIFailed, t.Cluster.ExternalID()) + return + } + for end, a := range tmp { + for _, k := range a.order { + ev := *a.events[k] + qty, props := ev.Quantity, ev.Properties + ev.Quantity, ev.Properties = 0, nil + accs[end].add(ev, qty, props) + } + } +} + +func (c *Collector) collectTenantAPI(ctx context.Context, t Target, conn *TenantConn, hostNodes map[string]*nodeInfo, windows []Window, accs map[time.Time]*accumulator, st *Stats) error { + base := c.baseFor(t) + list, err := conn.Kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list nodes: %w", err) + } + conn.Graveyard.SeenNodes(list.Items, c.opts.Now()) + private := map[string]*nodeInfo{} + consider := func(n *corev1.Node, deletedAt time.Time) { + if _, onHost := hostNodes[n.Name]; onHost && !conn.External { + return // a control plane cluster node: metered there (shared or dedicated) + } + if n.Labels[LabelFakeNode] == "true" { + return + } + ni := c.parseNode(n) + ni.deletedAt = deletedAt + private[ni.name] = ni + } + live := map[string]bool{} + for i := range list.Items { + live[list.Items[i].Name] = true + consider(&list.Items[i], time.Time{}) + } + for _, d := range conn.Graveyard.Nodes() { + if !live[d.obj.Name] { + consider(d.obj, d.at) + } + } + if len(private) == 0 { + return nil // a shared tenant cluster: everything is metered from the control plane cluster + } + dra := c.draFor(ctx, conn.Dynamic, "") + applyDRANodes(private, dra) + st.PrivateNodes += len(private) + for _, n := range private { + if n.down { + st.NodesDown++ + } + } + + if c.opts.PrivateNodeBilling == "usage" { + podList, err := conn.Kube.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list pods: %w", err) + } + conn.Graveyard.SeenPods(podList.Items, c.opts.Now()) + pods := withDeleted(podList.Items, conn.Graveyard.Pods("")) + var usageByPod map[string][2]float64 + if latest := windows[len(windows)-1]; latest.Latest && c.opts.Basis != "requests" && conn.Metrics != nil { + usageByPod = podUsage(ctx, conn.Metrics, "") + } + include := func(p *corev1.Pod) bool { + _, onPrivate := private[p.Spec.NodeName] + return onPrivate && !c.excludedTenantNS[p.Namespace] + } + c.meterPods(pods, include, func(p *corev1.Pod) string { return p.Namespace }, + private, nil, dra, windows, accs, st, base, usageByPod, "private_node") + } else { + for name, n := range private { + if n.labels[LabelBillable] == "false" { + continue + } + c.meterWholeNode(name, n, windows, accs, base, "private_node") + } + } + + // Volumes and load balancers of such a tenant cluster live in it too. + pvcs, err := conn.Kube.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list pvcs: %w", err) + } + meterVolumes(pvcs.Items, func(map[string]string) bool { return true }, windows, accs, base) + svcs, err := conn.Kube.CoreV1().Services("").List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list services: %w", err) + } + meterLoadBalancers(svcs.Items, func(*corev1.Service) bool { return true }, windows, accs, base) + return nil +} diff --git a/internal/metrics/tenant_test.go b/internal/metrics/tenant_test.go new file mode 100644 index 0000000..a9c4b3d --- /dev/null +++ b/internal/metrics/tenant_test.go @@ -0,0 +1,259 @@ +package metrics + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +type fakeTenantAPI struct { + conns map[string]*TenantConn + err error +} + +func (f fakeTenantAPI) Conn(_ context.Context, cl discovery.TenantCluster) (*TenantConn, error) { + if f.err != nil { + return nil, f.err + } + if c, ok := f.conns[cl.Name]; ok { + return c, nil + } + return nil, ErrNoTenantAPI +} + +func tenantConn(objs ...runtime.Object) *TenantConn { + return &TenantConn{Kube: fake.NewSimpleClientset(objs...)} +} + +// collectWith is collect with a hook to attach tenant APIs or a graveyard. +func collectWith(t *testing.T, opts Options, host []runtime.Object, windows []Window, setup func(*Collector)) (map[time.Time][]usage.Event, Stats) { + t.Helper() + if opts.GPUResources == nil { + opts.GPUResources = []string{"nvidia.com/gpu"} + } + if opts.Basis == "" { + opts.Basis = "requests" + } + opts.Region = "ap-southeast-2" + opts.Now = func() time.Time { return we.Add(2 * time.Second) } + c := NewCollector(fake.NewSimpleClientset(host...), nil, opts) + setup(c) + out, st, err := c.Collect(context.Background(), []Target{target("gpu", true)}, windows) + if err != nil { + t.Fatal(err) + } + return out, st +} + +func lbService(name string) *corev1.Service { + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "inference", CreationTimestamp: metav1.NewTime(ws.Add(-time.Hour))}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + Status: corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: "10.0.0.9"}}}}} +} + +func boundPVC(name, size string) *corev1.PersistentVolumeClaim { + class := "fast-nvme" + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "training", CreationTimestamp: metav1.NewTime(ws.Add(-time.Hour))}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &class}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound, Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(size)}}} +} + +func TestPrivateNodesBilledWholeFromTheTenantAPI(t *testing.T) { + host := []runtime.Object{node("cp-worker-1", nil, 0, true)} + gpuNode := node("gpu-a100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-80GB", "node.kubernetes.io/instance-type": "a2-ultragpu-8g"}, 8, true) + tenant := tenantConn( + node("cp-worker-1", nil, 0, true), // a synced copy of a control plane cluster node: never billed here + node("vcluster-fake", map[string]string{LabelFakeNode: "true"}, 0, true), + gpuNode, + node("byo-gpu", map[string]string{"nvidia.com/gpu.product": "NVIDIA-L4", LabelBillable: "false"}, 1, true), + pod("trainer", "gpu-a100-1", withGPU("nvidia.com/gpu", 8)), + boundPVC("ckpt", "100Gi"), + lbService("serve"), + ) + out, st := collectWith(t, Options{}, host, latest(), func(c *Collector) { + c.UseTenantAPI(fakeTenantAPI{conns: map[string]*TenantConn{"gpu": tenant}}) + }) + evs := out[we] + if st.PrivateNodes != 2 { + t.Errorf("private nodes = %d, want gpu-a100-1 and byo-gpu", st.PrivateNodes) + } + if got, n := find(evs, usage.MetricPrivateNodeHours, "resource_id", "gpu-a100-1", "sku", "a2-ultragpu-8g", usage.DimBillingMode, "private_node"); n != 1 || !near(got, usage.Round(1.0/60)) { + t.Fatalf("private node hours = %v (%d events)", got, n) + } + if got, _ := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "private_node"); !near(got, usage.Round(8.0/60)) { + t.Fatalf("private node GPU hours = %v, want the node's 8 GPUs", got) + } + if _, n := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "shared"); n != 0 { + t.Fatal("pods on a private node were billed again per pod") + } + for _, name := range []string{"cp-worker-1", "vcluster-fake", "byo-gpu"} { + if _, n := find(evs, usage.MetricPrivateNodeHours, "resource_id", name); n != 0 { + t.Errorf("node %s must not be billed as a private node", name) + } + } + if got, _ := find(evs, usage.MetricStorageGBHours, "sku", "fast-nvme"); !near(got, usage.Round(100.0/60)) { + t.Errorf("tenant storage GiB-hours = %v", got) + } + if got, _ := find(evs, usage.MetricLBHours); !near(got, usage.Round(1.0/60)) { + t.Errorf("tenant load balancer hours = %v", got) + } +} + +func TestSharedTenantClusterIsNeverBilledTwice(t *testing.T) { + // Shared tenant clusters see the control plane cluster's nodes (synced or + // as placeholders); with a tenant API configured, nothing extra is billed. + h100 := node("h100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true) + host := []runtime.Object{h100, pod("a", "h100-1", withGPU("nvidia.com/gpu", 2))} + tenant := tenantConn(node("h100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true), + node("h100-2", map[string]string{LabelFakeNode: "true"}, 8, true)) + withAPI, st := collectWith(t, Options{}, host, latest(), func(c *Collector) { + c.UseTenantAPI(fakeTenantAPI{conns: map[string]*TenantConn{"gpu": tenant}}) + }) + without := collect(t, Options{}, host, latest()) + if st.PrivateNodes != 0 || len(withAPI[we]) != len(without[we]) { + t.Fatalf("tenant API changed a shared tenant cluster's bill: %d vs %d events, %d private nodes", len(withAPI[we]), len(without[we]), st.PrivateNodes) + } + if got, _ := find(withAPI[we], usage.MetricGPUHours); !near(got, usage.Round(2.0/60)) { + t.Errorf("GPU hours = %v", got) + } +} + +func TestPrivateNodeUsageModeBillsPods(t *testing.T) { + gpuNode := node("gpu-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true) + dcgm := pod("dcgm-exporter", "gpu-1", withRequests("1", "1Gi")) + dcgm.Namespace = "kube-system" + trainer := pod("trainer", "gpu-1", withGPU("nvidia.com/gpu", 2), withRequests("4", "8Gi")) + trainer.Namespace = "research" + tenant := tenantConn(gpuNode, dcgm, trainer) + out, _ := collectWith(t, Options{PrivateNodeBilling: "usage", MeterByNamespace: true}, nil, latest(), func(c *Collector) { + c.UseTenantAPI(fakeTenantAPI{conns: map[string]*TenantConn{"gpu": tenant}}) + }) + evs := out[we] + if got, _ := find(evs, usage.MetricGPUHours, usage.DimBillingMode, "private_node", usage.DimNamespace, "research"); !near(got, usage.Round(2.0/60)) { + t.Errorf("usage-mode GPU hours = %v", got) + } + if got, _ := find(evs, usage.MetricCPUCoreHours, usage.DimBillingMode, "private_node"); !near(got, usage.Round(4.0/60)) { + t.Errorf("usage-mode CPU core-hours = %v, want only the tenant's own pods (kube-system excluded)", got) + } + if _, n := find(evs, usage.MetricPrivateNodeHours); n != 0 { + t.Error("usage mode must not also bill the whole node") + } +} + +func TestTenantAPIOutageIsReportedAndAddsNothing(t *testing.T) { + host := []runtime.Object{node("cpu-1", nil, 0, true), pod("web", "cpu-1", withRequests("1", "1Gi"))} + out, st := collectWith(t, Options{}, host, latest(), func(c *Collector) { + c.UseTenantAPI(fakeTenantAPI{err: errors.New("connection refused")}) + }) + if len(st.TenantAPIFailed) != 1 || st.TenantAPIFailed[0] != "vcluster-team-a-gpu" { + t.Fatalf("failed tenant APIs = %v", st.TenantAPIFailed) + } + if got, _ := find(out[we], usage.MetricCPUCoreHours); !near(got, usage.Round(1.0/60)) { + t.Errorf("control plane cluster metering must continue during a tenant API outage, CPU = %v", got) + } +} + +func TestCollectTenantAPIOnlyMetersTenantSources(t *testing.T) { + host := []runtime.Object{node("cpu-1", nil, 0, true), pod("web", "cpu-1", withRequests("1", "1Gi"))} + tenant := tenantConn(node("gpu-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-L4"}, 1, true)) + opts := Options{GPUResources: []string{"nvidia.com/gpu"}, Basis: "requests", Region: "r", Now: func() time.Time { return we }} + c := NewCollector(fake.NewSimpleClientset(host...), nil, opts) + c.UseTenantAPI(fakeTenantAPI{conns: map[string]*TenantConn{"gpu": tenant}}) + out, _, err := c.CollectTenantAPI(context.Background(), []Target{target("gpu", true)}, []Window{{Start: ws, End: we}}) + if err != nil { + t.Fatal(err) + } + for _, e := range out[we] { + if e.Dim(usage.DimBillingMode) != "private_node" { + t.Errorf("unexpected %s event from a tenant-API-only collection", e.Metric) + } + } + if got, _ := find(out[we], usage.MetricGPUHours); !near(got, usage.Round(1.0/60)) { + t.Errorf("GPU hours = %v", got) + } +} + +func TestExternalTenantClustersShareNoNodes(t *testing.T) { + host := []runtime.Object{node("node-1", nil, 0, true)} + tenant := tenantConn(node("node-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-L40S"}, 4, true)) + tenant.External = true + out, _ := collectWith(t, Options{}, host, latest(), func(c *Collector) { + c.UseTenantAPI(fakeTenantAPI{conns: map[string]*TenantConn{"gpu": tenant}}) + }) + if got, _ := find(out[we], usage.MetricGPUHours, usage.DimBillingMode, "private_node"); !near(got, usage.Round(4.0/60)) { + t.Errorf("standalone tenant cluster node with a colliding name: GPU hours = %v", got) + } +} + +func TestDeletedPodsAndNodesBilledUntilDeletion(t *testing.T) { + g := NewGraveyard(time.Hour, func() time.Time { return we }) + // A job that started 10s into the window and was deleted 30s in, gone + // from every list by the time the window closes. + job := pod("short-job", "h100-1", withGPU("nvidia.com/gpu", 1), startedAt(ws.Add(10*time.Second))) + job.UID = types.UID("job-1") + g.RecordPod(job, ws.Add(30*time.Second)) + // A dedicated Auto Node scaled down 45s into the window. + autoNode := node("auto-gpu-1", map[string]string{"vcluster.loft.sh/managed-by": ns, "nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true) + g.RecordNode(autoNode, ws.Add(45*time.Second)) + host := []runtime.Object{node("h100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-H100"}, 8, true)} + out, _ := collectWith(t, Options{}, host, latest(), func(c *Collector) { c.UseGraveyard(g) }) + if got, _ := find(out[we], usage.MetricGPUHours, usage.DimBillingMode, "shared"); !near(got, usage.Round(20.0/3600)) { + t.Errorf("deleted job GPU hours = %v, want exactly the 20s it ran", got) + } + if got, _ := find(out[we], usage.MetricGPUHours, usage.DimBillingMode, "dedicated_node"); !near(got, usage.Round(8*45/3600.0)) { + t.Errorf("scaled-down node GPU hours = %v, want 8 GPUs for 45s", got) + } + expired := NewGraveyard(time.Minute, func() time.Time { return we.Add(2 * time.Hour) }) + expired.RecordPod(job, ws.Add(30*time.Second)) + if len(expired.Pods("")) != 0 { + t.Error("deleted pods past retention must be dropped") + } +} + +func TestGKETimeSharingLabels(t *testing.T) { + l4 := node("gke-l4", map[string]string{"cloud.google.com/gke-accelerator": "nvidia-l4", + "cloud.google.com/gke-gpu-sharing-strategy": "time-sharing", "cloud.google.com/gke-max-shared-clients-per-gpu": "4"}, 4, true) + evs := collect(t, Options{}, []runtime.Object{l4, pod("infer", "gke-l4", withGPU("nvidia.com/gpu", 1))}, latest())[we] + if got, _ := find(evs, usage.MetricGPUHours, "sku", "nvidia-l4-timeslice-4"); !near(got, usage.Round(0.25/60)) { + t.Errorf("GKE time-shared GPU hours = %v", got) + } +} + +// Seen on GCP: after a reboot or a spot restart, a Ready private node +// advertises no GPUs for minutes while the driver container rebuilds. The +// GPUs are installed (GPU Feature Discovery still labels them), so the time +// is GPU downtime, not usage and not silence. +func TestPrivateNodeGPUsNotAllocatableAreDowntime(t *testing.T) { + loading := node("gpu-a100-1", map[string]string{"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-40GB", "nvidia.com/gpu.count": "1", "node.kubernetes.io/instance-type": "a2-highgpu-1g"}, 0, true) + mig := node("gpu-a100-2", map[string]string{"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-40GB-MIG-1g.5gb", "nvidia.com/gpu.count": "7", "nvidia.com/mig.strategy": "single"}, 0, true) + cpu := node("cpu-1", nil, 0, true) + out, _ := collectWith(t, Options{}, nil, latest(), func(c *Collector) { + c.UseTenantAPI(fakeTenantAPI{conns: map[string]*TenantConn{"gpu": tenantConn(loading, mig, cpu)}}) + }) + evs := out[we] + if _, n := find(evs, usage.MetricGPUHours); n != 0 { + t.Fatal("GPUs that are not allocatable were billed") + } + for _, name := range []string{"gpu-a100-1", "gpu-a100-2"} { + if got, n := find(evs, usage.MetricGPUDowntimeHours, "resource_id", name, "sku", "NVIDIA-A100-SXM4-40GB"); n != 1 || !near(got, usage.Round(1.0/60)) { + t.Errorf("%s GPU downtime = %v (%d events), want one GPU for the window", name, got, n) + } + } + if _, n := find(evs, usage.MetricGPUDowntimeHours, "resource_id", "cpu-1"); n != 0 { + t.Error("a node without GPUs reported GPU downtime") + } + if got, _ := find(evs, usage.MetricPrivateNodeHours, "resource_id", "gpu-a100-1"); !near(got, usage.Round(1.0/60)) { + t.Errorf("the Ready node itself is still billed: node hours = %v", got) + } +} diff --git a/internal/pipeline/dispatcher.go b/internal/pipeline/dispatcher.go new file mode 100644 index 0000000..780f203 --- /dev/null +++ b/internal/pipeline/dispatcher.go @@ -0,0 +1,330 @@ +// Package pipeline delivers ledger records to every configured destination. +// +// Each destination gets its own goroutine and its own cursor into the +// spool, so a backend that is down or slow only delays itself. Delivery is +// at-least-once; event IDs are deterministic, so backends deduplicate. +package pipeline + +import ( + "context" + "errors" + "fmt" + "log" + "math/rand" + "sort" + "sync" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/telemetry" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// Options tune delivery. +type Options struct { + BatchSize int // records read per delivery attempt + MinBackoff time.Duration // first retry delay + MaxBackoff time.Duration // retry delay cap + IdlePoll time.Duration // fallback poll when no append wakes us + // Bootstrap, when set, must succeed for a destination before any event + // is delivered to it (meters and billable metrics must exist first, or + // some backends store events without attributing them). + Bootstrap func(context.Context, destinations.Destination) error + // Tenants, when set, is used to create each tenant in a destination + // before its first event reaches it. + Tenants *TenantRegistry +} + +func (o *Options) defaults() { + if o.BatchSize <= 0 { + o.BatchSize = 500 + } + if o.MinBackoff <= 0 { + o.MinBackoff = time.Second + } + if o.MaxBackoff <= 0 { + o.MaxBackoff = 5 * time.Minute + } + if o.IdlePoll <= 0 { + o.IdlePoll = 5 * time.Second + } +} + +// Status is a destination's delivery state. +type Status struct { + Name string `json:"name"` + Ready bool `json:"ready"` + Cursor uint64 `json:"cursor"` + Committed uint64 `json:"committed"` + LagRecords uint64 `json:"lag_records"` + Delivered uint64 `json:"delivered_events"` + Skipped uint64 `json:"skipped_events"` + DeadLettered uint64 `json:"dead_lettered_events"` + LastSuccess time.Time `json:"last_success,omitempty"` + LastError string `json:"last_error,omitempty"` + LastErrorAt time.Time `json:"last_error_at,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures"` + OldestPending time.Time `json:"oldest_pending,omitempty"` +} + +// Dispatcher fans the ledger out to destinations. +type Dispatcher struct { + spool *spool.Spool + dests []destinations.Destination + opts Options + tel *telemetry.Registry + + mu sync.Mutex + status map[string]*Status +} + +// New builds a dispatcher. tel may be nil. +func New(s *spool.Spool, dests []destinations.Destination, opts Options, tel *telemetry.Registry) *Dispatcher { + opts.defaults() + if tel == nil { + tel = telemetry.New() + } + d := &Dispatcher{spool: s, dests: dests, opts: opts, tel: tel, status: map[string]*Status{}} + for _, dest := range dests { + d.status[dest.Name()] = &Status{Name: dest.Name()} + } + return d +} + +// Run delivers until ctx is cancelled. +func (d *Dispatcher) Run(ctx context.Context) { + var wg sync.WaitGroup + for _, dest := range d.dests { + wg.Add(1) + go func(dest destinations.Destination) { + defer wg.Done() + d.loop(ctx, dest) + }(dest) + } + wg.Wait() +} + +// Statuses returns a snapshot sorted by name. +func (d *Dispatcher) Statuses() []Status { + committed := d.spool.Committed() + d.mu.Lock() + defer d.mu.Unlock() + out := make([]Status, 0, len(d.status)) + for _, st := range d.status { + s := *st + s.Committed = committed + if committed > s.Cursor { + s.LagRecords = committed - s.Cursor + } + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// Destinations returns the configured destinations. +func (d *Dispatcher) Destinations() []destinations.Destination { return d.dests } + +func (d *Dispatcher) update(name string, fn func(*Status)) { + d.mu.Lock() + defer d.mu.Unlock() + fn(d.status[name]) +} + +func (d *Dispatcher) loop(ctx context.Context, dest destinations.Destination) { + name := dest.Name() + labels := map[string]string{"destination": name} + cursor := d.spool.Cursor(name) + d.update(name, func(s *Status) { s.Cursor = cursor }) + var backoff time.Duration + + if d.opts.Bootstrap != nil { + for ctx.Err() == nil { + err := d.opts.Bootstrap(ctx, dest) + if err == nil { + break + } + log.Printf("[pipeline] %s: bootstrap failed (delivery paused, will retry): %v", name, err) + d.update(name, func(s *Status) { s.LastError, s.LastErrorAt = "bootstrap: "+err.Error(), time.Now().UTC() }) + backoff = d.sleepBackoff(ctx, backoff) + } + backoff = 0 + } + d.update(name, func(s *Status) { s.Ready = true }) + + for ctx.Err() == nil { + cursor = d.spool.Cursor(name) // re-read so an operator rewind takes effect + changed := d.spool.Changed() // taken before Read so no append is missed + recs, err := d.spool.Read(cursor, d.opts.BatchSize) + if err != nil { + log.Printf("[pipeline] %s: read spool: %v", name, err) + backoff = d.sleepBackoff(ctx, backoff) + continue + } + if len(recs) == 0 { + d.tel.Set("vbilling_destination_lag_seconds", "Age of the oldest event not yet delivered.", labels, 0) + select { + case <-ctx.Done(): + return + case <-changed: + case <-time.After(d.opts.IdlePoll): + } + continue + } + + var events []usage.Event + var skipped uint64 + for _, r := range recs { + if r.Type != spool.TypeEvent || r.Event == nil { + continue + } + if !destinations.Accepts(dest, r.Event.Metric) { + skipped++ + continue + } + events = append(events, *r.Event) + } + if len(events) > 0 { + oldest := events[0].RecordedAt + d.update(name, func(s *Status) { s.OldestPending = oldest }) + d.tel.Set("vbilling_destination_lag_seconds", "Age of the oldest event not yet delivered.", labels, time.Since(oldest).Seconds()) + + dead, err := d.deliver(ctx, dest, events) + if err != nil { + if ctx.Err() != nil { + return + } + log.Printf("[pipeline] %s: delivery failed (will retry): %v", name, err) + d.tel.Add("vbilling_delivery_failures_total", "Failed delivery attempts.", labels, 1) + d.update(name, func(s *Status) { + s.LastError = err.Error() + s.LastErrorAt = time.Now().UTC() + s.ConsecutiveFailures++ + }) + backoff = d.sleepBackoff(ctx, backoff) + continue + } + delivered := uint64(len(events) - dead) + d.tel.Add("vbilling_events_delivered_total", "Events accepted by a destination.", labels, float64(delivered)) + if dead > 0 { + d.tel.Add("vbilling_events_dead_lettered_total", "Events a destination permanently rejected.", labels, float64(dead)) + } + d.update(name, func(s *Status) { + s.Delivered += delivered + s.DeadLettered += uint64(dead) + }) + } + + last := recs[len(recs)-1].Seq + if err := d.spool.Ack(name, last); err != nil { + log.Printf("[pipeline] %s: ack %d: %v", name, last, err) + backoff = d.sleepBackoff(ctx, backoff) + continue + } + cursor = last + backoff = 0 + d.update(name, func(s *Status) { + s.Cursor = cursor + s.Skipped += skipped + s.LastSuccess = time.Now().UTC() + s.ConsecutiveFailures = 0 + s.OldestPending = time.Time{} + }) + d.tel.Set("vbilling_destination_cursor", "Last ledger sequence acknowledged by a destination.", labels, float64(cursor)) + } +} + +// ensureTenants creates any tenant in the batch that this destination has +// not seen yet. Permanent failures are logged and delivery continues (the +// events may then be dead-lettered); anything else is retried. +func (d *Dispatcher) ensureTenants(ctx context.Context, dest destinations.Destination, events []usage.Event) error { + if d.opts.Tenants == nil { + return nil + } + seen := map[string]bool{} + for _, e := range events { + if seen[e.Tenant] { + continue + } + seen[e.Tenant] = true + t, _ := d.opts.Tenants.Get(e.Tenant) + if !d.opts.Tenants.NeedsEnsure(dest.Name(), t) { + continue + } + if err := dest.EnsureTenant(ctx, t); err != nil { + if destinations.IsPermanent(err) { + log.Printf("[pipeline] %s: tenant %s rejected: %v", dest.Name(), t.ID, err) + continue + } + return fmt.Errorf("ensure tenant %s: %w", t.ID, err) + } + d.opts.Tenants.MarkEnsured(dest.Name(), t) + } + return nil +} + +// deliver sends events, bisecting on permanent errors to isolate the events +// the backend rejects. Isolated events are dead-lettered; it returns how many. +func (d *Dispatcher) deliver(ctx context.Context, dest destinations.Destination, events []usage.Event) (int, error) { + if err := d.ensureTenants(ctx, dest, events); err != nil { + return 0, err + } + err := dest.SendEvents(ctx, events) + if err == nil { + return 0, nil + } + var partial *destinations.PartialError + if errors.As(err, &partial) { + dead := 0 + for _, ev := range events { + reason, rejected := partial.Rejected[ev.ID] + if !rejected { + continue + } + log.Printf("[pipeline] %s: dead-lettering event %s (%s/%s): %s", dest.Name(), ev.ID, ev.Tenant, ev.Metric, reason) + if derr := d.spool.AddDeadLetter(dest.Name(), ev, reason); derr != nil { + return 0, derr + } + dead++ + } + return dead, nil + } + if !destinations.IsPermanent(err) { + return 0, err + } + if len(events) == 1 { + log.Printf("[pipeline] %s: dead-lettering event %s (%s/%s): %v", + dest.Name(), events[0].ID, events[0].Tenant, events[0].Metric, err) + if derr := d.spool.AddDeadLetter(dest.Name(), events[0], err.Error()); derr != nil { + return 0, derr + } + return 1, nil + } + mid := len(events) / 2 + a, err := d.deliver(ctx, dest, events[:mid]) + if err != nil { + return 0, err + } + b, err := d.deliver(ctx, dest, events[mid:]) + if err != nil { + return 0, err + } + return a + b, nil +} + +func (d *Dispatcher) sleepBackoff(ctx context.Context, prev time.Duration) time.Duration { + next := prev * 2 + if next < d.opts.MinBackoff { + next = d.opts.MinBackoff + } + if next > d.opts.MaxBackoff { + next = d.opts.MaxBackoff + } + jitter := time.Duration(rand.Int63n(int64(next)/5 + 1)) + select { + case <-ctx.Done(): + case <-time.After(next + jitter): + } + return next +} diff --git a/internal/pipeline/dispatcher_test.go b/internal/pipeline/dispatcher_test.go new file mode 100644 index 0000000..7c8c109 --- /dev/null +++ b/internal/pipeline/dispatcher_test.go @@ -0,0 +1,316 @@ +package pipeline + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/destinations" + "github.com/vclusterlabs-experiments/vbilling/internal/spool" + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +type fakeDest struct { + ensured []string + ensureErr int // fail EnsureTenant this many times + partial bool // report poison events with PartialError instead of failing the batch + name string + mu sync.Mutex + got map[string]int // event ID -> times received + failN int // fail the first N calls with a retryable error + calls int + poison map[string]bool // event IDs rejected permanently + billable bool +} + +func newFake(name string) *fakeDest { + return &fakeDest{name: name, got: map[string]int{}, poison: map[string]bool{}} +} + +func (f *fakeDest) Name() string { return f.name } +func (f *fakeDest) Bootstrap(context.Context, []usage.MetricDef) error { return nil } +func (f *fakeDest) EnsureTenant(_ context.Context, t usage.Tenant) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.ensureErr > 0 { + f.ensureErr-- + return errors.New("billing API timeout") + } + f.ensured = append(f.ensured, t.ID) + return nil +} +func (f *fakeDest) RemoveTenant(context.Context, usage.Tenant) error { return nil } +func (f *fakeDest) Accepts(metric string) bool { + return !f.billable || destinations.BillableOnly(metric) +} +func (f *fakeDest) SendEvents(_ context.Context, evs []usage.Event) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + if f.calls <= f.failN { + return errors.New("503 service unavailable") + } + if f.partial { + rejected := map[string]string{} + for _, e := range evs { + if f.poison[e.ID] { + rejected[e.ID] = "invalid customer" + continue + } + f.got[e.ID]++ + } + if len(rejected) > 0 { + return &destinations.PartialError{Rejected: rejected} + } + return nil + } + for _, e := range evs { + if f.poison[e.ID] { + return destinations.Permanent(fmt.Errorf("customer for %s not found", e.Tenant)) + } + } + for _, e := range evs { + f.got[e.ID]++ + } + return nil +} + +func (f *fakeDest) received() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.got) +} + +var t0 = time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) + +func fill(t *testing.T, s *spool.Spool, windows, perWindow int, metric string) []usage.Event { + t.Helper() + start := t0 + if wm, ok := s.Watermark(spool.SourceCollector); ok { + start = wm // continue after whatever is already committed + } + var all []usage.Event + for w := 0; w < windows; w++ { + ws := start.Add(time.Duration(w) * time.Minute) + var evs []usage.Event + for i := 0; i < perWindow; i++ { + e := usage.Event{Tenant: fmt.Sprintf("t%d", i), Metric: metric, Quantity: 1, WindowStart: ws, WindowEnd: ws.Add(time.Minute), RecordedAt: time.Now()} + e.Finalize() + evs = append(evs, e) + } + if err := s.AppendWindow(spool.SourceCollector, ws.Add(time.Minute), evs, ""); err != nil { + t.Fatal(err) + } + all = append(all, evs...) + } + return all +} + +func run(t *testing.T, s *spool.Spool, until func() bool, dests ...destinations.Destination) *Dispatcher { + t.Helper() + d := New(s, dests, Options{BatchSize: 7, MinBackoff: time.Millisecond, MaxBackoff: 5 * time.Millisecond, IdlePoll: 10 * time.Millisecond}, nil) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan struct{}) + go func() { d.Run(ctx); close(done) }() + for !until() { + if ctx.Err() != nil { + t.Fatal("timed out waiting for delivery") + } + time.Sleep(2 * time.Millisecond) + } + cancel() + <-done + return d +} + +func openSpool(t *testing.T, dir string) *spool.Spool { + t.Helper() + s, err := spool.Open(dir, spool.Options{NoSync: true}) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestDeliversEverythingAndAcks(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + dest := newFake("stripe") + s.Cursor("stripe") // register before data so it starts at seq 0 + all := fill(t, s, 5, 4, usage.MetricGPUHours) + + run(t, s, func() bool { return dest.received() == len(all) }, dest) + for _, e := range all { + if dest.got[e.ID] != 1 { + t.Fatalf("event %s received %d times", e.ID, dest.got[e.ID]) + } + } + if s.Cursor("stripe") != s.Committed() { + t.Fatalf("cursor %d != committed %d", s.Cursor("stripe"), s.Committed()) + } +} + +func TestRetriesTransientFailuresWithoutLoss(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + dest := newFake("metronome") + dest.failN = 4 + s.Cursor("metronome") + all := fill(t, s, 3, 3, usage.MetricGPUHours) + + d := run(t, s, func() bool { return dest.received() == len(all) }, dest) + st := d.Statuses()[0] + if st.DeadLettered != 0 || st.Delivered != uint64(len(all)) { + t.Fatalf("status after retries: %+v", st) + } +} + +func TestPoisonEventIsIsolatedAndDeadLettered(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + dest := newFake("stripe") + s.Cursor("stripe") + all := fill(t, s, 2, 5, usage.MetricGPUHours) + bad := all[3] + dest.poison[bad.ID] = true + + d := run(t, s, func() bool { return dest.received() == len(all)-1 && s.Cursor("stripe") == s.Committed() }, dest) + if dest.got[bad.ID] != 0 { + t.Fatal("poison event delivered") + } + dls, _ := s.DeadLetters("stripe") + if len(dls) != 1 || dls[0].Event.ID != bad.ID { + t.Fatalf("dead letters: %+v", dls) + } + if st := d.Statuses()[0]; st.DeadLettered != 1 { + t.Fatalf("status: %+v", st) + } +} + +func TestOneDestinationDownDoesNotBlockAnother(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + up, down := newFake("webhook"), newFake("metronome") + down.failN = 1 << 30 + s.Cursor("webhook") + s.Cursor("metronome") + all := fill(t, s, 4, 2, usage.MetricCPUCoreHours) + + run(t, s, func() bool { return up.received() == len(all) }, up, down) + if down.received() != 0 { + t.Fatal("failing destination should have received nothing") + } + if s.Cursor("metronome") != 0 { + t.Fatalf("failing destination cursor advanced to %d", s.Cursor("metronome")) + } +} + +func TestFilterSkipsInformationalMetrics(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + dest := newFake("stripe") + dest.billable = true + s.Cursor("stripe") + fill(t, s, 2, 2, usage.MetricGPUUtilization) // informational + billable := fill(t, s, 1, 2, usage.MetricGPUHours) + + run(t, s, func() bool { return s.Cursor("stripe") == s.Committed() }, dest) + if dest.received() != len(billable) { + t.Fatalf("received %d events, want only the %d billable ones", dest.received(), len(billable)) + } +} + +func TestResumesFromCursorAfterRestart(t *testing.T) { + dir := t.TempDir() + s := openSpool(t, dir) + first := newFake("lago") + s.Cursor("lago") + batch1 := fill(t, s, 2, 2, usage.MetricGPUHours) + run(t, s, func() bool { return first.received() == len(batch1) }, first) + s.Close() + + s = openSpool(t, dir) + defer s.Close() + second := newFake("lago") + ws := t0.Add(10 * time.Minute) + e := usage.Event{Tenant: "t0", Metric: usage.MetricGPUHours, Quantity: 1, WindowStart: ws, WindowEnd: ws.Add(time.Minute), RecordedAt: time.Now()} + e.Finalize() + if err := s.AppendWindow(spool.SourceCollector, ws.Add(time.Minute), []usage.Event{e}, ""); err != nil { + t.Fatal(err) + } + run(t, s, func() bool { return second.received() == 1 }, second) + if second.got[e.ID] != 1 { + t.Fatal("after restart only the new event should be delivered") + } +} + +func TestPartialErrorDeadLettersOnlyRejectedEvents(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + dest := newFake("metronome") + dest.partial = true + s.Cursor("metronome") + all := fill(t, s, 1, 6, usage.MetricGPUHours) + dest.poison[all[1].ID] = true + dest.poison[all[4].ID] = true + + d := run(t, s, func() bool { return s.Cursor("metronome") == s.Committed() }, dest) + if dest.received() != 4 { + t.Fatalf("received %d, want 4", dest.received()) + } + if dest.calls != 1 { + t.Fatalf("partial rejection should not trigger resends, got %d calls", dest.calls) + } + if st := d.Statuses()[0]; st.DeadLettered != 2 || st.Delivered != 4 { + t.Fatalf("status: %+v", st) + } +} + +func TestBootstrapGateAndTenantEnsuredBeforeEvents(t *testing.T) { + s := openSpool(t, t.TempDir()) + defer s.Close() + dest := newFake("metronome") + dest.ensureErr = 2 + s.Cursor("metronome") + all := fill(t, s, 2, 3, usage.MetricGPUHours) + + reg := NewTenantRegistry() + reg.Upsert(usage.Tenant{ID: "t0", DisplayName: "Tenant Zero"}) + bootstrapCalls := 0 + d := New(s, []destinations.Destination{dest}, Options{ + BatchSize: 100, MinBackoff: time.Millisecond, MaxBackoff: 2 * time.Millisecond, IdlePoll: 5 * time.Millisecond, + Tenants: reg, + Bootstrap: func(context.Context, destinations.Destination) error { + bootstrapCalls++ + if bootstrapCalls < 3 { + return errors.New("metronome unavailable") + } + return nil + }, + }, nil) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go d.Run(ctx) + for dest.received() < len(all) { + if ctx.Err() != nil { + t.Fatal("timed out") + } + time.Sleep(time.Millisecond) + } + cancel() + if bootstrapCalls != 3 { + t.Fatalf("bootstrap calls = %d", bootstrapCalls) + } + dest.mu.Lock() + defer dest.mu.Unlock() + if len(dest.ensured) != 3 { // t0, t1, t2 once each, after two transient failures + t.Fatalf("ensured = %v", dest.ensured) + } + if reg.NeedsEnsure("metronome", usage.Tenant{ID: "t0", DisplayName: "Tenant Zero"}) { + t.Fatal("tenant not marked ensured") + } +} diff --git a/internal/pipeline/tenants.go b/internal/pipeline/tenants.go new file mode 100644 index 0000000..a2ac2bf --- /dev/null +++ b/internal/pipeline/tenants.go @@ -0,0 +1,86 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "sync" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// TenantRegistry is the shared view of billing tenants. The controller +// fills it from discovery; the dispatcher uses it to make sure a tenant +// exists in a destination before that destination sees its first event. +type TenantRegistry struct { + mu sync.Mutex + tenants map[string]usage.Tenant + ensured map[string]map[string]string // destination -> tenant -> fingerprint +} + +func NewTenantRegistry() *TenantRegistry { + return &TenantRegistry{tenants: map[string]usage.Tenant{}, ensured: map[string]map[string]string{}} +} + +func (r *TenantRegistry) Upsert(t usage.Tenant) { + r.mu.Lock() + defer r.mu.Unlock() + r.tenants[t.ID] = t +} + +func (r *TenantRegistry) Delete(id string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.tenants, id) + for _, m := range r.ensured { + delete(m, id) + } +} + +// Get returns a known tenant, or a minimal one for tenants only seen in +// ingested events. +func (r *TenantRegistry) Get(id string) (usage.Tenant, bool) { + r.mu.Lock() + defer r.mu.Unlock() + t, ok := r.tenants[id] + if !ok { + return usage.Tenant{ID: id, DisplayName: id}, false + } + return t, true +} + +// All returns tenants sorted by ID. +func (r *TenantRegistry) All() []usage.Tenant { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]usage.Tenant, 0, len(r.tenants)) + for _, t := range r.tenants { + out = append(out, t) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// NeedsEnsure reports whether t (as it is now) has not yet been ensured in dest. +func (r *TenantRegistry) NeedsEnsure(dest string, t usage.Tenant) bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.ensured[dest][t.ID] != fingerprint(t) +} + +func (r *TenantRegistry) MarkEnsured(dest string, t usage.Tenant) { + r.mu.Lock() + defer r.mu.Unlock() + if r.ensured[dest] == nil { + r.ensured[dest] = map[string]string{} + } + r.ensured[dest][t.ID] = fingerprint(t) +} + +func fingerprint(t usage.Tenant) string { + t.CreatedAt = t.CreatedAt.UTC() + b, _ := json.Marshal(t) // map keys marshal sorted + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:12]) +} diff --git a/internal/spool/bench_test.go b/internal/spool/bench_test.go new file mode 100644 index 0000000..69d3b58 --- /dev/null +++ b/internal/spool/bench_test.go @@ -0,0 +1,45 @@ +package spool + +import ( + "fmt" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// BenchmarkScan measures query scans over 50k events (one hour of 100 +// tenant clusters at ~8 events/minute). +func BenchmarkScan(b *testing.B) { + c := &clock{t: t0} + s, err := Open(b.TempDir(), Options{NoSync: true, Now: c.Now}) + if err != nil { + b.Fatal(err) + } + defer s.Close() + for w := 0; w < 60; w++ { + ws := t0.Add(time.Duration(w) * time.Minute) + c.t = ws.Add(time.Minute) + var evs []usage.Event + for i := 0; i < 833; i++ { + e := usage.Event{Tenant: fmt.Sprintf("tenant-%d", i%100), Metric: usage.MetricGPUHours, Quantity: 0.1, SKU: "NVIDIA-H100", + WindowStart: ws, WindowEnd: ws.Add(time.Minute), Dimensions: map[string]string{"tenant_cluster": fmt.Sprintf("c-%d", i)}} + e.Finalize() + evs = append(evs, e) + } + if err := s.AppendWindow(SourceCollector, ws.Add(time.Minute), evs, ""); err != nil { + b.Fatal(err) + } + } + for _, r := range []struct { + name string + from, to time.Time + }{{"full-hour", t0, t0.Add(time.Hour)}, {"five-minutes", t0.Add(30 * time.Minute), t0.Add(35 * time.Minute)}} { + b.Run(r.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + n := 0 + s.Scan(r.from, r.to, func(*usage.Event) error { n++; return nil }) + } + }) + } +} diff --git a/internal/spool/spool.go b/internal/spool/spool.go new file mode 100644 index 0000000..d9f9a3d --- /dev/null +++ b/internal/spool/spool.go @@ -0,0 +1,1145 @@ +// Package spool is vBilling's durable usage ledger and delivery outbox. +// +// Usage is appended to segment files before any destination sees it. Each +// append is one atomic batch: the events of a window followed by a commit +// record, written with a single write and fsync. On restart, anything after +// the last commit is discarded and the window is collected again, so a crash +// can never produce half a window or a duplicate one. +// +// Every record is hash-chained to the previous one (sha256 over the record's +// JSON, which embeds the previous hash), so the ledger is tamper-evident and +// can be verified end to end. Each destination reads the ledger through its +// own cursor, which is what lets one slow or failing billing backend lag +// without blocking the others. +// +// Line format: "\t\n". +package spool + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +// Record types. +const ( + TypeEvent = "event" + TypeCommit = "commit" +) + +// SourceCollector is the source name the controller commits windows under. +const SourceCollector = "collector" + +// ErrWindowCommitted is returned when a window at or before the source's +// watermark is appended again. +var ErrWindowCommitted = errors.New("window already committed") + +// Record is one ledger entry. +type Record struct { + Seq uint64 `json:"seq"` + Type string `json:"type"` + At time.Time `json:"at"` + Prev string `json:"prev"` + Event *usage.Event `json:"event,omitempty"` + Commit *Commit `json:"commit,omitempty"` + + Hash string `json:"-"` +} + +// Commit closes an atomic batch. +type Commit struct { + Source string `json:"source"` + WindowEnd time.Time `json:"window_end,omitempty"` + Events int `json:"events"` + Note string `json:"note,omitempty"` +} + +// Options tune the spool. Zero values pick defaults. +type Options struct { + SegmentMaxRecords int + SegmentMaxAge time.Duration + // Retention is how long closed segments are kept once every destination + // has consumed them. Zero keeps data forever. + Retention time.Duration + // DedupeWindow bounds how long ingested event IDs are remembered. + DedupeWindow time.Duration + // NoSync skips fsync (tests only). + NoSync bool + Now func() time.Time +} + +func (o *Options) defaults() { + if o.SegmentMaxRecords <= 0 { + o.SegmentMaxRecords = 20000 + } + if o.SegmentMaxAge <= 0 { + o.SegmentMaxAge = time.Hour + } + if o.DedupeWindow <= 0 { + o.DedupeWindow = 48 * time.Hour + } + if o.Now == nil { + o.Now = time.Now + } +} + +type segment struct { + First uint64 `json:"first"` + Last uint64 `json:"last"` // 0 when empty + FirstAt time.Time `json:"first_at"` + LastAt time.Time `json:"last_at"` + LastHash string `json:"last_hash"` + Size int64 `json:"size"` + Name string `json:"name"` + + index []offset // sparse seq -> byte offset, built lazily for closed segments +} + +type offset struct { + seq uint64 + off int64 +} + +const indexEvery = 256 + +type manifest struct { + Segments []*segment `json:"segments"` + Watermarks map[string]time.Time `json:"watermarks"` + Anchor string `json:"anchor"` // prev hash of the first retained record +} + +// Spool is safe for concurrent use. +type Spool struct { + dir string + opts Options + + mu sync.Mutex + segs []*segment + active *os.File + seq uint64 // last written + committed uint64 // readers see records <= committed + head string + anchor string + watermarks map[string]time.Time + cursors map[string]uint64 + recentIDs map[string]time.Time + changed chan struct{} + closed bool +} + +// Open opens or creates a spool in dir and recovers its state. +func Open(dir string, opts Options) (*Spool, error) { + opts.defaults() + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, fmt.Errorf("create spool dir: %w", err) + } + s := &Spool{ + dir: dir, + opts: opts, + watermarks: map[string]time.Time{}, + cursors: map[string]uint64{}, + recentIDs: map[string]time.Time{}, + changed: make(chan struct{}), + } + if err := s.recover(); err != nil { + return nil, err + } + if err := s.loadCursors(); err != nil { + return nil, err + } + if err := s.loadDedupe(); err != nil { + return nil, err + } + if err := s.openActive(); err != nil { + return nil, err + } + return s, nil +} + +func segName(first uint64) string { return fmt.Sprintf("seg-%020d.log", first) } + +func parseSegName(name string) (uint64, bool) { + if !strings.HasPrefix(name, "seg-") || !strings.HasSuffix(name, ".log") { + return 0, false + } + n, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(name, "seg-"), ".log"), 10, 64) + return n, err == nil +} + +// recover rebuilds in-memory state from the manifest and segment files, +// truncating any torn or uncommitted tail of the newest segment. +func (s *Spool) recover() error { + var man manifest + if b, err := os.ReadFile(filepath.Join(s.dir, "manifest.json")); err == nil { + if err := json.Unmarshal(b, &man); err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("read manifest: %w", err) + } + known := map[string]*segment{} + for _, sg := range man.Segments { + known[sg.Name] = sg + } + for k, v := range man.Watermarks { + s.watermarks[k] = v + } + s.anchor = man.Anchor + + entries, err := os.ReadDir(s.dir) + if err != nil { + return fmt.Errorf("list spool dir: %w", err) + } + var names []string + for _, e := range entries { + if _, ok := parseSegName(e.Name()); ok && !e.IsDir() { + names = append(names, e.Name()) + } + } + sort.Strings(names) // zero-padded, so lexical == numeric + + for i, name := range names { + first, _ := parseSegName(name) + info, err := os.Stat(filepath.Join(s.dir, name)) + if err != nil { + return err + } + last := i == len(names)-1 + if sg, ok := known[name]; ok && !last && sg.Size == info.Size() { + s.segs = append(s.segs, sg) + continue + } + sg, err := s.scanSegment(name, first, last) + if err != nil { + return err + } + s.segs = append(s.segs, sg) + } + + // Derive seq/head/committed from the newest non-empty segment. + for i := len(s.segs) - 1; i >= 0; i-- { + if s.segs[i].Last != 0 { + s.seq = s.segs[i].Last + s.head = s.segs[i].LastHash + break + } + } + if s.seq == 0 && len(s.segs) > 0 { + // Every segment is empty; continue numbering from the newest name. + s.seq = s.segs[len(s.segs)-1].First - 1 + s.head = s.anchor + } + s.committed = s.seq + return nil +} + +// scanSegment reads a segment, building its metadata. For the newest segment +// it also truncates a torn line or an uncommitted tail. +func (s *Spool) scanSegment(name string, first uint64, newest bool) (*segment, error) { + path := filepath.Join(s.dir, name) + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + sg := &segment{First: first, Name: name} + r := bufio.NewReaderSize(f, 1<<20) + var ( + off int64 + goodEnd int64 // end of the last committed record + lastCommitSq uint64 + lastCommitAt time.Time + lastCommitH string + pendingWM = map[string]time.Time{} + ) + for { + line, err := r.ReadBytes('\n') + if len(line) == 0 && err == io.EOF { + break + } + if err != nil && err != io.EOF { + return nil, err + } + if err == io.EOF { + // A final line without a newline is a torn write. + if !newest { + return nil, fmt.Errorf("segment %s: torn line at offset %d in a closed segment", name, off) + } + break + } + rec, hash, perr := parseLine(line) + if perr != nil { + // A complete line that fails its hash is corruption or tampering, + // not a crash artifact. Refuse to guess. + return nil, fmt.Errorf("segment %s corrupt at offset %d: %v", name, off, perr) + } + if (rec.Seq-first)%indexEvery == 0 { + sg.index = append(sg.index, offset{seq: rec.Seq, off: off}) + } + off += int64(len(line)) + if rec.Type == TypeCommit && rec.Commit != nil { + goodEnd = off + lastCommitSq = rec.Seq + lastCommitAt = rec.At + lastCommitH = hash + if !rec.Commit.WindowEnd.IsZero() { + pendingWM[rec.Commit.Source] = rec.Commit.WindowEnd + } + } + if sg.FirstAt.IsZero() { + sg.FirstAt = rec.At + } + } + if newest && info.Size() > goodEnd { + // Drop the torn or uncommitted tail; that window gets re-collected. + if err := f.Truncate(goodEnd); err != nil { + return nil, fmt.Errorf("truncate %s: %w", name, err) + } + // Index entries past the truncation point are invalid. + kept := sg.index[:0] + for _, ix := range sg.index { + if ix.off < goodEnd { + kept = append(kept, ix) + } + } + sg.index = kept + off = goodEnd + } + if lastCommitSq != 0 { + sg.Last = lastCommitSq + sg.LastAt = lastCommitAt + sg.LastHash = lastCommitH + } else { + sg.FirstAt = time.Time{} + } + sg.Size = off + for k, v := range pendingWM { + if v.After(s.watermarks[k]) { + s.watermarks[k] = v + } + } + return sg, nil +} + +func parseLine(line []byte) (*Record, string, error) { + line = bytes.TrimSuffix(line, []byte("\n")) + tab := bytes.IndexByte(line, '\t') + if tab != 64 { + return nil, "", errors.New("malformed line") + } + hash := string(line[:tab]) + body := line[tab+1:] + sum := sha256.Sum256(body) + if hex.EncodeToString(sum[:]) != hash { + return nil, "", errors.New("hash mismatch") + } + var rec Record + if err := json.Unmarshal(body, &rec); err != nil { + return nil, "", err + } + rec.Hash = hash + return &rec, hash, nil +} + +func (s *Spool) openActive() error { + if len(s.segs) == 0 { + sg := &segment{First: s.seq + 1, Name: segName(s.seq + 1)} + s.segs = append(s.segs, sg) + } + sg := s.segs[len(s.segs)-1] + f, err := os.OpenFile(filepath.Join(s.dir, sg.Name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + if err != nil { + return fmt.Errorf("open active segment: %w", err) + } + s.active = f + return nil +} + +// Changed returns a channel closed on the next successful append. +func (s *Spool) Changed() <-chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + return s.changed +} + +// Watermark returns the last committed window end for a source. +func (s *Spool) Watermark(source string) (time.Time, bool) { + s.mu.Lock() + defer s.mu.Unlock() + t, ok := s.watermarks[source] + return t, ok +} + +// Committed returns the highest committed sequence number. +func (s *Spool) Committed() uint64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.committed +} + +// AppendWindow atomically appends a window's events for a source. A zero +// windowEnd skips watermark checks (ingested batches). +func (s *Spool) AppendWindow(source string, windowEnd time.Time, events []usage.Event, note string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return errors.New("spool closed") + } + if !windowEnd.IsZero() { + if wm, ok := s.watermarks[source]; ok && !windowEnd.After(wm) { + return fmt.Errorf("%w: %s <= %s", ErrWindowCommitted, windowEnd.Format(time.RFC3339), wm.Format(time.RFC3339)) + } + } + return s.appendLocked(source, windowEnd.UTC(), events, note) +} + +// AppendIngest appends externally supplied events, dropping IDs seen within +// the dedupe window. It returns the IDs that were accepted and the duplicates. +func (s *Spool) AppendIngest(source string, events []usage.Event) (accepted, duplicates []string, err error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nil, nil, errors.New("spool closed") + } + now := s.opts.Now() + seen := map[string]bool{} + var keep []usage.Event + for _, ev := range events { + if _, dup := s.recentIDs[ev.ID]; dup || seen[ev.ID] { + duplicates = append(duplicates, ev.ID) + continue + } + seen[ev.ID] = true + keep = append(keep, ev) + } + if len(keep) == 0 { + return nil, duplicates, nil + } + if err := s.appendLocked(source, time.Time{}, keep, ""); err != nil { + return nil, nil, err + } + if err := s.recordDedupe(keep, now); err != nil { + return nil, nil, err + } + for _, ev := range keep { + accepted = append(accepted, ev.ID) + } + return accepted, duplicates, nil +} + +func (s *Spool) appendLocked(source string, windowEnd time.Time, events []usage.Event, note string) error { + if err := s.maybeRotateLocked(); err != nil { + return err + } + now := s.opts.Now().UTC() + sg := s.segs[len(s.segs)-1] + startSize := sg.Size + seq, head := s.seq, s.head + + var buf bytes.Buffer + var newIndex []offset + write := func(rec *Record) error { + seq++ + rec.Seq = seq + rec.At = now + rec.Prev = head + body, err := json.Marshal(rec) + if err != nil { + return err + } + sum := sha256.Sum256(body) + head = hex.EncodeToString(sum[:]) + if (seq-sg.First)%indexEvery == 0 { + newIndex = append(newIndex, offset{seq: seq, off: startSize + int64(buf.Len())}) + } + buf.WriteString(head) + buf.WriteByte('\t') + buf.Write(body) + buf.WriteByte('\n') + return nil + } + for i := range events { + ev := events[i] + if err := write(&Record{Type: TypeEvent, Event: &ev}); err != nil { + return fmt.Errorf("encode event: %w", err) + } + } + if err := write(&Record{Type: TypeCommit, Commit: &Commit{Source: source, WindowEnd: windowEnd, Events: len(events), Note: note}}); err != nil { + return err + } + + if _, err := s.active.Write(buf.Bytes()); err != nil { + _ = s.active.Truncate(startSize) + return fmt.Errorf("write spool: %w", err) + } + if !s.opts.NoSync { + if err := s.active.Sync(); err != nil { + _ = s.active.Truncate(startSize) + return fmt.Errorf("fsync spool: %w", err) + } + } + + if sg.Last == 0 { + sg.FirstAt = now + } + sg.Last = seq + sg.LastAt = now + sg.LastHash = head + sg.Size = startSize + int64(buf.Len()) + sg.index = append(sg.index, newIndex...) + s.seq, s.head, s.committed = seq, head, seq + if !windowEnd.IsZero() { + s.watermarks[source] = windowEnd + } + close(s.changed) + s.changed = make(chan struct{}) + return nil +} + +func (s *Spool) maybeRotateLocked() error { + sg := s.segs[len(s.segs)-1] + if sg.Last == 0 { + return nil + } + count := int(sg.Last-sg.First) + 1 + if count < s.opts.SegmentMaxRecords && s.opts.Now().Sub(sg.FirstAt) < s.opts.SegmentMaxAge { + return nil + } + if err := s.active.Close(); err != nil { + return err + } + next := &segment{First: s.seq + 1, Name: segName(s.seq + 1)} + s.segs = append(s.segs, next) + f, err := os.OpenFile(filepath.Join(s.dir, next.Name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + if err != nil { + return err + } + s.active = f + return s.writeManifestLocked() +} + +func (s *Spool) writeManifestLocked() error { + man := manifest{Watermarks: s.watermarks, Anchor: s.anchor} + for _, sg := range s.segs[:len(s.segs)-1] { // closed segments only + man.Segments = append(man.Segments, sg) + } + return writeJSONAtomic(filepath.Join(s.dir, "manifest.json"), man, s.opts.NoSync) +} + +// snapshot copies segment metadata under the lock so readers never race the +// writer on the active segment. +func (s *Spool) snapshot() (uint64, []segment) { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]segment, len(s.segs)) + for i, sg := range s.segs { + out[i] = *sg + } + return s.committed, out +} + +var ( + seqPrefix = []byte(`{"seq":`) + windowStartMarker = []byte(`"window_start":"`) + windowEndMarker = []byte(`"window_end":"`) +) + +// peekWindow finds an event record's window without decoding it. Commit +// records have no window and report ok=false. +func peekWindow(line []byte) (ws, we time.Time, ok bool) { + at := func(marker []byte) (time.Time, bool) { + i := bytes.Index(line, marker) + if i < 0 { + return time.Time{}, false + } + rest := line[i+len(marker):] + j := bytes.IndexByte(rest, '"') + if j < 0 { + return time.Time{}, false + } + t, err := time.Parse(time.RFC3339Nano, string(rest[:j])) + return t, err == nil + } + ws, ok1 := at(windowStartMarker) + we, ok2 := at(windowEndMarker) + return ws, we, ok1 && ok2 +} + +// parseLineNoVerify decodes a line without re-hashing it (queries only). +func parseLineNoVerify(line []byte) (*Record, error) { + line = bytes.TrimSuffix(line, []byte("\n")) + if len(line) < 66 || line[64] != '\t' { + return nil, errors.New("malformed line") + } + var rec Record + if err := json.Unmarshal(line[65:], &rec); err != nil { + return nil, err + } + rec.Hash = string(line[:64]) + return &rec, nil +} + +// peekSeq extracts the sequence number without decoding the record. Record +// marshals Seq first, so the body always starts with {"seq":N, +func peekSeq(line []byte) (uint64, bool) { + if len(line) < 65+len(seqPrefix) || !bytes.HasPrefix(line[65:], seqPrefix) { + return 0, false + } + rest := line[65+len(seqPrefix):] + end := bytes.IndexByte(rest, ',') + if end <= 0 { + return 0, false + } + n, err := strconv.ParseUint(string(rest[:end]), 10, 64) + return n, err == nil +} + +// Read returns up to max committed records with seq > after. +func (s *Spool) Read(after uint64, max int) ([]Record, error) { + committed, segs := s.snapshot() + if after >= committed || max <= 0 { + return nil, nil + } + // Empty segments only ever sit at the end, so this predicate is monotonic. + i := sort.Search(len(segs), func(i int) bool { return segs[i].Last == 0 || segs[i].Last > after }) + var out []Record + for ; i < len(segs) && len(out) < max; i++ { + err := s.readSegment(&segs[i], after+1, committed, func(r *Record) bool { + out = append(out, *r) + return len(out) < max + }) + if err != nil { + return nil, err + } + } + return out, nil +} + +// readSegment streams records with from <= seq <= upto from one segment. +// Delivery reads verify each line's hash; read-only queries can skip it and +// pass a prefilter that rejects lines before they are decoded. +func (s *Spool) readSegment(sg *segment, from, upto uint64, fn func(*Record) bool) error { + return s.readSegmentOpts(sg, from, upto, true, nil, fn) +} + +func (s *Spool) readSegmentOpts(sg *segment, from, upto uint64, verify bool, keep func(line []byte) bool, fn func(*Record) bool) error { + if sg.Last == 0 || sg.Last < from || sg.First > upto { + return nil + } + f, err := os.Open(filepath.Join(s.dir, sg.Name)) + if err != nil { + return err + } + defer f.Close() + + var start int64 + for _, ix := range sg.index { + if ix.seq > from { + break + } + start = ix.off + } + if _, err := f.Seek(start, io.SeekStart); err != nil { + return err + } + r := bufio.NewReaderSize(f, 1<<20) + for { + line, err := r.ReadBytes('\n') + if err != nil { + return nil // EOF or torn line beyond the committed point + } + // Skip cheaply until the first wanted record. + seq, seqOK := peekSeq(line) + if seqOK && seq < from { + continue + } + if seqOK && seq > upto { + return nil + } + if keep != nil && !keep(line) { + continue + } + var rec *Record + var perr error + if verify { + rec, _, perr = parseLine(line) + } else { + rec, perr = parseLineNoVerify(line) + } + if perr != nil { + return fmt.Errorf("segment %s: %w", sg.Name, perr) + } + if rec.Seq > upto { + return nil + } + if rec.Seq < from { + continue + } + if !fn(rec) { + return nil + } + } +} + +// scanSkew bounds how far before its window an event can be appended. The +// ingest API rejects windows ending more than a few minutes in the future; +// the margin keeps Scan correct even with skewed clients. +const scanSkew = time.Hour + +// Scan visits committed event records whose window overlaps [from, to). +// Events are appended at or after their window ends, so segments written +// well before `from` are skipped. +func (s *Spool) Scan(from, to time.Time, fn func(*usage.Event) error) error { + committed, segs := s.snapshot() + for i := range segs { + sg := &segs[i] + if sg.Last == 0 || sg.LastAt.Before(from.Add(-scanSkew)) { + continue + } + var cbErr error + inRange := func(line []byte) bool { + ws, we, ok := peekWindow(line) + return ok && we.After(from) && ws.Before(to) + } + err := s.readSegmentOpts(sg, sg.First, committed, false, inRange, func(r *Record) bool { + if r.Type != TypeEvent || r.Event == nil { + return true + } + e := r.Event + if e.WindowEnd.After(from) && e.WindowStart.Before(to) { + if cbErr = fn(e); cbErr != nil { + return false + } + } + return true + }) + if err != nil { + return err + } + if cbErr != nil { + return cbErr + } + } + return nil +} + +// Cursor returns a destination's last acknowledged sequence number. A +// destination seen for the first time starts at the current head, so adding +// a backend never replays history into it unless asked (see SetCursor). +func (s *Spool) Cursor(dest string) uint64 { + s.mu.Lock() + defer s.mu.Unlock() + c, ok := s.cursors[dest] + if !ok { + c = s.committed + s.cursors[dest] = c + _ = s.writeCursorsLocked() + } + return c +} + +// SetCursor moves a destination's cursor, e.g. to replay from the start. +func (s *Spool) SetCursor(dest string, seq uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + if seq > s.committed { + seq = s.committed + } + s.cursors[dest] = seq + return s.writeCursorsLocked() +} + +// Ack records that a destination has durably received everything <= seq. +func (s *Spool) Ack(dest string, seq uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + if seq <= s.cursors[dest] { + return nil + } + s.cursors[dest] = seq + return s.writeCursorsLocked() +} + +// FirstSeq returns the oldest retained sequence number (0 if empty). +func (s *Spool) FirstSeq() uint64 { + s.mu.Lock() + defer s.mu.Unlock() + for _, sg := range s.segs { + if sg.Last != 0 { + return sg.First + } + } + return 0 +} + +func (s *Spool) writeCursorsLocked() error { + return writeJSONAtomic(filepath.Join(s.dir, "cursors.json"), s.cursors, s.opts.NoSync) +} + +func (s *Spool) loadCursors() error { + b, err := os.ReadFile(filepath.Join(s.dir, "cursors.json")) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if err := json.Unmarshal(b, &s.cursors); err != nil { + return fmt.Errorf("parse cursors: %w", err) + } + for k, v := range s.cursors { + if v > s.committed { // cursor ahead of a truncated tail + s.cursors[k] = s.committed + } + } + return nil +} + +// --- ingest dedupe index --- + +func (s *Spool) recordDedupe(events []usage.Event, now time.Time) error { + f, err := os.OpenFile(filepath.Join(s.dir, "ingest-ids.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + if err != nil { + return err + } + defer f.Close() + var buf bytes.Buffer + for _, ev := range events { + s.recentIDs[ev.ID] = now + fmt.Fprintf(&buf, "%d\t%s\n", now.Unix(), ev.ID) + } + if _, err := f.Write(buf.Bytes()); err != nil { + return err + } + if !s.opts.NoSync { + return f.Sync() + } + return nil +} + +func (s *Spool) loadDedupe() error { + path := filepath.Join(s.dir, "ingest-ids.log") + b, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + cutoff := s.opts.Now().Add(-s.opts.DedupeWindow) + var keep bytes.Buffer + for _, line := range strings.Split(string(b), "\n") { + ts, id, ok := strings.Cut(line, "\t") + if !ok { + continue + } + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + continue + } + at := time.Unix(sec, 0) + if at.Before(cutoff) { + continue + } + s.recentIDs[id] = at + keep.WriteString(line) + keep.WriteByte('\n') + } + return writeFileAtomic(path, keep.Bytes(), s.opts.NoSync) +} + +// --- retention --- + +// Compact deletes closed segments older than the retention period that +// every listed destination has already consumed. +func (s *Spool) Compact(consumers []string) (removed int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.opts.Retention <= 0 || len(s.segs) < 2 { + return 0, s.compactDedupeLocked() + } + minCursor := s.committed + for _, c := range consumers { + cur, ok := s.cursors[c] + if !ok { + cur = s.committed + } + if cur < minCursor { + minCursor = cur + } + } + cutoff := s.opts.Now().Add(-s.opts.Retention) + keepFrom := 0 + for i, sg := range s.segs[:len(s.segs)-1] { + if sg.Last != 0 && (sg.LastAt.After(cutoff) || sg.Last > minCursor) { + break + } + if err := os.Remove(filepath.Join(s.dir, sg.Name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return removed, err + } + if sg.LastHash != "" { + s.anchor = sg.LastHash + } + removed++ + keepFrom = i + 1 + } + if err := s.compactDedupeLocked(); err != nil { + return removed, err + } + if removed > 0 { + s.segs = append([]*segment(nil), s.segs[keepFrom:]...) + return removed, s.writeManifestLocked() + } + return 0, nil +} + +func (s *Spool) compactDedupeLocked() error { + cutoff := s.opts.Now().Add(-s.opts.DedupeWindow) + pruned := false + for id, at := range s.recentIDs { + if at.Before(cutoff) { + delete(s.recentIDs, id) + pruned = true + } + } + if !pruned { + return nil + } + var buf bytes.Buffer + for id, at := range s.recentIDs { + fmt.Fprintf(&buf, "%d\t%s\n", at.Unix(), id) + } + return writeFileAtomic(filepath.Join(s.dir, "ingest-ids.log"), buf.Bytes(), s.opts.NoSync) +} + +// --- verification --- + +// VerifyReport summarizes a ledger integrity check. +type VerifyReport struct { + OK bool `json:"ok"` + Records uint64 `json:"records"` + Segments int `json:"segments"` + FirstSeq uint64 `json:"first_seq"` + LastSeq uint64 `json:"last_seq"` + Head string `json:"head"` + Errors []string `json:"errors,omitempty"` +} + +// Verify re-hashes every retained record and checks the chain links. +func (s *Spool) Verify() VerifyReport { + committed, segs := s.snapshot() + s.mu.Lock() + prev := s.anchor + s.mu.Unlock() + + rep := VerifyReport{Segments: len(segs)} + expectSeq := uint64(0) + fail := func(msg string) { + if len(rep.Errors) < 20 { + rep.Errors = append(rep.Errors, msg) + } + } + for _, sg := range segs { + f, err := os.Open(filepath.Join(s.dir, sg.Name)) + if err != nil { + fail(err.Error()) + continue + } + r := bufio.NewReaderSize(f, 1<<20) + for { + line, err := r.ReadBytes('\n') + if err != nil { + break + } + rec, hash, perr := parseLine(line) + if perr != nil { + fail(fmt.Sprintf("%s: %v", sg.Name, perr)) + continue + } + if rec.Seq > committed { + break + } + if expectSeq != 0 && rec.Seq != expectSeq { + fail(fmt.Sprintf("seq gap: expected %d got %d", expectSeq, rec.Seq)) + } + // The first retained record links to the anchor left behind by + // retention; with no anchor (genesis) it must have no parent. + if rec.Prev != prev { + fail(fmt.Sprintf("chain break at seq %d", rec.Seq)) + } + if rep.Records == 0 { + rep.FirstSeq = rec.Seq + } + prev = hash + expectSeq = rec.Seq + 1 + rep.Records++ + rep.LastSeq = rec.Seq + } + f.Close() + } + rep.Head = prev + rep.OK = len(rep.Errors) == 0 + return rep +} + +// --- dead letters --- + +// DeadLetter is an event a destination permanently rejected. +type DeadLetter struct { + At time.Time `json:"at"` + Reason string `json:"reason"` + Event usage.Event `json:"event"` +} + +func (s *Spool) deadLetterPath(dest string) string { + return filepath.Join(s.dir, "deadletter-"+dest+".log") +} + +// AddDeadLetter parks a rejected event for operator review. +func (s *Spool) AddDeadLetter(dest string, ev usage.Event, reason string) error { + s.mu.Lock() + defer s.mu.Unlock() + b, err := json.Marshal(DeadLetter{At: s.opts.Now().UTC(), Reason: reason, Event: ev}) + if err != nil { + return err + } + f, err := os.OpenFile(s.deadLetterPath(dest), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(append(b, '\n')) + return err +} + +// DeadLetters lists parked events for a destination. +func (s *Spool) DeadLetters(dest string) ([]DeadLetter, error) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := os.ReadFile(s.deadLetterPath(dest)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []DeadLetter + for _, line := range bytes.Split(b, []byte("\n")) { + if len(line) == 0 { + continue + } + var dl DeadLetter + if err := json.Unmarshal(line, &dl); err == nil { + out = append(out, dl) + } + } + return out, nil +} + +// ClearDeadLetters removes all parked events for a destination. +func (s *Spool) ClearDeadLetters(dest string) error { + s.mu.Lock() + defer s.mu.Unlock() + err := os.Remove(s.deadLetterPath(dest)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +// --- stats & lifecycle --- + +// Stats is a point-in-time view of the spool. +type Stats struct { + Committed uint64 `json:"committed"` + FirstSeq uint64 `json:"first_seq"` + Segments int `json:"segments"` + Bytes int64 `json:"bytes"` + Watermarks map[string]time.Time `json:"watermarks"` + Cursors map[string]uint64 `json:"cursors"` + OldestAt time.Time `json:"oldest_at,omitempty"` +} + +func (s *Spool) Stats() Stats { + s.mu.Lock() + defer s.mu.Unlock() + st := Stats{Committed: s.committed, Segments: len(s.segs), Watermarks: map[string]time.Time{}, Cursors: map[string]uint64{}} + for k, v := range s.watermarks { + st.Watermarks[k] = v + } + for k, v := range s.cursors { + st.Cursors[k] = v + } + for _, sg := range s.segs { + st.Bytes += sg.Size + if sg.Last != 0 && st.FirstSeq == 0 { + st.FirstSeq = sg.First + st.OldestAt = sg.FirstAt + } + } + return st +} + +// Close flushes the manifest and closes the active segment. +func (s *Spool) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nil + } + s.closed = true + err := s.writeManifestLocked() + if cerr := s.active.Close(); err == nil { + err = cerr + } + return err +} + +func writeJSONAtomic(path string, v any, noSync bool) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return writeFileAtomic(path, b, noSync) +} + +func writeFileAtomic(path string, b []byte, noSync bool) error { + tmp := path + ".tmp" + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640) + if err != nil { + return err + } + if _, err := f.Write(b); err != nil { + f.Close() + return err + } + if !noSync { + if err := f.Sync(); err != nil { + f.Close() + return err + } + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/internal/spool/spool_test.go b/internal/spool/spool_test.go new file mode 100644 index 0000000..cd49a80 --- /dev/null +++ b/internal/spool/spool_test.go @@ -0,0 +1,409 @@ +package spool + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/vclusterlabs-experiments/vbilling/internal/usage" +) + +var t0 = time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) + +type clock struct { + mu sync.Mutex + t time.Time +} + +func (c *clock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t } +func (c *clock) Add(d time.Duration) { + c.mu.Lock() + c.t = c.t.Add(d) + c.mu.Unlock() +} + +func ev(tenant string, ws time.Time, q float64) usage.Event { + e := usage.Event{Tenant: tenant, Metric: usage.MetricGPUHours, Quantity: q, WindowStart: ws, WindowEnd: ws.Add(time.Minute)} + e.Finalize() + return e +} + +func open(t *testing.T, dir string, c *clock, opts Options) *Spool { + t.Helper() + opts.NoSync = true + opts.Now = c.Now + s, err := Open(dir, opts) + if err != nil { + t.Fatalf("open: %v", err) + } + return s +} + +func appendWindows(t *testing.T, s *Spool, from time.Time, n, perWindow int) { + t.Helper() + for w := 0; w < n; w++ { + ws := from.Add(time.Duration(w) * time.Minute) + var events []usage.Event + for i := 0; i < perWindow; i++ { + events = append(events, ev(fmt.Sprintf("tenant-%d", i), ws, 0.1)) + } + if err := s.AppendWindow(SourceCollector, ws.Add(time.Minute), events, ""); err != nil { + t.Fatalf("append window %d: %v", w, err) + } + } +} + +func TestAppendReadAckReopen(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{}) + if got := s.Cursor("stripe"); got != 0 { + t.Fatalf("new cursor on empty spool = %d", got) + } + appendWindows(t, s, t0, 3, 2) // 3 windows x (2 events + 1 commit) = 9 records + + recs, err := s.Read(0, 100) + if err != nil { + t.Fatal(err) + } + if len(recs) != 9 || recs[0].Seq != 1 || recs[8].Seq != 9 { + t.Fatalf("read %d records, first=%d", len(recs), recs[0].Seq) + } + if recs[2].Type != TypeCommit || recs[2].Commit.Events != 2 { + t.Fatalf("expected commit record, got %+v", recs[2]) + } + if err := s.Ack("stripe", 6); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + + s = open(t, dir, c, Options{}) + defer s.Close() + if s.Committed() != 9 { + t.Fatalf("committed after reopen = %d", s.Committed()) + } + if s.Cursor("stripe") != 6 { + t.Fatalf("cursor after reopen = %d", s.Cursor("stripe")) + } + wm, ok := s.Watermark(SourceCollector) + if !ok || !wm.Equal(t0.Add(3*time.Minute)) { + t.Fatalf("watermark after reopen = %s", wm) + } + recs, _ = s.Read(6, 100) + if len(recs) != 3 || recs[0].Seq != 7 { + t.Fatalf("read after cursor: %d records", len(recs)) + } + // A destination added later starts at the head: no surprise replay. + if s.Cursor("webhook") != 9 { + t.Fatalf("new destination cursor = %d, want head", s.Cursor("webhook")) + } + if rep := s.Verify(); !rep.OK || rep.Records != 9 { + t.Fatalf("verify: %+v", rep) + } +} + +func TestWindowCannotBeCommittedTwice(t *testing.T) { + c := &clock{t: t0} + s := open(t, t.TempDir(), c, Options{}) + defer s.Close() + appendWindows(t, s, t0, 1, 1) + err := s.AppendWindow(SourceCollector, t0.Add(time.Minute), []usage.Event{ev("a", t0, 1)}, "") + if err == nil || !strings.Contains(err.Error(), "already committed") { + t.Fatalf("expected ErrWindowCommitted, got %v", err) + } + // Other sources keep independent watermarks. + if err := s.AppendWindow("slurm", t0.Add(time.Minute), []usage.Event{ev("a", t0, 1)}, ""); err != nil { + t.Fatalf("independent source rejected: %v", err) + } +} + +func activeSegment(t *testing.T, dir string) string { + t.Helper() + matches, _ := filepath.Glob(filepath.Join(dir, "seg-*.log")) + if len(matches) == 0 { + t.Fatal("no segments") + } + return matches[len(matches)-1] +} + +func TestRecoveryTruncatesTornAndUncommittedTail(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{}) + appendWindows(t, s, t0, 2, 2) // 6 records, committed + s.Close() + + // Simulate a crash mid-batch: two complete event lines with valid hashes + // and no commit, followed by a torn line. + path := activeSegment(t, dir) + f, _ := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + prev := s.head + for seq := 7; seq <= 8; seq++ { + body := fmt.Sprintf(`{"seq":%d,"type":"event","at":"2026-10-05T10:03:00Z","prev":"%s","event":{"tenant":"x"}}`, seq, prev) + sum := sha256.Sum256([]byte(body)) + prev = hex.EncodeToString(sum[:]) + fmt.Fprintf(f, "%s\t%s\n", prev, body) + } + f.WriteString(`deadbeef {"seq":9,"ty`) + f.Close() + + s = open(t, dir, c, Options{}) + defer s.Close() + if s.Committed() != 6 { + t.Fatalf("committed after recovery = %d, want 6", s.Committed()) + } + recs, _ := s.Read(0, 100) + if len(recs) != 6 { + t.Fatalf("recovered %d records, want 6", len(recs)) + } + // The truncated window can be collected again and the chain stays intact. + appendWindows(t, s, t0.Add(2*time.Minute), 1, 1) + if rep := s.Verify(); !rep.OK || rep.Records != 8 { + t.Fatalf("verify after recovery: %+v", rep) + } +} + +func TestCorruptCommittedLineRefusesToOpen(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{}) + appendWindows(t, s, t0, 1, 2) + s.Close() + + path := activeSegment(t, dir) + b, _ := os.ReadFile(path) + tampered := strings.Replace(string(b), `"quantity":0.1`, `"quantity":0.2`, 1) + os.WriteFile(path, []byte(tampered), 0o640) + + if _, err := Open(dir, Options{NoSync: true, Now: c.Now}); err == nil || !strings.Contains(err.Error(), "corrupt") { + t.Fatalf("expected corruption error, got %v", err) + } +} + +func TestVerifyDetectsRehashedTampering(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{}) + appendWindows(t, s, t0, 2, 2) + s.Close() + + // Rewrite one record and recompute its own hash: the line now looks + // valid on its own, but the next record no longer links to it. + path := activeSegment(t, dir) + b, _ := os.ReadFile(path) + lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") + _, body, _ := strings.Cut(lines[1], "\t") + body = strings.Replace(body, `"quantity":0.1`, `"quantity":9.9`, 1) + sum := sha256.Sum256([]byte(body)) + lines[1] = hex.EncodeToString(sum[:]) + "\t" + body + os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o640) + + s = open(t, dir, c, Options{}) + defer s.Close() + rep := s.Verify() + if rep.OK || len(rep.Errors) == 0 || !strings.Contains(rep.Errors[0], "chain break at seq 3") { + t.Fatalf("tampering not detected: %+v", rep) + } +} + +func TestRotationReadAcrossSegmentsAndScan(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{SegmentMaxRecords: 7}) + for w := 0; w < 10; w++ { // 30 records; rotation only at batch boundaries + c.Add(time.Minute) // append each window right after it closes + appendWindows(t, s, t0.Add(time.Duration(w)*time.Minute), 1, 2) + } + segs, _ := filepath.Glob(filepath.Join(dir, "seg-*.log")) + if len(segs) < 3 { + t.Fatalf("expected rotation, got %d segments", len(segs)) + } + + var seqs []uint64 + after := uint64(0) + for { + recs, err := s.Read(after, 4) + if err != nil { + t.Fatal(err) + } + if len(recs) == 0 { + break + } + for _, r := range recs { + seqs = append(seqs, r.Seq) + } + after = recs[len(recs)-1].Seq + } + if len(seqs) != 30 { + t.Fatalf("paged read returned %d records", len(seqs)) + } + for i, q := range seqs { + if q != uint64(i+1) { + t.Fatalf("sequence gap at %d: %d", i, q) + } + } + + var n int + err := s.Scan(t0.Add(3*time.Minute), t0.Add(5*time.Minute), func(e *usage.Event) error { n++; return nil }) + if err != nil || n != 4 { + t.Fatalf("scan windows 3-4: n=%d err=%v", n, err) + } + s.Close() + + // Closed segments come back from the manifest; reads still work. + s = open(t, dir, c, Options{SegmentMaxRecords: 7}) + defer s.Close() + recs, _ := s.Read(10, 5) + if len(recs) != 5 || recs[0].Seq != 11 { + t.Fatalf("read after reopen: %d records starting %d", len(recs), recs[0].Seq) + } + if rep := s.Verify(); !rep.OK || rep.Records != 30 { + t.Fatalf("verify: %+v", rep) + } +} + +func TestCompactHonorsRetentionAndCursors(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{SegmentMaxRecords: 3, Retention: time.Hour}) + defer s.Close() + s.Cursor("metronome") + appendWindows(t, s, t0, 6, 2) // each window is its own segment + + c.Add(2 * time.Hour) + if n, _ := s.Compact([]string{"metronome"}); n != 0 { + t.Fatalf("compacted %d segments that metronome has not consumed", n) + } + s.Ack("metronome", 9) // consumed the first three windows + n, err := s.Compact([]string{"metronome"}) + if err != nil || n != 3 { + t.Fatalf("compact removed %d (err=%v), want 3", n, err) + } + if s.FirstSeq() != 10 { + t.Fatalf("first retained seq = %d", s.FirstSeq()) + } + if rep := s.Verify(); !rep.OK || rep.FirstSeq != 10 { + t.Fatalf("verify after compaction: %+v", rep) + } + // Reads below the retained range simply start at what is left. + recs, _ := s.Read(0, 1) + if len(recs) != 1 || recs[0].Seq != 10 { + t.Fatalf("read after compaction: %+v", recs) + } +} + +func TestIngestDedupeSurvivesRestart(t *testing.T) { + dir := t.TempDir() + c := &clock{t: t0} + s := open(t, dir, c, Options{DedupeWindow: time.Hour}) + a, b := ev("t1", t0, 1), ev("t2", t0, 1) + acc, dup, err := s.AppendIngest("ingest:gateway", []usage.Event{a, b, a}) + if err != nil || len(acc) != 2 || len(dup) != 1 { + t.Fatalf("first ingest acc=%v dup=%v err=%v", acc, dup, err) + } + s.Close() + + s = open(t, dir, c, Options{DedupeWindow: time.Hour}) + acc, dup, _ = s.AppendIngest("ingest:gateway", []usage.Event{a}) + if len(acc) != 0 || len(dup) != 1 { + t.Fatalf("replayed ID accepted after restart: acc=%v", acc) + } + s.Close() + + c.Add(2 * time.Hour) // past the dedupe window + s = open(t, dir, c, Options{DedupeWindow: time.Hour}) + defer s.Close() + acc, _, _ = s.AppendIngest("ingest:gateway", []usage.Event{a}) + if len(acc) != 1 { + t.Fatal("ID outside dedupe window should be accepted") + } +} + +func TestConcurrentAppendAndRead(t *testing.T) { + c := &clock{t: t0} + s := open(t, t.TempDir(), c, Options{SegmentMaxRecords: 50}) + defer s.Close() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + appendWindows(t, s, t0, 100, 3) + }() + go func() { + defer wg.Done() + after := uint64(0) + for after < 400 { + recs, err := s.Read(after, 17) + if err != nil { + t.Error(err) + return + } + for _, r := range recs { + if r.Seq != after+1 { + t.Errorf("out of order: %d after %d", r.Seq, after) + return + } + after = r.Seq + } + if len(recs) == 0 { + <-s.Changed() + } + } + }() + wg.Wait() +} + +func TestDeadLetters(t *testing.T) { + c := &clock{t: t0} + s := open(t, t.TempDir(), c, Options{}) + defer s.Close() + if err := s.AddDeadLetter("stripe", ev("t", t0, 1), "customer not found"); err != nil { + t.Fatal(err) + } + dls, _ := s.DeadLetters("stripe") + if len(dls) != 1 || dls[0].Reason != "customer not found" { + t.Fatalf("dead letters: %+v", dls) + } + s.ClearDeadLetters("stripe") + if dls, _ := s.DeadLetters("stripe"); len(dls) != 0 { + t.Fatal("dead letters not cleared") + } +} + +func TestScanPrefilterMatchesFullDecode(t *testing.T) { + c := &clock{t: t0} + s := open(t, t.TempDir(), c, Options{SegmentMaxRecords: 10}) + defer s.Close() + for w := 0; w < 30; w++ { + c.Add(time.Minute) + appendWindows(t, s, t0.Add(time.Duration(w)*time.Minute), 1, 3) + } + count := func(from, to time.Time) int { + n := 0 + if err := s.Scan(from, to, func(e *usage.Event) error { n++; return nil }); err != nil { + t.Fatal(err) + } + return n + } + if got := count(t0, t0.Add(30*time.Minute)); got != 90 { + t.Fatalf("full range: %d", got) + } + if got := count(t0.Add(10*time.Minute), t0.Add(12*time.Minute)); got != 6 { + t.Fatalf("two windows: %d", got) + } + if got := count(t0.Add(time.Hour), t0.Add(2*time.Hour)); got != 0 { + t.Fatalf("empty range: %d", got) + } + ws, we, ok := peekWindow([]byte(`aaaa\t{"seq":1,"event":{"window_start":"2026-10-05T10:00:00Z","window_end":"2026-10-05T10:01:00Z"}}`)) + if !ok || !ws.Equal(t0) || !we.Equal(t0.Add(time.Minute)) { + t.Fatalf("peekWindow: %v %v %v", ws, we, ok) + } +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 0000000..3468a8d --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,131 @@ +// Package telemetry is a dependency-free Prometheus text exposition registry +// for vBilling's own health: delivery lag, failures, dead letters, windows. +package telemetry + +import ( + "fmt" + "io" + "math" + "sort" + "strings" + "sync" +) + +type kind int + +const ( + counter kind = iota + gauge +) + +type family struct { + help string + kind kind + series map[string]float64 // rendered label set -> value +} + +// Registry holds metric families. +type Registry struct { + mu sync.Mutex + families map[string]*family +} + +// Default is the process-wide registry. +var Default = New() + +// New returns an empty registry. +func New() *Registry { return &Registry{families: map[string]*family{}} } + +func (r *Registry) fam(name, help string, k kind) *family { + f, ok := r.families[name] + if !ok { + f = &family{help: help, kind: k, series: map[string]float64{}} + r.families[name] = f + } + return f +} + +// Add increments a counter. +func (r *Registry) Add(name, help string, labels map[string]string, v float64) { + r.mu.Lock() + defer r.mu.Unlock() + r.fam(name, help, counter).series[renderLabels(labels)] += v +} + +// Set sets a gauge. +func (r *Registry) Set(name, help string, labels map[string]string, v float64) { + r.mu.Lock() + defer r.mu.Unlock() + r.fam(name, help, gauge).series[renderLabels(labels)] = v +} + +// Get returns a series value (tests and status endpoints). +func (r *Registry) Get(name string, labels map[string]string) float64 { + r.mu.Lock() + defer r.mu.Unlock() + if f, ok := r.families[name]; ok { + return f.series[renderLabels(labels)] + } + return 0 +} + +// WriteText writes the Prometheus text exposition format. +func (r *Registry) WriteText(w io.Writer) error { + r.mu.Lock() + defer r.mu.Unlock() + names := make([]string, 0, len(r.families)) + for n := range r.families { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + f := r.families[n] + typ := "counter" + if f.kind == gauge { + typ = "gauge" + } + if _, err := fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s %s\n", n, f.help, n, typ); err != nil { + return err + } + keys := make([]string, 0, len(f.series)) + for k := range f.series { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if _, err := fmt.Fprintf(w, "%s%s %s\n", n, k, formatValue(f.series[k])); err != nil { + return err + } + } + } + return nil +} + +func formatValue(v float64) string { + if math.IsInf(v, 1) { + return "+Inf" + } + return fmt.Sprintf("%g", v) +} + +func renderLabels(labels map[string]string) string { + if len(labels) == 0 { + return "" + } + keys := make([]string, 0, len(labels)) + for k := range labels { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + b.WriteByte('{') + for i, k := range keys { + if i > 0 { + b.WriteByte(',') + } + v := strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`).Replace(labels[k]) + fmt.Fprintf(&b, `%s="%s"`, k, v) + } + b.WriteByte('}') + return b.String() +} diff --git a/internal/tenantapi/tenantapi.go b/internal/tenantapi/tenantapi.go new file mode 100644 index 0000000..40b3128 --- /dev/null +++ b/internal/tenantapi/tenantapi.go @@ -0,0 +1,274 @@ +// Package tenantapi connects vBilling to tenant clusters' own API servers, +// for tenant clusters whose workloads never reach the control plane cluster +// (vCluster Private Nodes, Auto Nodes, Standalone). +// +// Credentials, in order: +// - external tenant clusters (Standalone, or any cluster elsewhere) use a +// kubeconfig file listed in TENANT_CLUSTERS_FILE; +// - tenant clusters on this control plane cluster use the Secret named +// TENANT_KUBECONFIG_SECRET (default vbilling-kubeconfig) in their +// namespace. vCluster writes it through exportKubeConfig.additionalSecrets +// with a read-only service account, so vBilling never holds admin access; +// - with TENANT_ADMIN_FALLBACK=true, the tenant cluster's admin kubeconfig +// (Secret vc-) is used when no dedicated Secret exists. +// +// Kubeconfigs that point at localhost (the vCluster default) are rewritten to +// the tenant cluster's Service, https://..svc:443, verified +// as . (a name vCluster's serving certificate includes). +package tenantapi + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "net" + "net/url" + "os" + "sync" + "time" + + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + metricsclient "k8s.io/metrics/pkg/client/clientset/versioned" + "sigs.k8s.io/yaml" + + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/metrics" +) + +// DefaultSecretName is the Secret vBilling reads in a tenant cluster's namespace. +const DefaultSecretName = "vbilling-kubeconfig" + +// External is a tenant cluster reached through a kubeconfig file. +type External struct { + Name string `json:"name"` + Kubeconfig string `json:"kubeconfig"` + Tenant string `json:"tenant,omitempty"` + DisplayName string `json:"displayName,omitempty"` + Project string `json:"project,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` // other vbilling.vcluster.com/ settings: email, currency, plan, tenant-class +} + +// LoadExternal reads TENANT_CLUSTERS_FILE (a YAML list of External). +func LoadExternal(path string) ([]External, error) { + if path == "" { + return nil, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var out []External + if err := yaml.UnmarshalStrict(data, &out); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + seen := map[string]bool{} + for _, e := range out { + if e.Name == "" || e.Kubeconfig == "" { + return nil, fmt.Errorf("%s: every tenant cluster needs name and kubeconfig", path) + } + if seen[e.Name] { + return nil, fmt.Errorf("%s: duplicate tenant cluster %q", path, e.Name) + } + seen[e.Name] = true + } + return out, nil +} + +// Cluster is the discovered tenant cluster for an external entry. +func (e External) Cluster() discovery.TenantCluster { + ann := map[string]string{} + for k, v := range e.Metadata { + ann[discovery.AnnotationPrefix+k] = v + } + if e.Tenant != "" { + ann[discovery.AnnotationPrefix+"tenant"] = e.Tenant + } + return discovery.TenantCluster{Name: e.Name, Instance: e.Name, External: true, Ready: true, Source: "external", + Project: e.Project, DisplayName: e.DisplayName, Annotations: ann} +} + +// Options configure the Manager. +type Options struct { + SecretName string + AdminFallback bool + External []External + WatchPods bool // watch pod deletions too (usage-mode billing) + Retention time.Duration // how long deleted nodes and pods are kept + Timeout time.Duration + Now func() time.Time +} + +// Manager implements metrics.TenantAPI with cached clients per tenant cluster. +type Manager struct { + host kubernetes.Interface + ctx context.Context + opts Options + ext map[string]External + mu sync.Mutex + entries map[string]*entry +} + +type entry struct { + conn *metrics.TenantConn + version string + cancel context.CancelFunc +} + +// New returns a Manager. ctx bounds the deletion watches it starts. +func New(ctx context.Context, host kubernetes.Interface, opts Options) *Manager { + if opts.SecretName == "" { + opts.SecretName = DefaultSecretName + } + if opts.Timeout == 0 { + opts.Timeout = 15 * time.Second + } + if opts.Now == nil { + opts.Now = time.Now + } + m := &Manager{host: host, ctx: ctx, opts: opts, ext: map[string]External{}, entries: map[string]*entry{}} + for _, e := range opts.External { + m.ext[e.Name] = e + } + return m +} + +// Clusters returns the external tenant clusters, for discovery. +func (m *Manager) Clusters() []discovery.TenantCluster { + out := make([]discovery.TenantCluster, 0, len(m.opts.External)) + for _, e := range m.opts.External { + out = append(out, e.Cluster()) + } + return out +} + +// Conn returns a connection to the tenant cluster's API, or +// metrics.ErrNoTenantAPI when none is configured for it. +func (m *Manager) Conn(ctx context.Context, cl discovery.TenantCluster) (*metrics.TenantConn, error) { + key := cl.ExternalID() + cfg, version, err := m.restConfig(ctx, cl) + if err != nil { + return nil, err + } + m.mu.Lock() + defer m.mu.Unlock() + if e, ok := m.entries[key]; ok { + if e.version == version { + return e.conn, nil + } + e.cancel() // credentials rotated + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + mc, err := metricsclient.NewForConfig(cfg) + if err != nil { + return nil, err + } + g := metrics.NewGraveyard(m.opts.Retention, m.opts.Now) + wctx, cancel := context.WithCancel(m.ctx) + go g.Watch(wctx, kube, "", m.opts.WatchPods, true) + conn := &metrics.TenantConn{Kube: kube, Metrics: mc, Dynamic: dyn, Graveyard: g, External: cl.External} + m.entries[key] = &entry{conn: conn, version: version, cancel: cancel} + log.Printf("[tenantapi] %s: metering through the tenant cluster's API at %s", key, cfg.Host) + return conn, nil +} + +// Retain stops watches of tenant clusters that are gone. +func (m *Manager) Retain(clusters []discovery.TenantCluster) { + keep := map[string]bool{} + for i := range clusters { + keep[clusters[i].ExternalID()] = true + } + m.mu.Lock() + defer m.mu.Unlock() + for k, e := range m.entries { + if !keep[k] { + e.cancel() + delete(m.entries, k) + } + } +} + +func (m *Manager) restConfig(ctx context.Context, cl discovery.TenantCluster) (*rest.Config, string, error) { + if cl.External { + e, ok := m.ext[cl.Name] + if !ok { + return nil, "", metrics.ErrNoTenantAPI + } + data, err := os.ReadFile(e.Kubeconfig) + if err != nil { + return nil, "", fmt.Errorf("read kubeconfig: %w", err) + } + cfg, err := clientcmd.RESTConfigFromKubeConfig(data) + if err != nil { + return nil, "", fmt.Errorf("parse kubeconfig %s: %w", e.Kubeconfig, err) + } + cfg.Timeout = m.opts.Timeout + sum := sha256.Sum256(data) + return cfg, hex.EncodeToString(sum[:8]), nil + } + names := []string{m.opts.SecretName} + if m.opts.AdminFallback { + names = append(names, "vc-"+cl.Name) + } + for _, name := range names { + sec, err := m.host.CoreV1().Secrets(cl.Namespace).Get(ctx, name, metav1.GetOptions{}) + if kerrors.IsNotFound(err) || kerrors.IsForbidden(err) { + continue + } + if err != nil { + return nil, "", fmt.Errorf("read secret %s/%s: %w", cl.Namespace, name, err) + } + data := firstKey(sec.Data, "config", "kubeconfig", "value") + if data == nil { + return nil, "", fmt.Errorf("secret %s/%s has no kubeconfig (keys config, kubeconfig or value)", cl.Namespace, name) + } + cfg, err := clientcmd.RESTConfigFromKubeConfig(data) + if err != nil { + return nil, "", fmt.Errorf("parse kubeconfig in secret %s/%s: %w", cl.Namespace, name, err) + } + if host, serverName, ok := inCluster(cfg.Host, cl.Name, cl.Namespace); ok { + cfg.Host, cfg.TLSClientConfig.ServerName = host, serverName + } + cfg.Timeout = m.opts.Timeout + return cfg, name + "@" + sec.ResourceVersion, nil + } + return nil, "", metrics.ErrNoTenantAPI +} + +// inCluster points kubeconfigs written for local port-forwarding (the +// vCluster default, https://localhost:8443) at the tenant cluster's Service. +// The serving certificate covers . but not the .svc form, +// so that is the name TLS verifies. +func inCluster(host, name, namespace string) (newHost, serverName string, ok bool) { + u, err := url.Parse(host) + if err != nil { + return "", "", false + } + switch h := u.Hostname(); { + case h == "localhost", net.ParseIP(h) != nil && net.ParseIP(h).IsLoopback(): + return fmt.Sprintf("https://%s.%s.svc:443", name, namespace), name + "." + namespace, true + } + return "", "", false +} + +func firstKey(data map[string][]byte, keys ...string) []byte { + for _, k := range keys { + if v, ok := data[k]; ok && len(v) > 0 { + return v + } + } + return nil +} diff --git a/internal/tenantapi/tenantapi_test.go b/internal/tenantapi/tenantapi_test.go new file mode 100644 index 0000000..f71dc66 --- /dev/null +++ b/internal/tenantapi/tenantapi_test.go @@ -0,0 +1,139 @@ +package tenantapi + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/vclusterlabs-experiments/vbilling/internal/discovery" + "github.com/vclusterlabs-experiments/vbilling/internal/metrics" +) + +func kubeconfig(server string) []byte { + return []byte(`apiVersion: v1 +kind: Config +clusters: +- name: t + cluster: {server: ` + server + `, insecure-skip-tls-verify: true} +users: +- name: reader + user: {token: read-only-token} +contexts: +- name: t + context: {cluster: t, user: reader} +current-context: t +`) +} + +func secret(ns, name, server string) *corev1.Secret { + return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, ResourceVersion: "1"}, + Data: map[string][]byte{"config": kubeconfig(server)}} +} + +var team = discovery.TenantCluster{Name: "team-p", Namespace: "team-p"} + +func TestLeastPrivilegeSecretAndInClusterAddress(t *testing.T) { + host := fake.NewSimpleClientset(secret("team-p", DefaultSecretName, "https://localhost:8443")) + m := New(context.Background(), host, Options{}) + cfg, version, err := m.restConfig(context.Background(), team) + if err != nil { + t.Fatal(err) + } + if cfg.Host != "https://team-p.team-p.svc:443" || cfg.TLSClientConfig.ServerName != "team-p.team-p" { + t.Errorf("host = %s (TLS name %q), want the tenant cluster's Service verified as team-p.team-p", cfg.Host, cfg.TLSClientConfig.ServerName) + } + if cfg.BearerToken != "read-only-token" || !strings.HasPrefix(version, DefaultSecretName+"@") { + t.Errorf("credentials from %s, token %q", version, cfg.BearerToken) + } + for _, in := range []string{"https://127.0.0.1:8443", "https://[::1]:8443"} { + if got, name, ok := inCluster(in, "a", "b"); !ok || got != "https://a.b.svc:443" || name != "a.b" { + t.Errorf("inCluster(%s) = %s, %s, %v", in, got, name, ok) + } + } + if _, _, ok := inCluster("https://team-p.example.com:6443", "a", "b"); ok { + t.Error("a reachable server must be kept") + } +} + +func TestAdminKubeconfigOnlyWhenAllowed(t *testing.T) { + host := fake.NewSimpleClientset(secret("team-p", "vc-team-p", "https://localhost:8443")) + if _, err := New(context.Background(), host, Options{}).Conn(context.Background(), team); !errors.Is(err, metrics.ErrNoTenantAPI) { + t.Fatalf("without TENANT_ADMIN_FALLBACK the admin kubeconfig must not be used, err = %v", err) + } + m := New(context.Background(), host, Options{AdminFallback: true}) + if _, version, err := m.restConfig(context.Background(), team); err != nil || !strings.HasPrefix(version, "vc-team-p@") { + t.Fatalf("admin fallback: %s, %v", version, err) + } +} + +func TestConnCachedUntilCredentialsRotate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + host := fake.NewSimpleClientset(secret("team-p", DefaultSecretName, "https://localhost:8443")) + m := New(ctx, host, Options{Retention: time.Hour}) + a, err := m.Conn(ctx, team) + if err != nil { + t.Fatal(err) + } + if b, _ := m.Conn(ctx, team); b != a { + t.Error("connection not reused") + } + rotated := secret("team-p", DefaultSecretName, "https://localhost:8443") + rotated.ResourceVersion = "2" + if _, err := host.CoreV1().Secrets("team-p").Update(ctx, rotated, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if c, _ := m.Conn(ctx, team); c == a { + t.Error("rotated credentials must build a new connection") + } + m.Retain(nil) + if len(m.entries) != 0 { + t.Error("connections of removed tenant clusters must be dropped") + } +} + +func TestExternalTenantClusters(t *testing.T) { + dir := t.TempDir() + kc := filepath.Join(dir, "gpu-standalone.kubeconfig") + if err := os.WriteFile(kc, kubeconfig("https://10.0.0.5:6443"), 0o600); err != nil { + t.Fatal(err) + } + list := filepath.Join(dir, "clusters.yaml") + body := "- name: gpu-standalone\n kubeconfig: " + kc + "\n tenant: acme\n displayName: Acme AI\n metadata: {currency: aud}\n" + if err := os.WriteFile(list, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + ext, err := LoadExternal(list) + if err != nil { + t.Fatal(err) + } + cl := ext[0].Cluster() + if cl.ExternalID() != "external-gpu-standalone" || cl.TenantID(discovery.TenantPerCluster) != "acme" || cl.Meta("currency") != "aud" || !cl.External { + t.Fatalf("external cluster = %+v", cl) + } + m := New(context.Background(), fake.NewSimpleClientset(), Options{External: ext}) + cfg, _, err := m.restConfig(context.Background(), cl) + if err != nil || cfg.Host != "https://10.0.0.5:6443" { + t.Fatalf("external kubeconfig: %v, %v", cfg, err) + } + if _, err := LoadExternal(writeFile(t, dir, "bad.yaml", "- name: x\n")); err == nil { + t.Error("an entry without kubeconfig must be rejected") + } +} + +func writeFile(t *testing.T, dir, name, body string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return p +} diff --git a/internal/usage/catalog.go b/internal/usage/catalog.go new file mode 100644 index 0000000..cbc60ea --- /dev/null +++ b/internal/usage/catalog.go @@ -0,0 +1,178 @@ +package usage + +import ( + "fmt" + "sort" + "strings" + "sync" +) + +// Canonical metric codes emitted by the collector. +const ( + MetricCPUCoreHours = "vcluster_cpu_core_hours" + MetricMemoryGBHours = "vcluster_memory_gb_hours" + MetricStorageGBHours = "vcluster_storage_gb_hours" + MetricInstanceHours = "vcluster_instance_hours" + MetricGPUHours = "vcluster_gpu_hours" + MetricGPUUtilization = "vcluster_gpu_utilization" + MetricNetworkEgressGB = "vcluster_network_egress_gb" + MetricLBHours = "vcluster_lb_hours" + MetricPrivateNodeHours = "vcluster_private_node_hours" + // MetricGPUDowntimeHours records GPU time that was allocated but not + // billed because the node was unhealthy. It is informational: it makes + // downtime credits auditable without ever reaching an invoice. + MetricGPUDowntimeHours = "vcluster_gpu_downtime_hours" +) + +// MetricDef describes a metric so adapters can create the matching meter, +// billable metric or plan charge. +type MetricDef struct { + Code string `json:"code"` + Name string `json:"name"` + Unit string `json:"unit"` + Description string `json:"description"` + // Billable metrics are created in billing backends. Informational ones + // only reach the ledger, webhooks and exports. + Billable bool `json:"billable"` + // GroupKeys are the dimensions a rating engine should be able to price + // on. Every event of the metric carries all of them (tenant_cluster is + // added for invoice presentation), so backends can require them. + GroupKeys []string `json:"group_keys,omitempty"` + // Custom marks operator-declared metrics (ingest API). + Custom bool `json:"custom,omitempty"` +} + +var builtin = []MetricDef{ + {Code: MetricGPUHours, Name: "GPU Hours", Unit: "gpu-hours", Billable: true, + Description: "GPU-equivalent hours allocated to running workloads. MIG and time-sliced GPUs count at their fraction.", + GroupKeys: []string{"region", "sku", DimCapacityType, DimBillingMode}}, + {Code: MetricCPUCoreHours, Name: "CPU Core Hours", Unit: "core-hours", Billable: true, + Description: "vCPU core-hours (usage, requests or the larger of both, per configuration).", + GroupKeys: []string{"region", DimCapacityType, DimBillingMode}}, + {Code: MetricMemoryGBHours, Name: "Memory GiB Hours", Unit: "gib-hours", Billable: true, + Description: "Memory GiB-hours (usage, requests or the larger of both, per configuration).", + GroupKeys: []string{"region", DimCapacityType, DimBillingMode}}, + {Code: MetricStorageGBHours, Name: "Storage GiB Hours", Unit: "gib-hours", Billable: true, + Description: "Requested capacity of bound persistent volume claims, per storage class.", + GroupKeys: []string{"region", "sku"}}, + {Code: MetricInstanceHours, Name: "Tenant Cluster Hours", Unit: "hours", Billable: true, + Description: "Hours a tenant cluster control plane was running and ready.", + GroupKeys: []string{"region", DimTenantClass}}, + {Code: MetricPrivateNodeHours, Name: "Dedicated Node Hours", Unit: "node-hours", Billable: true, + Description: "Whole nodes allocated to one tenant, billed for full capacity.", + GroupKeys: []string{"region", "sku", DimCapacityType}}, + {Code: MetricNetworkEgressGB, Name: "Network Egress GiB", Unit: "gib", Billable: true, + Description: "Transmitted bytes from tenant workloads (requires Prometheus).", + GroupKeys: []string{"region"}}, + {Code: MetricLBHours, Name: "Load Balancer Hours", Unit: "hours", Billable: true, + Description: "Hours of Service type=LoadBalancer.", + GroupKeys: []string{"region"}}, + {Code: MetricGPUUtilization, Name: "GPU Utilization Hours", Unit: "util-hours", Billable: false, + Description: "Average DCGM GPU utilization (0-100) multiplied by hours. Informational (requires Prometheus + DCGM).", + GroupKeys: []string{"region", DimGPUType}}, + {Code: MetricGPUDowntimeHours, Name: "GPU Downtime Hours", Unit: "gpu-hours", Billable: false, + Description: "Allocated GPU time excluded from billing because the node was not ready or unhealthy.", + GroupKeys: []string{"region", DimGPUType}}, +} + +var ( + catMu sync.RWMutex + custom = map[string]MetricDef{} +) + +// Catalog returns builtin metrics followed by operator-declared custom ones. +func Catalog() []MetricDef { + catMu.RLock() + defer catMu.RUnlock() + out := make([]MetricDef, 0, len(builtin)+len(custom)) + out = append(out, builtin...) + codes := make([]string, 0, len(custom)) + for c := range custom { + codes = append(codes, c) + } + sort.Strings(codes) + for _, c := range codes { + out = append(out, custom[c]) + } + return out +} + +// Billable returns the subset of the catalog that billing backends create. +func Billable() []MetricDef { + var out []MetricDef + for _, d := range Catalog() { + if d.Billable { + out = append(out, d) + } + } + return out +} + +// Lookup finds a metric definition by code. +func Lookup(code string) (MetricDef, bool) { + for _, d := range builtin { + if d.Code == code { + return d, true + } + } + catMu.RLock() + defer catMu.RUnlock() + d, ok := custom[code] + return d, ok +} + +// RegisterCustom declares an operator metric, e.g. inference tokens pushed +// through the ingest API by a model gateway. +func RegisterCustom(d MetricDef) error { + if !metricRe.MatchString(d.Code) { + return fmt.Errorf("custom metric code %q must match %s", d.Code, metricRe) + } + if _, ok := Lookup(d.Code); ok { + return fmt.Errorf("metric %q already exists", d.Code) + } + if d.Name == "" { + d.Name = d.Code + } + if d.Unit == "" { + d.Unit = "units" + } + d.Custom = true + d.Billable = true + catMu.Lock() + custom[d.Code] = d + catMu.Unlock() + return nil +} + +// ParseCustomMetrics parses "code:unit[:group_key|group_key],..." as used by +// the CUSTOM_METRICS env var, e.g. +// "inference_input_tokens:tokens:model|region,slurm_gpu_hours:gpu-hours". +func ParseCustomMetrics(spec string) ([]MetricDef, error) { + var out []MetricDef + for _, item := range strings.Split(spec, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + parts := strings.Split(item, ":") + d := MetricDef{Code: parts[0]} + if len(parts) > 1 { + d.Unit = parts[1] + } + if len(parts) > 2 && parts[2] != "" { + d.GroupKeys = strings.Split(parts[2], "|") + } + if len(parts) > 3 { + return nil, fmt.Errorf("custom metric %q: expected code:unit[:key|key]", item) + } + out = append(out, d) + } + return out, nil +} + +// resetCustomForTest clears custom metrics (tests only). +func resetCustomForTest() { + catMu.Lock() + custom = map[string]MetricDef{} + catMu.Unlock() +} diff --git a/internal/usage/schema_test.go b/internal/usage/schema_test.go new file mode 100644 index 0000000..2b6e06d --- /dev/null +++ b/internal/usage/schema_test.go @@ -0,0 +1,66 @@ +package usage + +import ( + "encoding/json" + "os" + "sort" + "testing" + "time" +) + +// TestPublishedSchemaMatchesEvent keeps docs/schema/usage-event.v1.json and +// the Event struct in lockstep: same field names, same required set, same +// schema version, and a real event satisfies the documented constraints. +func TestPublishedSchemaMatchesEvent(t *testing.T) { + b, err := os.ReadFile("../../docs/schema/usage-event.v1.json") + if err != nil { + t.Fatal(err) + } + var schema struct { + Required []string `json:"required"` + Properties map[string]json.RawMessage `json:"properties"` + } + if err := json.Unmarshal(b, &schema); err != nil { + t.Fatal(err) + } + var version struct { + Const string `json:"const"` + } + json.Unmarshal(schema.Properties["schema_version"], &version) + if version.Const != SchemaVersion { + t.Fatalf("schema const %q != SchemaVersion %q", version.Const, SchemaVersion) + } + + ws := time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) + e := Event{Tenant: "acme", Metric: MetricGPUHours, Quantity: 1, WindowStart: ws, WindowEnd: ws.Add(time.Minute), + Region: "r", Project: "p", SKU: "s", ResourceID: "x", Source: "collector", RecordedAt: ws, + Dimensions: map[string]string{DimCapacityType: CapacitySpot}, Properties: map[string]any{"k": 1}} + e.Finalize() + raw, _ := json.Marshal(e) + var fields map[string]any + json.Unmarshal(raw, &fields) + + var structKeys, schemaKeys []string + for k := range fields { + structKeys = append(structKeys, k) + if _, ok := schema.Properties[k]; !ok { + t.Errorf("Event field %q is not documented in the schema", k) + } + } + for k := range schema.Properties { + schemaKeys = append(schemaKeys, k) + if _, ok := fields[k]; !ok { + t.Errorf("schema property %q does not exist on Event", k) + } + } + for _, r := range schema.Required { + if _, ok := fields[r]; !ok { + t.Errorf("required schema field %q missing from a finalized event", r) + } + } + sort.Strings(structKeys) + sort.Strings(schemaKeys) + if t.Failed() { + t.Logf("struct: %v\nschema: %v", structKeys, schemaKeys) + } +} diff --git a/internal/usage/usage.go b/internal/usage/usage.go new file mode 100644 index 0000000..0555fba --- /dev/null +++ b/internal/usage/usage.go @@ -0,0 +1,212 @@ +// Package usage defines vBilling's canonical usage schema. +// +// Every billable observation becomes an Event: a quantity of one metric, +// attributed to one tenant, over one closed time window, stamped with the +// dimensions a rating engine needs (region, project, SKU, capacity type, ...). +// Events carry a deterministic ID derived from their identity, so collecting +// or delivering the same window twice produces the same ID and every +// downstream system can deduplicate on it. +package usage + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math" + "regexp" + "sort" + "strings" + "time" +) + +// SchemaVersion identifies the wire format of Event. It changes only on +// breaking changes; new optional fields do not bump it. +const SchemaVersion = "vbilling.usage/v1" + +// Well-known dimension keys. Dimensions are part of an event's identity and +// are what rating engines group and price on. +const ( + DimTenantCluster = "tenant_cluster" // tenant cluster external ID + DimGPUType = "gpu_type" // normalized GPU model, e.g. NVIDIA-H100-80GB-HBM3 + DimGPUProfile = "gpu_profile" // full, mig-1g.10gb, shared-4, ... + DimCapacityType = "capacity_type" // on-demand, spot, preemptible, reserved + DimBillingMode = "billing_mode" // shared, dedicated_node, private_node + DimZone = "zone" + DimTenantClass = "tenant_class" // e.g. public, enterprise, government, dev + DimNamespace = "namespace" // namespace inside the tenant cluster (optional) + DimNode = "node" + DimInstanceType = "instance_type" +) + +// Capacity types. +const ( + CapacityOnDemand = "on-demand" + CapacitySpot = "spot" + CapacityPreemptible = "preemptible" + CapacityReserved = "reserved" +) + +// Event is one metered quantity for one tenant over one window. +type Event struct { + SchemaVersion string `json:"schema_version"` + // ID is deterministic for collector events (see DeriveID). Ingested + // events must supply their own stable ID. + ID string `json:"id"` + Tenant string `json:"tenant"` + Metric string `json:"metric"` + // Quantity is in Unit, accumulated over [WindowStart, WindowEnd). + Quantity float64 `json:"quantity"` + Unit string `json:"unit"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + + Region string `json:"region,omitempty"` + Project string `json:"project,omitempty"` + SKU string `json:"sku,omitempty"` + ResourceID string `json:"resource_id,omitempty"` + + // Dimensions are identity-bearing attributes used for grouping/pricing. + Dimensions map[string]string `json:"dimensions,omitempty"` + // Properties are informational and never part of the identity. + Properties map[string]any `json:"properties,omitempty"` + + // Source is "collector" for events vBilling measured itself, or + // "ingest:" for events pushed through the ingest API. + Source string `json:"source,omitempty"` + RecordedAt time.Time `json:"recorded_at"` +} + +// Tenant is a billing customer. One tenant can own several tenant clusters. +type Tenant struct { + ID string `json:"id"` + DisplayName string `json:"display_name"` + Email string `json:"email,omitempty"` + Currency string `json:"currency,omitempty"` + Region string `json:"region,omitempty"` + Project string `json:"project,omitempty"` + Class string `json:"class,omitempty"` + Plan string `json:"plan,omitempty"` + Clusters []string `json:"clusters,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + // ProviderIDs carries IDs the operator pinned for a provider, e.g. + // {"stripe": "cus_123"} from a tenant-cluster annotation. + ProviderIDs map[string]string `json:"provider_ids,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// DeriveID returns the deterministic ID for an event. The identity is the +// tenant, metric, window, region, project, SKU, resource ID and the sorted +// dimension set. Quantity and properties are deliberately excluded: the same +// observation re-measured after a restart must map to the same ID. +func DeriveID(e *Event) string { + var b strings.Builder + b.WriteString(SchemaVersion) + for _, part := range []string{ + e.Tenant, e.Metric, + e.WindowStart.UTC().Format(time.RFC3339Nano), + e.WindowEnd.UTC().Format(time.RFC3339Nano), + e.Region, e.Project, e.SKU, e.ResourceID, + } { + b.WriteByte(0) + b.WriteString(part) + } + keys := make([]string, 0, len(e.Dimensions)) + for k := range e.Dimensions { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + b.WriteByte(0) + b.WriteString(k) + b.WriteByte('=') + b.WriteString(e.Dimensions[k]) + } + sum := sha256.Sum256([]byte(b.String())) + return "vb1_" + hex.EncodeToString(sum[:16]) +} + +// Finalize fills schema version, unit, rounding and the derived ID. It is +// what the collector calls on every event it builds. +func (e *Event) Finalize() { + e.SchemaVersion = SchemaVersion + if e.Unit == "" { + if def, ok := Lookup(e.Metric); ok { + e.Unit = def.Unit + } + } + e.Quantity = Round(e.Quantity) + e.WindowStart = e.WindowStart.UTC() + e.WindowEnd = e.WindowEnd.UTC() + if e.ID == "" { + e.ID = DeriveID(e) + } +} + +var ( + metricRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,99}$`) + dimKeyRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + idRe = regexp.MustCompile(`^[A-Za-z0-9._:\-]{1,100}$`) +) + +// Validate checks the invariants every destination relies on. +func (e *Event) Validate() error { + var errs []error + if !idRe.MatchString(e.ID) { + errs = append(errs, fmt.Errorf("id %q must be 1-100 chars of [A-Za-z0-9._:-]", e.ID)) + } + if e.Tenant == "" || len(e.Tenant) > 128 { + errs = append(errs, errors.New("tenant must be 1-128 chars")) + } + if !metricRe.MatchString(e.Metric) { + errs = append(errs, fmt.Errorf("metric %q must match %s", e.Metric, metricRe)) + } + if math.IsNaN(e.Quantity) || math.IsInf(e.Quantity, 0) || e.Quantity < 0 { + errs = append(errs, fmt.Errorf("quantity %v must be a finite, non-negative number", e.Quantity)) + } + if e.WindowStart.IsZero() || e.WindowEnd.IsZero() || !e.WindowEnd.After(e.WindowStart) { + errs = append(errs, errors.New("window_end must be after window_start")) + } + for k := range e.Dimensions { + if !dimKeyRe.MatchString(k) { + errs = append(errs, fmt.Errorf("dimension key %q must match %s", k, dimKeyRe)) + } + } + return errors.Join(errs...) +} + +// Dim returns a dimension value or "". +func (e *Event) Dim(key string) string { + if e.Dimensions == nil { + return "" + } + return e.Dimensions[key] +} + +// Window returns the window of the given size that contains t, aligned to +// multiples of size since the Unix epoch. +func Window(t time.Time, size time.Duration) (start, end time.Time) { + start = t.UTC().Truncate(size) + return start, start.Add(size) +} + +// Overlap returns how much of [start, end) falls inside [ws, we). A zero end +// means the interval is still open. +func Overlap(start, end, ws, we time.Time) time.Duration { + if end.IsZero() || end.After(we) { + end = we + } + if start.Before(ws) { + start = ws + } + if !end.After(start) { + return 0 + } + return end.Sub(start) +} + +// Round keeps nine decimal places, enough for per-second GPU-hours +// (1s = 0.000277778h) without float noise leaking into invoices. +func Round(f float64) float64 { + return math.Round(f*1e9) / 1e9 +} diff --git a/internal/usage/usage_test.go b/internal/usage/usage_test.go new file mode 100644 index 0000000..065952f --- /dev/null +++ b/internal/usage/usage_test.go @@ -0,0 +1,163 @@ +package usage + +import ( + "math" + "strings" + "testing" + "time" +) + +func baseEvent() Event { + ws := time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) + return Event{ + Tenant: "acme", + Metric: MetricGPUHours, + Quantity: 8.0 / 60, + WindowStart: ws, + WindowEnd: ws.Add(time.Minute), + Region: "ap-southeast-2", + SKU: "NVIDIA-H100-80GB-HBM3", + ResourceID: "vcluster-team-a-gpu", + Dimensions: map[string]string{DimGPUType: "NVIDIA-H100-80GB-HBM3", DimCapacityType: CapacityOnDemand}, + } +} + +func TestDeriveIDIsDeterministicAndIgnoresQuantity(t *testing.T) { + a, b := baseEvent(), baseEvent() + b.Quantity = 99 + b.Properties = map[string]any{"noise": true} + if DeriveID(&a) != DeriveID(&b) { + t.Fatal("quantity/properties must not change the ID") + } + if !strings.HasPrefix(DeriveID(&a), "vb1_") || len(DeriveID(&a)) != 36 { + t.Fatalf("unexpected ID shape %q", DeriveID(&a)) + } +} + +func TestDeriveIDChangesWithIdentity(t *testing.T) { + base := baseEvent() + id := DeriveID(&base) + mutations := map[string]func(*Event){ + "tenant": func(e *Event) { e.Tenant = "other" }, + "metric": func(e *Event) { e.Metric = MetricCPUCoreHours }, + "window": func(e *Event) { + e.WindowStart = e.WindowStart.Add(time.Minute) + e.WindowEnd = e.WindowEnd.Add(time.Minute) + }, + "region": func(e *Event) { e.Region = "ap-southeast-4" }, + "sku": func(e *Event) { e.SKU = "other" }, + "resource": func(e *Event) { e.ResourceID = "other" }, + "dimension": func(e *Event) { e.Dimensions[DimCapacityType] = CapacitySpot }, + "new dim": func(e *Event) { e.Dimensions["extra"] = "x" }, + } + for name, mutate := range mutations { + e := baseEvent() + mutate(&e) + if DeriveID(&e) == id { + t.Errorf("%s change did not change the ID", name) + } + } +} + +func TestDeriveIDIsTimezoneIndependent(t *testing.T) { + a := baseEvent() + b := baseEvent() + syd := time.FixedZone("AEST", 10*3600) + b.WindowStart = b.WindowStart.In(syd) + b.WindowEnd = b.WindowEnd.In(syd) + if DeriveID(&a) != DeriveID(&b) { + t.Fatal("same instant in another zone must produce the same ID") + } +} + +func TestFinalizeAndValidate(t *testing.T) { + e := baseEvent() + e.Finalize() + if e.SchemaVersion != SchemaVersion || e.Unit != "gpu-hours" || e.ID == "" { + t.Fatalf("finalize did not fill fields: %+v", e) + } + if err := e.Validate(); err != nil { + t.Fatalf("valid event rejected: %v", err) + } + + bad := baseEvent() + bad.Finalize() + bad.Quantity = math.NaN() + bad.Metric = "Bad-Metric" + bad.WindowEnd = bad.WindowStart + bad.Dimensions["Bad Key"] = "x" + err := bad.Validate() + if err == nil { + t.Fatal("invalid event accepted") + } + for _, want := range []string{"quantity", "metric", "window_end", "dimension key"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestWindowAlignment(t *testing.T) { + ts := time.Date(2026, 10, 5, 10, 7, 42, 500, time.UTC) + ws, we := Window(ts, 5*time.Minute) + if !ws.Equal(time.Date(2026, 10, 5, 10, 5, 0, 0, time.UTC)) || !we.Equal(ws.Add(5*time.Minute)) { + t.Fatalf("got [%s, %s)", ws, we) + } +} + +func TestOverlap(t *testing.T) { + ws := time.Date(2026, 10, 5, 10, 0, 0, 0, time.UTC) + we := ws.Add(time.Minute) + cases := []struct { + name string + start, end time.Time + want time.Duration + }{ + {"open interval started before", ws.Add(-time.Hour), time.Time{}, time.Minute}, + {"started mid window", ws.Add(20 * time.Second), time.Time{}, 40 * time.Second}, + {"ended mid window", ws.Add(-time.Hour), ws.Add(15 * time.Second), 15 * time.Second}, + {"inside window", ws.Add(10 * time.Second), ws.Add(25 * time.Second), 15 * time.Second}, + {"ended before window", ws.Add(-time.Hour), ws.Add(-time.Second), 0}, + {"starts after window", we.Add(time.Second), time.Time{}, 0}, + } + for _, c := range cases { + if got := Overlap(c.start, c.end, ws, we); got != c.want { + t.Errorf("%s: got %s want %s", c.name, got, c.want) + } + } +} + +func TestCustomMetrics(t *testing.T) { + defer resetCustomForTest() + defs, err := ParseCustomMetrics("inference_output_tokens:tokens:model|region, slurm_gpu_hours:gpu-hours") + if err != nil { + t.Fatal(err) + } + for _, d := range defs { + if err := RegisterCustom(d); err != nil { + t.Fatal(err) + } + } + d, ok := Lookup("inference_output_tokens") + if !ok || d.Unit != "tokens" || !d.Billable || !d.Custom || len(d.GroupKeys) != 2 { + t.Fatalf("unexpected custom def %+v", d) + } + if err := RegisterCustom(MetricDef{Code: MetricGPUHours}); err == nil { + t.Fatal("expected collision with builtin metric") + } + if err := RegisterCustom(MetricDef{Code: "Bad Code"}); err == nil { + t.Fatal("expected invalid code error") + } + found := false + for _, c := range Billable() { + if c.Code == "slurm_gpu_hours" { + found = true + } + if c.Code == MetricGPUUtilization { + t.Error("informational metric listed as billable") + } + } + if !found { + t.Error("custom metric missing from Billable()") + } +} From 6dc4e347cfe8b9101bb8e2c48ad89d962edd3d15 Mon Sep 17 00:00:00 2001 From: Saiyam Pathak Date: Mon, 5 Oct 2026 23:42:27 +0530 Subject: [PATCH 2/4] Ship the v0.2 Helm chart, image and demo scripts The chart runs vBilling as a StatefulSet with a ledger volume, a Service, RBAC that only grows with the features enabled, the suspension admission policy, a ServiceMonitor, and Secrets or existingSecret for every key. The Lago demo now generates its RSA signing key at runtime. --- Dockerfile | 10 +- Makefile | 13 +- deploy/helm/vbilling/Chart.yaml | 12 +- deploy/helm/vbilling/templates/NOTES.txt | 17 ++ deploy/helm/vbilling/templates/_helpers.tpl | 33 +++ .../vbilling/templates/admissionpolicy.yaml | 35 +++ .../helm/vbilling/templates/clusterrole.yaml | 61 +++-- .../templates/prometheus-metrics.yaml | 12 + deploy/helm/vbilling/templates/secret.yaml | 30 +++ deploy/helm/vbilling/templates/service.yaml | 15 ++ .../vbilling/templates/servicemonitor.yaml | 17 ++ .../helm/vbilling/templates/statefulset.yaml | 242 ++++++++++++++++++ deploy/helm/vbilling/values.yaml | 201 ++++++++++++--- deploy/lago/docker-compose.yml | 4 +- scripts/demo.sh | 44 +--- scripts/e2e-kind.sh | 177 +++++++++++++ 16 files changed, 818 insertions(+), 105 deletions(-) create mode 100644 deploy/helm/vbilling/templates/NOTES.txt create mode 100644 deploy/helm/vbilling/templates/admissionpolicy.yaml create mode 100644 deploy/helm/vbilling/templates/prometheus-metrics.yaml create mode 100644 deploy/helm/vbilling/templates/secret.yaml create mode 100644 deploy/helm/vbilling/templates/service.yaml create mode 100644 deploy/helm/vbilling/templates/servicemonitor.yaml create mode 100644 deploy/helm/vbilling/templates/statefulset.yaml create mode 100755 scripts/e2e-kind.sh diff --git a/Dockerfile b/Dockerfile index 01d446a..be4e546 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,15 @@ FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder -ARG TARGETOS=linux -ARG TARGETARCH=amd64 +# Filled in by BuildKit from --platform (or the host); no defaults, so a plain +# `docker build` on arm64 produces an arm64 binary. +ARG TARGETOS +ARG TARGETARCH WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /vbilling ./cmd/vbilling +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o /vbilling ./cmd/vbilling FROM gcr.io/distroless/static:nonroot COPY --from=builder /vbilling /vbilling -USER nonroot:nonroot +USER 65532:65532 ENTRYPOINT ["/vbilling"] diff --git a/Makefile b/Makefile index cbd9de8..0150a77 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BINARY := vbilling IMAGE := ghcr.io/vclusterlabs-experiments/vbilling TAG ?= latest -.PHONY: build run test docker-build docker-push clean tidy +.PHONY: build run test lint e2e docker-build docker-push clean tidy build: CGO_ENABLED=0 go build -o bin/$(BINARY) ./cmd/vbilling @@ -11,7 +11,16 @@ run: build ./bin/$(BINARY) test: - go test ./... -v -race + go test ./... -race + +lint: + @test -z "$$(gofmt -l cmd internal)" || (gofmt -l cmd internal; exit 1) + go vet ./... + +# Full end-to-end run on kind with real tenant clusters, stripe-mock and a +# signed-webhook receiver (needs docker, kind, vcluster, helm, jq). +e2e: + ./scripts/e2e-kind.sh docker-build: docker build -t $(IMAGE):$(TAG) . diff --git a/deploy/helm/vbilling/Chart.yaml b/deploy/helm/vbilling/Chart.yaml index 96cee5c..0e076f5 100644 --- a/deploy/helm/vbilling/Chart.yaml +++ b/deploy/helm/vbilling/Chart.yaml @@ -1,20 +1,20 @@ apiVersion: v2 name: vbilling -description: vBilling — adapter-agnostic usage-based billing pipe for Tenant Clusters +description: vBilling meters tenant clusters and ships durable, reconcilable usage events to Stripe, Metronome, Lago or your own pipeline type: application -version: 0.1.0 -appVersion: "0.1.0" +version: 0.2.0 +appVersion: "0.2.0" home: https://github.com/vclusterlabs-experiments/vbilling keywords: - vcluster - billing - metering - - lago - - metronome - stripe + - metronome + - lago + - gpu - kubernetes - tenant-isolation - - gpu maintainers: - name: vCluster Labs url: https://www.vcluster.com diff --git a/deploy/helm/vbilling/templates/NOTES.txt b/deploy/helm/vbilling/templates/NOTES.txt new file mode 100644 index 0000000..c2bfa3f --- /dev/null +++ b/deploy/helm/vbilling/templates/NOTES.txt @@ -0,0 +1,17 @@ +vBilling {{ .Chart.AppVersion }} is metering tenant clusters in region "{{ .Values.region }}". +Destinations: {{ include "vbilling.adapters" . }} + + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "vbilling.fullname" . }} {{ .Values.api.service.port }} + curl -s localhost:{{ .Values.api.service.port }}/api/v1/status # destinations, lag, last window + curl -s localhost:{{ .Values.api.service.port }}/api/v1/usage?group_by=tenant,sku + curl -s localhost:{{ .Values.api.service.port }}/api/v1/reconcile # ledger vs billing backend, previous UTC day + open http://localhost:{{ .Values.api.service.port }}/ # usage dashboard +{{- if not .Values.persistence.enabled }} + +WARNING: persistence is disabled. The ledger is the durable outbox for usage that +has not reached your billing backend yet; without a volume it is lost on restart. +{{- end }} +{{- if not .Values.api.token }} + +WARNING: api.token is empty, so usage and admin endpoints are unauthenticated. Keep the Service internal. +{{- end }} diff --git a/deploy/helm/vbilling/templates/_helpers.tpl b/deploy/helm/vbilling/templates/_helpers.tpl index 0bc0203..3d449be 100644 --- a/deploy/helm/vbilling/templates/_helpers.tpl +++ b/deploy/helm/vbilling/templates/_helpers.tpl @@ -50,3 +50,36 @@ Service account name {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Adapter list (falls back to the deprecated single `adapter` value). +*/}} +{{- define "vbilling.adapters" -}} +{{- $a := .Values.adapters | default list -}} +{{- if and (eq (len $a) 0) .Values.adapter -}} +{{- $a = list .Values.adapter -}} +{{- end -}} +{{- join "," $a -}} +{{- end }} + +{{/* +Secret-backed env var. Uses the component's existingSecret when set, +otherwise the chart-managed secret when an inline value is given. +Args: dict "env" NAME "existing" SECRET "key" KEY "inline" VALUE "chartKey" KEY "root" $ +*/}} +{{- define "vbilling.secretEnv" -}} +{{- if .existing }} +- name: {{ .env }} + valueFrom: + secretKeyRef: + name: {{ .existing }} + key: {{ .key }} + optional: true +{{- else if .inline }} +- name: {{ .env }} + valueFrom: + secretKeyRef: + name: {{ include "vbilling.fullname" .root }} + key: {{ .chartKey }} +{{- end }} +{{- end }} diff --git a/deploy/helm/vbilling/templates/admissionpolicy.yaml b/deploy/helm/vbilling/templates/admissionpolicy.yaml new file mode 100644 index 0000000..930d296 --- /dev/null +++ b/deploy/helm/vbilling/templates/admissionpolicy.yaml @@ -0,0 +1,35 @@ +{{- if .Values.enforcement.admissionPolicy.enabled }} +# Blocks new pods in namespaces vBilling labeled suspended (ENFORCEMENT_MODE=enforce). +# Running workloads keep running; the tenant cluster's own control plane may restart. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ include "vbilling.fullname" . }}-suspended-tenants + labels: + {{- include "vbilling.labels" . | nindent 4 }} +spec: + failurePolicy: Fail + matchConstraints: + namespaceSelector: + matchLabels: + vbilling.vcluster.com/suspended: "true" + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["pods"] + validations: + - expression: "has(object.metadata.labels) && 'app' in object.metadata.labels && object.metadata.labels['app'] == 'vcluster'" + message: "This tenant is suspended for billing. New workloads are blocked until the account is current; running workloads are not affected." + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ include "vbilling.fullname" . }}-suspended-tenants + labels: + {{- include "vbilling.labels" . | nindent 4 }} +spec: + policyName: {{ include "vbilling.fullname" . }}-suspended-tenants + validationActions: [Deny] +{{- end }} diff --git a/deploy/helm/vbilling/templates/clusterrole.yaml b/deploy/helm/vbilling/templates/clusterrole.yaml index 9f89538..764d85b 100644 --- a/deploy/helm/vbilling/templates/clusterrole.yaml +++ b/deploy/helm/vbilling/templates/clusterrole.yaml @@ -5,31 +5,50 @@ metadata: labels: {{- include "vbilling.labels" . | nindent 4 }} rules: - # Discover vClusters via StatefulSets and Deployments + # Discover tenant clusters (StatefulSet or Deployment control planes). - apiGroups: ["apps"] resources: ["statefulsets", "deployments"] - verbs: ["list", "watch", "get"] - # Read pod details for GPU allocation and node mapping + verbs: ["get", "list", "watch"] + # Meter synced workloads, volumes, load balancers and nodes; read namespace + # metadata. watch: pods and nodes deleted between windows are still billed. - apiGroups: [""] - resources: ["pods"] - verbs: ["list", "get"] - # Read PVCs for storage billing - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["list", "get"] - # Read services for LoadBalancer billing - - apiGroups: [""] - resources: ["services"] - verbs: ["list", "get"] - # Read nodes for GPU type detection and spot/on-demand status + resources: ["pods", "persistentvolumeclaims", "services", "nodes", "namespaces"] + verbs: ["get", "list", "watch"] + # GPUs allocated through Dynamic Resource Allocation. + - apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims", "resourceslices"] + verbs: ["get", "list"] + {{- if .Values.tenantAPI.enabled }} + # Tenant clusters with their own nodes: only the read-only kubeconfig + # vCluster exports for vBilling. - apiGroups: [""] - resources: ["nodes"] - verbs: ["list", "get"] - # Read namespaces for vCluster namespace enumeration + resources: ["secrets"] + resourceNames: [{{ .Values.tenantAPI.secretName | quote }}] + verbs: ["get"] + {{- if .Values.tenantAPI.adminFallback }} + # tenantAPI.adminFallback: tenant clusters' admin kubeconfigs (vc-). - apiGroups: [""] - resources: ["namespaces"] - verbs: ["list", "get"] - # Read pod metrics from metrics-server + resources: ["secrets"] + verbs: ["get"] + {{- end }} + {{- end }} + # Usage-based CPU and memory from metrics-server. - apiGroups: ["metrics.k8s.io"] resources: ["pods"] - verbs: ["list", "get"] + verbs: ["get", "list"] + # Optional enrichment from vCluster Platform (project, display name). + - apiGroups: ["management.loft.sh"] + resources: ["virtualclusterinstances"] + verbs: ["get", "list"] + {{- if ne .Values.enforcement.mode "observe" }} + # Billing-state enforcement: annotate tenant clusters, label namespaces, emit events. + - apiGroups: ["apps"] + resources: ["statefulsets", "deployments"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create"] + {{- end }} diff --git a/deploy/helm/vbilling/templates/prometheus-metrics.yaml b/deploy/helm/vbilling/templates/prometheus-metrics.yaml new file mode 100644 index 0000000..ce45c5c --- /dev/null +++ b/deploy/helm/vbilling/templates/prometheus-metrics.yaml @@ -0,0 +1,12 @@ +{{- if .Values.prometheus.metrics }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "vbilling.fullname" . }}-prometheus-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "vbilling.labels" . | nindent 4 }} +data: + metrics.yaml: | + {{- toYaml .Values.prometheus.metrics | nindent 4 }} +{{- end }} diff --git a/deploy/helm/vbilling/templates/secret.yaml b/deploy/helm/vbilling/templates/secret.yaml new file mode 100644 index 0000000..496a39a --- /dev/null +++ b/deploy/helm/vbilling/templates/secret.yaml @@ -0,0 +1,30 @@ +{{- $data := dict -}} +{{- if and .Values.lago.apiKey (not .Values.lago.existingSecret) }}{{ $_ := set $data "lago-api-key" .Values.lago.apiKey }}{{ end -}} +{{- if not .Values.stripe.existingSecret }} +{{- if .Values.stripe.apiKey }}{{ $_ := set $data "stripe-api-key" .Values.stripe.apiKey }}{{ end }} +{{- if .Values.stripe.webhookSecret }}{{ $_ := set $data "stripe-webhook-secret" .Values.stripe.webhookSecret }}{{ end }} +{{- end -}} +{{- if not .Values.metronome.existingSecret }} +{{- if .Values.metronome.apiToken }}{{ $_ := set $data "metronome-api-token" .Values.metronome.apiToken }}{{ end }} +{{- if .Values.metronome.webhookSecret }}{{ $_ := set $data "metronome-webhook-secret" .Values.metronome.webhookSecret }}{{ end }} +{{- end -}} +{{- if and .Values.prometheus.headers (not .Values.prometheus.existingSecret) }}{{ $h := list }}{{ range $k, $v := .Values.prometheus.headers }}{{ $h = append $h (printf "%s=%s" $k $v) }}{{ end }}{{ $_ := set $data "prometheus-headers" (join "," $h) }}{{ end -}} +{{- if and .Values.webhook.secret (not .Values.webhook.existingSecret) }}{{ $_ := set $data "webhook-secret" .Values.webhook.secret }}{{ end -}} +{{- if not .Values.api.existingSecret }} +{{- if .Values.api.token }}{{ $_ := set $data "api-token" .Values.api.token }}{{ end }} +{{- if .Values.api.ingestToken }}{{ $_ := set $data "ingest-token" .Values.api.ingestToken }}{{ end }} +{{- end -}} +{{- if $data }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "vbilling.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "vbilling.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- range $k, $v := $data }} + {{ $k }}: {{ $v | quote }} + {{- end }} +{{- end }} diff --git a/deploy/helm/vbilling/templates/service.yaml b/deploy/helm/vbilling/templates/service.yaml new file mode 100644 index 0000000..9426f37 --- /dev/null +++ b/deploy/helm/vbilling/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "vbilling.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "vbilling.labels" . | nindent 4 }} +spec: + type: {{ .Values.api.service.type }} + selector: + {{- include "vbilling.selectorLabels" . | nindent 4 }} + ports: + - name: http + port: {{ .Values.api.service.port }} + targetPort: http diff --git a/deploy/helm/vbilling/templates/servicemonitor.yaml b/deploy/helm/vbilling/templates/servicemonitor.yaml new file mode 100644 index 0000000..370b99f --- /dev/null +++ b/deploy/helm/vbilling/templates/servicemonitor.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "vbilling.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "vbilling.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "vbilling.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + path: /metrics + interval: {{ .Values.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/helm/vbilling/templates/statefulset.yaml b/deploy/helm/vbilling/templates/statefulset.yaml new file mode 100644 index 0000000..a713cce --- /dev/null +++ b/deploy/helm/vbilling/templates/statefulset.yaml @@ -0,0 +1,242 @@ +{{- $adapters := include "vbilling.adapters" . | splitList "," -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "vbilling.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "vbilling.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + serviceName: {{ include "vbilling.fullname" . }} + selector: + matchLabels: + {{- include "vbilling.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "vbilling.selectorLabels" . | nindent 8 }} + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/prometheus-metrics: {{ toYaml .Values.prometheus.metrics | sha256sum }} + spec: + serviceAccountName: {{ include "vbilling.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: vbilling + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + ports: + - name: http + containerPort: {{ .Values.api.port }} + readinessProbe: + httpGet: {path: /readyz, port: http} + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + periodSeconds: 20 + env: + - name: ADAPTERS + value: {{ include "vbilling.adapters" . | quote }} + - name: REGION + value: {{ .Values.region | quote }} + - name: CLUSTER_NAME + value: {{ .Values.clusterName | quote }} + - name: DEFAULT_TENANT_CLASS + value: {{ .Values.tenantClass | quote }} + - name: TENANT_SOURCE + value: {{ .Values.tenantSource | quote }} + - name: DATA_DIR + value: /var/lib/vbilling + - name: RETENTION + value: {{ .Values.persistence.retention | quote }} + - name: LISTEN_ADDR + value: ":{{ .Values.api.port }}" + - name: DEFAULT_PLAN_CODE + value: {{ .Values.billing.defaultPlanCode | quote }} + - name: BILLING_CURRENCY + value: {{ .Values.billing.currency | quote }} + - name: COLLECTION_INTERVAL + value: {{ .Values.billing.collectionInterval | quote }} + - name: RECONCILE_INTERVAL + value: {{ .Values.billing.reconcileInterval | quote }} + - name: MAX_BACKFILL + value: {{ .Values.billing.maxBackfill | quote }} + - name: OFFBOARD_GRACE + value: {{ .Values.billing.offboardGrace | quote }} + - name: CPU_MEMORY_BASIS + value: {{ .Values.metering.cpuMemoryBasis | quote }} + - name: METER_CONTROL_PLANE + value: {{ .Values.metering.meterControlPlane | quote }} + - name: METER_BY_NAMESPACE + value: {{ .Values.metering.meterByNamespace | quote }} + - name: GPU_RESOURCES + value: {{ join "," .Values.metering.gpuResources | default "none" | quote }} + - name: SKU_LABEL + value: {{ .Values.metering.skuLabel | quote }} + - name: CAPACITY_TYPE_LABEL + value: {{ .Values.metering.capacityTypeLabel | quote }} + - name: UNHEALTHY_NODE_TAINTS + value: {{ join "," .Values.metering.unhealthyNodeTaints | default "none" | quote }} + - name: VCLUSTER_NODE_LABEL + value: {{ .Values.metering.dedicatedNodeLabel | quote }} + - name: REGION_FROM_NODE + value: {{ .Values.metering.regionFromNode | quote }} + - name: CUSTOM_METRICS + value: {{ .Values.metering.customMetrics | quote }} + - name: WATCH_DELETIONS + value: {{ .Values.metering.watchDeletions | quote }} + - name: DRA_GPU_DRIVERS + value: {{ join "," .Values.metering.draGPUDrivers | default "none" | quote }} + - name: TENANT_API + value: {{ .Values.tenantAPI.enabled | quote }} + - name: TENANT_KUBECONFIG_SECRET + value: {{ .Values.tenantAPI.secretName | quote }} + - name: TENANT_ADMIN_FALLBACK + value: {{ .Values.tenantAPI.adminFallback | quote }} + - name: PRIVATE_NODE_BILLING + value: {{ .Values.tenantAPI.privateNodeBilling | quote }} + - name: TENANT_EXCLUDE_NAMESPACES + value: {{ join "," .Values.tenantAPI.excludeNamespaces | default "none" | quote }} + {{- if .Values.tenantAPI.externalClustersSecret }} + - name: TENANT_CLUSTERS_FILE + value: /etc/vbilling/tenants/clusters.yaml + {{- end }} + - name: PLATFORM_CLUSTER + value: {{ .Values.platform.cluster | quote }} + - name: WATCH_NAMESPACES + value: {{ join "," .Values.watchNamespaces | quote }} + - name: ENFORCEMENT_MODE + value: {{ .Values.enforcement.mode | quote }} + - name: ENFORCEMENT_RULES + value: {{ .Values.enforcement.rules | quote }} + {{- if .Values.prometheus.url }} + - name: PROMETHEUS_URL + value: {{ .Values.prometheus.url | quote }} + - name: EGRESS_QUERY + value: {{ .Values.prometheus.egressQuery | quote }} + - name: GPU_UTIL_QUERY + value: {{ .Values.prometheus.gpuUtilQuery | quote }} + - name: GPU_COUNT_QUERY + value: {{ .Values.prometheus.gpuCountQuery | quote }} + - name: PROMETHEUS_PRESET + value: {{ .Values.prometheus.preset | quote }} + {{- if .Values.prometheus.metrics }} + - name: PROMETHEUS_METRICS_FILE + value: /etc/vbilling/prometheus/metrics.yaml + {{- end }} + {{- if .Values.prometheus.bearerTokenFile }} + - name: PROMETHEUS_BEARER_TOKEN_FILE + value: {{ .Values.prometheus.bearerTokenFile | quote }} + {{- end }} + {{- include "vbilling.secretEnv" (dict "env" "PROMETHEUS_HEADERS" "existing" .Values.prometheus.existingSecret "key" "headers" "inline" .Values.prometheus.headers "chartKey" "prometheus-headers" "root" .) | nindent 12 }} + {{- end }} + {{- include "vbilling.secretEnv" (dict "env" "API_TOKEN" "existing" .Values.api.existingSecret "key" "api-token" "inline" .Values.api.token "chartKey" "api-token" "root" .) | nindent 12 }} + {{- include "vbilling.secretEnv" (dict "env" "INGEST_TOKEN" "existing" .Values.api.existingSecret "key" "ingest-token" "inline" .Values.api.ingestToken "chartKey" "ingest-token" "root" .) | nindent 12 }} + {{- if has "lago" $adapters }} + - name: LAGO_API_URL + value: {{ .Values.lago.apiURL | quote }} + {{- include "vbilling.secretEnv" (dict "env" "LAGO_API_KEY" "existing" .Values.lago.existingSecret "key" "api-key" "inline" .Values.lago.apiKey "chartKey" "lago-api-key" "root" .) | nindent 12 }} + {{- end }} + {{- if or (has "stripe" $adapters) .Values.metronome.stripeLink }} + - name: STRIPE_API_BASE + value: {{ .Values.stripe.apiBase | quote }} + - name: STRIPE_API_VERSION + value: {{ .Values.stripe.apiVersion | quote }} + - name: STRIPE_SPLIT_METERS_BY + value: {{ join "," .Values.stripe.splitMetersBy | default "none" | quote }} + - name: STRIPE_AUTO_SUBSCRIBE + value: {{ .Values.stripe.autoSubscribe | quote }} + - name: STRIPE_MAX_RPS + value: {{ .Values.stripe.maxRPS | quote }} + - name: STRIPE_CANCEL_ON_REMOVE + value: {{ .Values.stripe.cancelOnRemove | quote }} + {{- include "vbilling.secretEnv" (dict "env" "STRIPE_API_KEY" "existing" .Values.stripe.existingSecret "key" "api-key" "inline" .Values.stripe.apiKey "chartKey" "stripe-api-key" "root" .) | nindent 12 }} + {{- end }} + {{- include "vbilling.secretEnv" (dict "env" "STRIPE_WEBHOOK_SECRET" "existing" .Values.stripe.existingSecret "key" "webhook-secret" "inline" .Values.stripe.webhookSecret "chartKey" "stripe-webhook-secret" "root" .) | nindent 12 }} + {{- if has "metronome" $adapters }} + - name: METRONOME_API_BASE + value: {{ .Values.metronome.apiBase | quote }} + - name: METRONOME_RATE_CARD + value: {{ .Values.metronome.rateCard | quote }} + - name: METRONOME_STRIPE_LINK + value: {{ .Values.metronome.stripeLink | quote }} + - name: METRONOME_STRIPE_COLLECTION_METHOD + value: {{ .Values.metronome.stripeCollectionMethod | quote }} + {{- include "vbilling.secretEnv" (dict "env" "METRONOME_API_TOKEN" "existing" .Values.metronome.existingSecret "key" "api-token" "inline" .Values.metronome.apiToken "chartKey" "metronome-api-token" "root" .) | nindent 12 }} + {{- end }} + {{- include "vbilling.secretEnv" (dict "env" "METRONOME_WEBHOOK_SECRET" "existing" .Values.metronome.existingSecret "key" "webhook-secret" "inline" .Values.metronome.webhookSecret "chartKey" "metronome-webhook-secret" "root" .) | nindent 12 }} + {{- if has "webhook" $adapters }} + - name: WEBHOOK_URL + value: {{ .Values.webhook.url | quote }} + {{- if .Values.webhook.headers }} + - name: WEBHOOK_HEADERS + value: {{ $h := list }}{{ range $k, $v := .Values.webhook.headers }}{{ $h = append $h (printf "%s=%s" $k $v) }}{{ end }}{{ join "," $h | quote }} + {{- end }} + {{- include "vbilling.secretEnv" (dict "env" "WEBHOOK_SECRET" "existing" .Values.webhook.existingSecret "key" "secret" "inline" .Values.webhook.secret "chartKey" "webhook-secret" "root" .) | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /var/lib/vbilling + {{- if .Values.tenantAPI.externalClustersSecret }} + - name: tenant-clusters + mountPath: /etc/vbilling/tenants + readOnly: true + {{- end }} + {{- if .Values.prometheus.metrics }} + - name: prometheus-metrics + mountPath: /etc/vbilling/prometheus + readOnly: true + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if or (not .Values.persistence.enabled) .Values.tenantAPI.externalClustersSecret .Values.prometheus.metrics }} + volumes: + {{- if not .Values.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- if .Values.tenantAPI.externalClustersSecret }} + - name: tenant-clusters + secret: + secretName: {{ .Values.tenantAPI.externalClustersSecret }} + {{- end }} + {{- if .Values.prometheus.metrics }} + - name: prometheus-metrics + configMap: + name: {{ include "vbilling.fullname" . }}-prometheus-metrics + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + {{- with .Values.persistence.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- end }} diff --git a/deploy/helm/vbilling/values.yaml b/deploy/helm/vbilling/values.yaml index 2aa77ee..b30c5a0 100644 --- a/deploy/helm/vbilling/values.yaml +++ b/deploy/helm/vbilling/values.yaml @@ -1,50 +1,169 @@ +# vBilling Helm values. Every value maps to an environment variable; see +# docs/configuration.html for the full reference. + replicaCount: 1 image: - # Build and push your own image: docker buildx build --platform linux/amd64,linux/arm64 -t /vbilling:latest --push . repository: ghcr.io/vclusterlabs-experiments/vbilling - tag: v0.1.0 + tag: v0.2.0 pullPolicy: IfNotPresent -# Billing adapter selection. Built-in: "lago", "noop". More adapters (Metronome, -# Stripe Meters, OpenMeter) are on the roadmap. Customers can also implement the -# Destination interface and register their own. -adapter: lago +# Destinations usage fans out to. Built-in: stripe, metronome, lago, webhook, noop. +# Each destination has its own cursor into the ledger, so one backend being +# down never blocks another. +adapters: + - lago +# Deprecated single-adapter form from v0.1; used only when `adapters` is empty. +adapter: "" -# Lago connection (used when adapter == "lago") -lago: - apiURL: "http://lago-api.lago-system.svc.cluster.local:3000" - apiKey: "" # REQUIRED when adapter=lago: set via --set or existingSecret - # Use existing secret for API key (key: "api-key") - existingSecret: "" +# Where this vBilling instance meters. Run one vBilling per control plane +# cluster (region): usage, ledger and billing state stay in that region. +region: default +clusterName: control-plane +# Default tenant class stamped on events (public, enterprise, government, dev, ...). +tenantClass: "" +# How tenant clusters map to billing customers: "cluster" (one customer per +# tenant cluster) or "project" (one per vCluster Platform project). The +# vbilling.vcluster.com/tenant label always wins. +tenantSource: cluster -# Billing configuration billing: - defaultPlanCode: "vcluster-standard" - currency: "USD" - collectionInterval: "60s" - reconcileInterval: "30s" - -# Pricing per unit (string format for decimal precision) -pricing: - cpuPerCoreHour: "0.05" - memoryPerGBHour: "0.01" - storagePerGBHour: "0.001" - instancePerHour: "0.10" - gpuDefaultPerHour: "2.50" - networkEgressPerGB: "0.05" - lbPerHour: "0.025" - # GPU type-specific pricing (comma-separated TYPE:PRICE) - gpuPrices: "NVIDIA-A100-SXM4-80GB:3.50,NVIDIA-H100-80GB-HBM3:5.00,NVIDIA-H100-SXM5-80GB:5.50,NVIDIA-L40S:1.80,NVIDIA-A10G:1.20,NVIDIA-T4:0.75" - spotDiscountPercent: 60 - -# Optional: Prometheus URL for DCGM + network metrics + defaultPlanCode: vcluster-standard # Lago plan code / Stripe plan tag + currency: USD + collectionInterval: 60s # metering window; must divide an hour + reconcileInterval: 30s + maxBackfill: 6h # windows missed during downtime are backfilled up to this age + offboardGrace: 1h # a tenant must be gone this long before RemoveTenant runs + +metering: + cpuMemoryBasis: usage # usage | requests | max + meterControlPlane: false # also bill the tenant cluster's own control plane pods + meterByNamespace: false # add the tenant cluster namespace as a dimension (internal chargeback) + gpuResources: [nvidia.com/gpu, amd.com/gpu] # MIG resources (nvidia.com/mig-*) are always detected + skuLabel: "" # node label whose value is the SKU (e.g. vbilling.vcluster.com/sku) + capacityTypeLabel: vbilling.vcluster.com/capacity-type # on-demand | spot | preemptible | reserved + unhealthyNodeTaints: [node.kubernetes.io/not-ready, node.kubernetes.io/unreachable] + dedicatedNodeLabel: "" # extra dedicated-node selector, "key=%s" (%s = tenant cluster name) + regionFromNode: false # use topology.kubernetes.io/region from nodes instead of `region` + # Operator metrics accepted by the ingest API, "code:unit[:key|key],..." + # e.g. "inference_output_tokens:tokens:model|region,slurm_gpu_hours:gpu-hours" + customMetrics: "" + watchDeletions: true # bill pods and nodes deleted between windows (short jobs, Auto Nodes scale-down) + draGPUDrivers: [gpu.nvidia.com, gpu.amd.com] # DRA drivers whose devices are GPUs + +# Tenant clusters with their own nodes (vCluster Private Nodes, Auto Nodes, +# Standalone) run workloads the control plane cluster never sees. vBilling +# meters them through their own API, using a read-only kubeconfig that vCluster +# exports into secretName (exportKubeConfig.additionalSecrets, see README). +# Nodes that also exist in the control plane cluster are never billed twice. +tenantAPI: + enabled: true + secretName: vbilling-kubeconfig + adminFallback: false # also use vc- admin kubeconfigs (grants vBilling read access to Secrets) + privateNodeBilling: node # node: bill private nodes whole | usage: bill the pods running on them + excludeNamespaces: [kube-system] # usage mode: namespaces never billed + # Tenant clusters outside this control plane cluster (vCluster Standalone): + # a Secret holding clusters.yaml plus the kubeconfig files it lists, mounted + # at /etc/vbilling/tenants. + externalClustersSecret: "" + prometheus: - url: "" # e.g. "http://prometheus.monitoring.svc.cluster.local:9090" + # Any Prometheus HTTP API (Prometheus, Thanos, Mimir, VictoriaMetrics). Enables + # egress and GPU utilization, e.g. http://prometheus.monitoring.svc:9090 + url: "" + egressQuery: "" # PromQL template with {{namespace}}, {{vcluster}}, {{window}} + # GPU utilization. The default reads DCGM_FI_DEV_GPU_UTIL with either the + # namespace or exported_namespace label. Same templates; "none" disables it. + gpuUtilQuery: "" + gpuCountQuery: "" # devices per GPU model, used with the default utilization query + # Auth. Headers are stored in the chart Secret, e.g. {X-Scope-OrgID: tenant-1} + # for Mimir or {Authorization: "Basic ..."}. Or point existingSecret at a + # Secret with key "headers" in "Name=value,Name2=value2" form. + headers: {} + existingSecret: "" + bearerTokenFile: "" # re-read per query, e.g. /var/run/secrets/kubernetes.io/serviceaccount/token + # "vcluster-platform": read vCluster Platform fleet observability, whose write + # gateway labels every series with vcluster_platform_instance/project. + preset: "" + # Metrics defined by PromQL, evaluated per tenant cluster at every window end. + # Templates: {{namespace}} {{vcluster}} {{instance}} {{project}} {{tenant}} {{tenant_cluster}} {{window}} {{window_seconds}} + metrics: [] + # - code: inference_output_tokens + # name: Inference output tokens + # unit: tokens + # billable: true + # dimensions: [model_name] + # sku: model_name + # query: sum by (model_name) (increase(vllm:generation_tokens_total{namespace="{{namespace}}"}[{{window}}])) + +platform: + cluster: "" # vCluster Platform cluster name of this control plane cluster (multi-cluster setups) -# Namespace filter (empty = watch all namespaces) watchNamespaces: [] +lago: + apiURL: "http://lago-api.lago-system.svc.cluster.local:3000" + apiKey: "" + existingSecret: "" # key: api-key + +stripe: + apiKey: "" # sk_test_... / sk_live_... / rk_... (restricted keys work) + existingSecret: "" # keys: api-key, webhook-secret + apiBase: https://api.stripe.com + apiVersion: "2026-09-30.endive" + # Stripe meters cannot price on dimensions, so meters are split by these + # event fields (H100 and L40S GPU-hours get separate meters and prices). + splitMetersBy: [sku] + autoSubscribe: false # subscribe tenants to metered prices tagged metadata[vbilling_plan]= + maxRPS: 0 # 0 = 20 in test mode, 200 in live mode + cancelOnRemove: false + webhookSecret: "" # whsec_... enables /webhooks/stripe (dunning, alerts) + +metronome: + apiToken: "" + existingSecret: "" # keys: api-token, webhook-secret + apiBase: https://api.metronome.com + rateCard: "" # rate card alias or ID: creates a contract per tenant + # Create a Stripe customer per tenant and link it in Metronome, so Metronome + # rates and Stripe collects payment, invoices and tax. Needs stripe.apiKey. + stripeLink: false + stripeCollectionMethod: charge_automatically + webhookSecret: "" # enables /webhooks/metronome (spend and credit alerts) + +webhook: + url: "" # CloudEvents batches to your data platform + secret: "" + existingSecret: "" # key: secret + headers: {} + +api: + port: 8080 + token: "" # protects usage and admin endpoints + ingestToken: "" # enables POST /api/v1/events (defaults to token) + existingSecret: "" # keys: api-token, ingest-token + service: + type: ClusterIP + port: 8080 + +enforcement: + # observe: record billing state only. annotate: also annotate tenant + # clusters and label their namespaces. enforce: also label suspended + # namespaces so the admission policy below blocks new pods. + mode: observe + rules: "" # "event.type=state,..." e.g. "alerts.spend_threshold_reached=suspended" + admissionPolicy: + enabled: false # ValidatingAdmissionPolicy (Kubernetes 1.30+) blocking new pods in suspended namespaces + +persistence: + enabled: true # the ledger is the durable outbox: keep it on a volume + size: 5Gi + storageClass: "" + retention: 168h + +serviceMonitor: + enabled: false + interval: 30s + serviceAccount: create: true name: "" @@ -53,10 +172,18 @@ serviceAccount: resources: requests: cpu: 50m - memory: 64Mi + memory: 96Mi limits: - cpu: 200m - memory: 128Mi + cpu: 500m + memory: 256Mi + +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault nodeSelector: {} tolerations: [] diff --git a/deploy/lago/docker-compose.yml b/deploy/lago/docker-compose.yml index 3876608..a6141b3 100644 --- a/deploy/lago/docker-compose.yml +++ b/deploy/lago/docker-compose.yml @@ -34,7 +34,7 @@ services: REDIS_URL: redis://redis:6379 SECRET_KEY_BASE: your-secret-key-base-for-demo-only-change-in-production-1234567890 LAGO_API_URL: http://localhost:3000 - LAGO_FRONT_URL: http://localhost:8080 + LAGO_FRONT_URL: http://localhost:8081 LAGO_RSA_PRIVATE_KEY: ${LAGO_RSA_PRIVATE_KEY} LAGO_ENCRYPTION_PRIMARY_KEY: demo-encryption-primary-key-at-least-32-chars-long-for-safety LAGO_ENCRYPTION_DETERMINISTIC_KEY: demo-deterministic-key-at-least-32-chars-long-for-safety @@ -88,7 +88,7 @@ services: API_URL: http://localhost:3000 APP_ENV: production ports: - - "8080:80" + - "8081:80" volumes: lago_pg_data: diff --git a/scripts/demo.sh b/scripts/demo.sh index 85c9448..0545e92 100755 --- a/scripts/demo.sh +++ b/scripts/demo.sh @@ -74,6 +74,8 @@ echo "" echo ">>> Step 3: Starting Lago (billing engine)..." mkdir -p "$LAGO_DIR" +# Lago signs its tokens with an RSA key: generate one for this demo, never commit one. +RSA_KEY="$(openssl genrsa -traditional 2048 2>/dev/null || openssl genrsa 2048 2>/dev/null)" cat > "$LAGO_DIR/docker-compose.yml" <<'COMPOSE' version: "3.8" @@ -113,35 +115,9 @@ services: REDIS_URL: redis://redis:6379 SECRET_KEY_BASE: your-secret-key-base-for-demo-only-change-in-production LAGO_API_URL: http://localhost:3000 - LAGO_FRONT_URL: http://localhost:8080 + LAGO_FRONT_URL: http://localhost:8081 RSA_PRIVATE_KEY: | - -----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn/yGJB+W7mmMCD+sCGpjdOdLqhwFOdhsrB - UtIhcezLbfCHA2Dc3bMwj7NPHM2GIFvZMh6tQ18BG9VLbgBrplIaS3VNwfdYLlWF - VNGURYDF0mWXw9MxXR7a+kT8GbdQMP7j62riBsTELY1SFSYB/FEM6gZlPysl7vEq - ktSpPbCIKbhsfGRaBJAjS0ISlvCi/l4o/JKMoXDbnGaPwbps/ZuMWjPqC0Nv2Y2B - MTbUUfR7kzVGUVQ7d/FY0vYxqDEsBQ6P7MhmN0JG/uaF3VbUfVpR9P+0Bmby+E2T - E1AQMRPN5jBseXlqwA3CkFsDJnAP7BfLBcCe8QIDAQABAoIBAE2W7Fz6cV3kQPbC - r3PvEa4B+c8VZF8RR1G0jWGCN+bCMi+o3yB7SXQKKQGZ3D3P1GWI4CTZY3EThG0 - oOzfFYXC/Rd09+bHxTBylHFdKRwTnaY5dStk+YjZEy1MNKFL4UcGIB94msGJzPEG - F0aBhxTqBSCkzZ3ID+NP3sRk5ZIlx+RqGqr0FFCIYgVBT3A8AS/XRaOL1END/yOo - SU7K9+zzW6sL6dqR3VIoK0vVkfxYX3mFzVISYYRvJO6kHU8HHaLFLCHzGlnxfJl7 - EQIuP42K3Bl3m7vX3MFN7i9N7a/FNNqXHBBYJBizGQ8dKf+a3u0P1zAj3o+y1K0E - mMEZ2QECgYEA56smzRdNllokVsKuGbBx5IRxlOjxjT9MEVXF3JCCYqe2+3Ku0xJK - OUBlr7GD6pFjw1FBBTCDFX+t0j+xrMmia5dFtAPjZ78VLe/1mlp90B24ezPOclfa - bYGwlFDI2FTQJBYa/i7T4M7VbL7s+n4V2XfyPJJYzZ+PuDDhnJ4EoECgYEA53T1 - T5mf7APvGa2j5b0W/g5KPHsrM6V9j0pNsMY7RYTLvlMkxDEgYrfMinYx6YWP9DEh - h2VU7Pq6VkCF+y6MiSIF0l4tXwK7C7xOSUCB1MfmXz/BOJPyGiw7PBqsW/R9A6u - G7N+G7kj7nKl1OZVbn3YZkjlm/U/e2nR1F+BhmECgYBkZ+hs5rlWp2dnVv/f0GhI - ihbR+MVP0zR5BPNpNPiIxx6BTZJNPU3x4W5eUaNfPqfV0ovGpWSP/hue1pS5LfTf - Y5N1MwHXlpYTFNYHXqZS0oB3mTBhxDLxI07CVGd+RXWmpHBfEgz6KOKVGL7IYR+d - n2xRMPJ5xpN3xT+D08F3AQKBgCoNNgNJ4MGBA1NJia9lM+EBvI9FKb+09MjQFfl1 - t3bCqRhLmsLfId9+BGlhY3L6pq3l+r3di2YhxtRBr/B3mKT4xZgoB78DFc7zuH9I - CILMi7C5I2JwIgaFF9uqjPdi6gW+vmJbPk5MTAd/yHp2SohcEJFVtKAHQ8GRWWN/ - 2z2hAoGBAMsHqW/dwLe0p/LrhGqEPp3iiKcCSvZLfJSe3h9cQ1SmFiPVbMgPcEaJ - kfDEsi9VwT31mLNjVdNjBBh5LSb0S2lInJC3dcSn3BN+WYnJH1SOEH7OYLZQ+Jxz - bGiW1JqQ5dIJdMrm/5+BRY87qB7TRPFhHqUOqaxMiZwH/ZVvuKR6 - -----END RSA PRIVATE KEY----- + __RSA_PRIVATE_KEY__ LAGO_ENCRYPTION_PRIMARY_KEY: demo-encryption-primary-key LAGO_ENCRYPTION_DETERMINISTIC_KEY: demo-encryption-deterministic-key LAGO_ENCRYPTION_KEY_DERIVATION_SALT: demo-encryption-derivation-salt @@ -192,11 +168,13 @@ services: API_URL: http://api:3000 APP_ENV: production ports: - - "8080:80" + - "8081:80" volumes: lago_pg_data: COMPOSE +RSA_KEY="$RSA_KEY" awk '/__RSA_PRIVATE_KEY__/ { n = split(ENVIRON["RSA_KEY"], k, "\n"); for (i = 1; i <= n; i++) print " " k[i]; next } { print }' \ + "$LAGO_DIR/docker-compose.yml" > "$LAGO_DIR/docker-compose.yml.tmp" && mv "$LAGO_DIR/docker-compose.yml.tmp" "$LAGO_DIR/docker-compose.yml" cd "$LAGO_DIR" docker compose up -d @@ -208,13 +186,13 @@ for i in $(seq 1 60); do fi sleep 2 done -echo " Lago UI: http://localhost:8080" +echo " Lago UI: http://localhost:8081" echo " Lago API: http://localhost:3000" echo "" # --- Step 4: Get Lago API key --- echo ">>> Step 4: Setting up Lago API key..." -echo " NOTE: On first use, open http://localhost:8080 to create an organization." +echo " NOTE: On first use, open http://localhost:8081 to create an organization." echo " Then go to Developer > API Keys to get your API key." echo "" echo " For this demo, you can set it via:" @@ -306,11 +284,11 @@ echo " │ └── vCluster: team-gpu (ML training)" echo " │" echo " └── Lago (billing engine)" echo " ├── API: http://localhost:3000" -echo " └── UI: http://localhost:8080" +echo " └── UI: http://localhost:8081" echo "" echo "Next steps:" echo "" -echo "1. Open Lago UI: http://localhost:8080" +echo "1. Open Lago UI: http://localhost:8081" echo " - Create an organization (first-time setup)" echo " - Go to Developer > API Keys > copy the API key" echo "" diff --git a/scripts/e2e-kind.sh b/scripts/e2e-kind.sh new file mode 100755 index 0000000..3669a65 --- /dev/null +++ b/scripts/e2e-kind.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# End-to-end test for vBilling against real tenant clusters. +# +# Builds a kind control plane cluster with three workers advertising +# simulated GPUs (an H100 node, an L40S spot node, a dedicated bare-metal +# H100 node), creates three vCluster tenant clusters with GPU workloads, and +# runs vBilling against them with two destinations: +# - stripe-mock (Stripe's official mock, validating every request against +# Stripe's OpenAPI spec) +# - a local receiver verifying vBilling's signed CloudEvents webhooks +# It then asserts metered quantities, delivery, ledger integrity, and the +# billing-state enforcement loop (signed Stripe webhook -> admission policy +# blocks new pods -> payment lifts the block). +# +# Usage: scripts/e2e-kind.sh [--keep] (--keep leaves everything running) +# Requires: docker, kind, kubectl, helm, vcluster (0.30+), go, jq, python3. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +WORK=${WORK:-$(mktemp -d -t vbilling-e2e)} +CLUSTER=vbilling-e2e +KEEP=${1:-} +API=127.0.0.1:18080 +TOKEN=e2e-api +STRIPE_WHSEC=whsec_e2e +WEBHOOK_SECRET=e2e-webhook-secret +export KUBECONFIG=$WORK/kubeconfig # never touches ~/.kube/config + +pass=0; fail=0 +ok() { echo " PASS $*"; pass=$((pass+1)); } +bad() { echo " FAIL $*"; fail=$((fail+1)); } +step() { echo; echo "==> $*"; } +check() { if eval "$2"; then ok "$1"; else bad "$1"; fi; } +api() { curl -fsS -H "Authorization: Bearer $TOKEN" "http://$API$1"; } + +PIDS=() +cleanup() { + if [ "$KEEP" != "--keep" ]; then + for p in "${PIDS[@]:-}"; do kill "$p" 2>/dev/null || true; done + kind delete cluster --name "$CLUSTER" >/dev/null 2>&1 || true + docker rm -f vb-stripe-mock >/dev/null 2>&1 || true + else + echo "Left running (vBilling, receiver, cluster). KUBECONFIG=$KUBECONFIG, API http://$API (token $TOKEN), work dir $WORK" + fi +} +trap cleanup EXIT + +step "kind control plane cluster with simulated GPU nodes" +cat > "$WORK/kind.yaml" <<'EOF' +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + labels: {nvidia.com/gpu.product: NVIDIA-H100-80GB-HBM3, topology.kubernetes.io/zone: syd-1a} + - role: worker + labels: {nvidia.com/gpu.product: NVIDIA-L40S, vbilling.vcluster.com/capacity-type: spot, topology.kubernetes.io/zone: syd-1b} + - role: worker + labels: {nvidia.com/gpu.product: NVIDIA-H100-80GB-HBM3, node.kubernetes.io/instance-type: bm.gpu.h100.8, vcluster.loft.sh/managed-by: team-b, topology.kubernetes.io/zone: syd-1a} +EOF +kind create cluster --name "$CLUSTER" --config "$WORK/kind.yaml" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null 2>&1 +for n in $CLUSTER-worker $CLUSTER-worker2 $CLUSTER-worker3; do + kubectl patch node "$n" --subresource=status --type=json \ + -p '[{"op":"add","path":"/status/capacity/nvidia.com~1gpu","value":"8"},{"op":"add","path":"/status/allocatable/nvidia.com~1gpu","value":"8"}]' >/dev/null +done +kubectl taint node $CLUSTER-worker3 dedicated=team-b:NoSchedule >/dev/null +kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml >/dev/null +kubectl -n kube-system patch deployment metrics-server --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' >/dev/null + +step "three tenant clusters" +for t in team-a team-b team-c; do vcluster create "$t" -n "$t" --driver helm --connect=false >/dev/null 2>&1 & done +wait +for t in team-a team-b team-c; do kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 "sts/$t" -n "$t" --timeout=300s >/dev/null; done +P=vbilling.vcluster.com +kubectl annotate ns team-a $P/tenant=acme "$P/display-name=Acme AI" $P/tenant-class=public $P/project=training --overwrite >/dev/null +kubectl annotate ns team-c $P/tenant=acme $P/project=inference --overwrite >/dev/null +kubectl annotate ns team-b "$P/display-name=Globex Research" $P/tenant-class=enterprise --overwrite >/dev/null + +gpu_deploy() { # name ns replicas gpus nodeSelector [toleration] + cat < "$WORK/team-a.yaml" +{ echo "apiVersion: v1"; echo "kind: Namespace"; echo "metadata: {name: inference}"; echo "---"; gpu_deploy llm inference 1 1 "nvidia.com/gpu.product: NVIDIA-L40S"; } > "$WORK/team-c.yaml" +gpu_deploy pretrain default 1 8 "vcluster.loft.sh/managed-by: team-b" "{key: dedicated, operator: Equal, value: team-b, effect: NoSchedule}" > "$WORK/team-b.yaml" +for t in team-a team-b team-c; do vcluster connect "$t" -n "$t" --driver helm -- kubectl apply -f "$WORK/$t.yaml" >/dev/null 2>&1; done +for t in team-a team-b team-c; do kubectl wait --for=condition=Ready pods -n "$t" -l vcluster.loft.sh/managed-by="$t" --timeout=180s >/dev/null 2>&1 || true; done + +step "stripe-mock, webhook receiver, vBilling" +docker rm -f vb-stripe-mock >/dev/null 2>&1 || true +docker run -d --name vb-stripe-mock -p 12111:12111 stripe/stripe-mock:latest >/dev/null +cat > "$WORK/receiver.py" < "$WORK/vbilling.log" 2>&1 & PIDS+=($!) +# Wait until a few windows have closed after every workload was running, so +# the window checked below is fully covered (pods started mid-window are, +# correctly, billed only for the seconds they ran). +until [ "$(grep -c 'window .* events' "$WORK/vbilling.log" 2>/dev/null)" -ge 5 ]; do sleep 2; done +sleep 5 + +step "metering" +WIN=$(api "/api/v1/events?metric=vcluster_gpu_hours" | jq -s -r '[.[] | .window_start] | unique | .[-1]') +WEND=$(date -u -j -v+20S -f "%Y-%m-%dT%H:%M:%SZ" "$WIN" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "$WIN + 20 seconds" +%Y-%m-%dT%H:%M:%SZ) +q() { api "/api/v1/usage?from=$WIN&to=$WEND&group_by=tenant,sku,billing_mode&metric=vcluster_gpu_hours" | jq -r "[.rows[] | select($1) | .quantity] | add // 0"; } +check "acme H100: 2 pods x 2 GPUs x 20s = 0.022222222 GPU-hours" '[ "$(q '"'"'.tenant=="acme" and .sku=="NVIDIA-H100-80GB-HBM3"'"'"')" = "0.022222222" ]' +check "acme L40S (spot): 1 GPU x 20s = 0.005555556" '[ "$(q '"'"'.tenant=="acme" and .sku=="NVIDIA-L40S"'"'"')" = "0.005555556" ]' +check "Globex dedicated node: 8 GPUs x 20s = 0.044444444, billing_mode=dedicated_node" '[ "$(q '"'"'.tenant=="vcluster-team-b-team-b" and .billing_mode=="dedicated_node"'"'"')" = "0.044444444" ]' +check "Globex pod on its own dedicated node is not billed again as shared" '[ "$(q '"'"'.tenant=="vcluster-team-b-team-b" and .billing_mode=="shared"'"'"')" = "0" ]' +check "two tenant clusters roll up into one customer" '[ "$(api /api/v1/tenants | jq "[.tenants[] | select(.id==\"acme\") | .clusters | length] | .[0]")" = "2" ]' + +step "delivery and integrity" +sleep 5 +check "stripe and webhook delivering with no lag or dead letters" '[ "$(api /api/v1/destinations | jq "[.destinations[] | select(.ready and .lag_records==0 and .dead_letters==0)] | length")" = "2" ]' +check "ledger hash chain verifies" '[ "$(api /api/v1/ledger/verify | jq .ok)" = "true" ]' +check "every webhook signature valid" '[ "$(jq -s "all(.sig_ok)" "$WORK/received.jsonl")" = "true" ]' +check "no duplicate event IDs delivered" '[ "$(jq -s "[.[] | select(.type|endswith(\"usage.v1\")) | .id] | (length == (unique|length))" "$WORK/received.jsonl")" = "true" ]' + +step "billing-state enforcement" +(cd "$ROOT" && helm template vbilling deploy/helm/vbilling --set enforcement.admissionPolicy.enabled=true --show-only templates/admissionpolicy.yaml) | kubectl apply -f - >/dev/null +CUS=$(jq -r '.["cus/acme"]' "$WORK/data/state-stripe.json") +send() { python3 - "$1" "$2" "$3" "$CUS" <<'EOF' +import hmac, hashlib, json, sys, time, urllib.request +typ, status, evid, cus = sys.argv[1:5] +body = json.dumps({"id": evid, "type": typ, "data": {"object": {"customer": cus, "status": status}}}).encode() +t = str(int(time.time())); sig = hmac.new(b"whsec_e2e", (t + ".").encode() + body, hashlib.sha256).hexdigest() +urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:18080/webhooks/stripe", data=body, headers={"Stripe-Signature": f"t={t},v1={sig}"})) +EOF +} +send customer.subscription.updated unpaid evt_e2e_1 +check "unpaid subscription suspends both of acme's namespaces" '[ "$(kubectl get ns team-a -o jsonpath="{.metadata.labels.vbilling\.vcluster\.com/suspended}")" = "true" ] && [ "$(kubectl get ns team-c -o jsonpath="{.metadata.labels.vbilling\.vcluster\.com/suspended}")" = "true" ]' +check "other tenants are untouched" '[ -z "$(kubectl get ns team-b -o jsonpath="{.metadata.labels.vbilling\.vcluster\.com/suspended}")" ]' +vcluster connect team-a -n team-a --driver helm -- kubectl run probe -n training --image=busybox:1.36 --restart=Never -- sleep 600 >/dev/null 2>&1 +sleep 10 +check "admission policy blocks new pods for the suspended tenant" '[ "$(kubectl get pods -n team-a --no-headers 2>/dev/null | grep -c "^probe-")" = "0" ]' +check "running workloads keep running" '[ "$(kubectl get pods -n team-a --no-headers | grep trainer | grep -c Running)" = "2" ]' +send invoice.paid "" evt_e2e_2 +sleep 20 +check "a paid invoice lifts the suspension and the pod starts" '[ "$(kubectl get pods -n team-a --no-headers | grep "^probe-" | grep -c Running)" = "1" ]' + +echo +echo "$pass passed, $fail failed (logs: $WORK/vbilling.log)" +[ "$fail" -eq 0 ] From b4d8aba2bb7aa12da0fa4d7dbdf223b74902238c Mon Sep 17 00:00:00 2001 From: Saiyam Pathak Date: Mon, 5 Oct 2026 23:42:28 +0530 Subject: [PATCH 3/4] Document v0.2 and add an end-to-end GKE GPU example examples/gke-gpu goes from an empty project to Stripe invoices: a time-sliced L4 tenant cluster, an A100 joined as a private node with the GPU Operator and MIG, DCGM through Prometheus, and vBilling in Stripe test mode. The docs site gets the example page, troubleshooting for private nodes and DCGM, and quick starts that keep keys in Secrets. --- README.md | 644 +++++++++--------- docs/configuration.html | 571 ++++++++++++---- docs/example-gke.html | 418 ++++++++++++ docs/getting-started.html | 42 +- docs/images/dashboard-gke.png | Bin 0 -> 171809 bytes docs/images/dashboard.jpg | Bin 0 -> 63734 bytes docs/index.html | 277 ++++++-- docs/metrics.html | 328 ++++----- docs/private-nodes.html | 388 +++-------- docs/schema/usage-event.v1.json | 46 ++ docs/troubleshooting.html | 68 +- examples/gke-gpu/01-prometheus.yaml | 95 +++ examples/gke-gpu/02-tenant-shared.yaml | 7 + examples/gke-gpu/03-tenant-private.yaml | 46 ++ examples/gke-gpu/04-node-startup.sh | 16 + examples/gke-gpu/05-gpu-operator-values.yaml | 9 + examples/gke-gpu/06-prometheus-agent.yaml | 69 ++ examples/gke-gpu/07-vbilling-values.yaml | 41 ++ examples/gke-gpu/README.md | 161 +++++ .../gke-gpu/workloads/private-training.yaml | 22 + .../gke-gpu/workloads/shared-inference.yaml | 71 ++ 21 files changed, 2262 insertions(+), 1057 deletions(-) create mode 100644 docs/example-gke.html create mode 100644 docs/images/dashboard-gke.png create mode 100644 docs/images/dashboard.jpg create mode 100644 docs/schema/usage-event.v1.json create mode 100644 examples/gke-gpu/01-prometheus.yaml create mode 100644 examples/gke-gpu/02-tenant-shared.yaml create mode 100644 examples/gke-gpu/03-tenant-private.yaml create mode 100644 examples/gke-gpu/04-node-startup.sh create mode 100644 examples/gke-gpu/05-gpu-operator-values.yaml create mode 100644 examples/gke-gpu/06-prometheus-agent.yaml create mode 100644 examples/gke-gpu/07-vbilling-values.yaml create mode 100644 examples/gke-gpu/README.md create mode 100644 examples/gke-gpu/workloads/private-training.yaml create mode 100644 examples/gke-gpu/workloads/shared-inference.yaml diff --git a/README.md b/README.md index 98ed1b0..3c4c1f4 100644 --- a/README.md +++ b/README.md @@ -3,410 +3,442 @@

- Usage-based billing for AI Clouds running vCluster or vMetal
- Auto-discovers Tenant Clusters, meters node capacity and GPU SKUs, and streams usage events to your billing adapter. + Usage metering for AI Clouds running vCluster
+ Meters tenant clusters to the second, keeps a durable, verifiable ledger, and ships reconcilable usage to Stripe, Metronome, Lago or your own pipeline.

- Build - Go Report + Experimental License - Lago adapter - Coming soon + Destinations

--- -Built for AI Clouds and platform teams running Kubernetes with vCluster or vMetal. vBilling is the pipe, not the billing engine. You keep your billing backend (Lago today; Metronome, Stripe Meters, OpenMeter, or a custom adapter coming next). +vBilling is the **metering pipe**, not the billing engine. It turns what runs in your tenant clusters into canonical usage events. Your billing platform does the pricing, invoicing, tax and payments. vBilling makes sure every billable GPU-second reaches it exactly once, attributed to the right customer, region and SKU, and that you can prove it afterwards. -## Architecture - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ Control Plane Cluster │ -│ │ -│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ -│ │ Tenant │ │ Tenant │ │ Tenant │ │ -│ │ Cluster │ │ Cluster │ │ Cluster │ │ -│ │ team-alpha │ │ team-beta │ │ team-gpu │ │ -│ │ │ │ │ │ │ │ -│ │ private · │ │ private · │ │ private · │ │ -│ │ 4× A100 │ │ 2× L40S │ │ 8× H100 │ │ -│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │ -│ │ │ │ │ -│ └──────────────────┼──────────────────┘ │ -│ │ │ -│ ┌───────────▼────────────┐ │ -│ │ vBilling │ │ -│ │ Controller │ │ -│ │ │ │ -│ │ • Auto-discovers │ │ -│ │ • Meters node capacity│ │ -│ │ • Streams events │ │ -│ └───────────┬────────────┘ │ -└─────────────────────────────┼───────────────────────────────────────┘ - │ Usage events (HTTP) - ┌────────────▼────────────┐ - │ Billing Adapter │ - │ │ - │ Lago · Stripe · │ - │ Metronome · Custom │ - │ │ - │ Plans · Subscriptions │ ← Provider configures - │ Invoices · Wallets │ - └─────────────────────────┘ -``` +> vBilling lives in `vClusterLabs-Experiments`: it is an experimental project, not a supported vCluster Labs product. -vBilling handles **metrics collection and event delivery**. Your billing adapter handles **pricing, plans, and invoicing**. Providers configure pricing in the adapter. vBilling never decides what to charge. +## What's new in v0.2 -## Deployment Model +- **Stripe and Metronome adapters.** Use Stripe Billing Meters on their own, or Metronome for rating with Stripe for payment, invoicing and tax. vBilling creates the Stripe customer and links it in Metronome. +- **Durable, hash-chained ledger.** Every window is committed atomically to disk before any backend sees it. If vBilling restarts or a backend has an outage, no usage is lost, and missed windows are backfilled. +- **Several destinations at once.** Each destination has its own cursor, so a down billing API never blocks your data platform feed. +- **Deterministic event IDs.** Retries, replays and two replicas all deduplicate downstream. +- **Second-level allocation metering.** GPU time is metered from container start to finish, including fractional GPUs (MIG mixed and single strategy, GKE GPU partitions, time-slicing including GKE time-sharing) and GPUs allocated through DRA ResourceClaims. +- **Every tenancy model.** Shared nodes and dedicated nodes are metered from the control plane cluster. Tenant clusters with their own nodes (vCluster Private Nodes, Auto Nodes, Standalone) are metered through their own API, with a read-only kubeconfig vCluster exports for vBilling. No node is ever billed twice. +- **Nothing slips between windows.** Pods and nodes deleted between two collections (short jobs, Auto Nodes scale-down) are billed until the second they were deleted. +- **Bill anything Prometheus knows.** Define billable metrics as PromQL (inference tokens from vLLM, any exporter's counter), or read vCluster Platform fleet observability with one preset. +- **Fair metering.** GPU time on NotReady or unhealthy nodes is not billed. It is recorded as auditable downtime instead. +- **Reconciliation API.** It recomputes usage from the ledger and compares it with what Stripe or Metronome recorded. +- **Billing-state enforcement.** Payment failures and spend alerts can label, annotate or soft-suspend tenant clusters. +- **Ingest API.** Accepts usage vBilling cannot see itself: inference tokens, Slurm accounting, storage and network exporters. +- **Built-in dashboard, Prometheus metrics, and CSV/JSONL exports.** +- **An end-to-end example on GKE.** [`examples/gke-gpu`](examples/gke-gpu) goes from an empty project to Stripe invoices: a time-sliced L4 tenant cluster, an A100 joined as a private node with the NVIDIA GPU Operator and MIG, DCGM through your own Prometheus, and a spot preemption billed to the second. -Each tenant gets dedicated bare-metal nodes (GPUs, high-memory, etc.). Billing is based on full node allocation. The entire node is theirs. +## Architecture ``` -team-gpu's dedicated nodes -├── node-1: 8× H100, 96 CPU, 1TB RAM → metered at full node capacity -├── node-2: 8× H100, 96 CPU, 1TB RAM → metered at full node capacity -└── node-3: 8× H100, 96 CPU, 1TB RAM → metered at full node capacity + Control plane cluster (one per region) Destinations + ┌───────────────────────────────────────────────────────┐ + │ tenant clusters: team-a (shared) team-c (spot) │ + │ team-b (dedicated 8x H100) │ + │ pods · PVCs · load balancers · nodes │ ┌─► Stripe Billing Meters + │ │ │ │ + │ ┌─────────────────────▼───────────────────────────┐ │ ├─► Metronome ──► Stripe (payments, tax) + │ │ vBilling │ │ │ + │ │ discover ─► meter window ─► ledger (PVC) ─► fan-out ───┼─► Lago + │ │ ▲ hash-chained │ │ │ + │ │ ingest API ────┘ one cursor per │ │ └─► webhook: CloudEvents to your data platform + │ │ (tokens, Slurm, ...) destination │ │ + │ │ │ │ + │ │ enforcement ◄── /webhooks ◄──────────────────────────── Stripe dunning, Metronome alerts + │ │ label · annotate · admission policy │ │ + │ │ API + dashboard: usage, exports, reconcile │ │ + │ └─────────────────────────────────────────────────┘ │ + └───────────────────────────────────────────────────────┘ ``` -> Pod-level metering for shared-node platforms is present in the code but not the documented path. Primary focus is dedicated-node tenants running on AI Clouds. +1. **Discover** tenant clusters (vCluster StatefulSets and Deployments, enriched from vCluster Platform when available) and group them into billing tenants. +2. **Meter** each closed window (60s by default, aligned to the wall clock) into canonical events. +3. **Commit** the window atomically to the ledger. +4. **Deliver** to every destination at least once, with deterministic IDs so the effect is exactly-once. +5. **Close the loop**: billing webhooks drive the tenant's billing state back onto its clusters. + +## What gets metered + +| Metric | Unit | How | Key dimensions | +|---|---|---|---| +| `vcluster_gpu_hours` | gpu-hours | Container start to finish, per second. MIG at slices/7 (A30: /4) under the single or mixed strategy and on GKE GPU partitions, time-sliced at 1/replicas. Dedicated nodes in whole GPUs | `sku`, `gpu_profile`, `capacity_type`, `billing_mode`, `zone` | +| `vcluster_cpu_core_hours` | core-hours | metrics-server usage, requests, or the larger of both (`CPU_MEMORY_BASIS`) | `capacity_type`, `billing_mode` | +| `vcluster_memory_gb_hours` | gib-hours | As CPU | `capacity_type`, `billing_mode` | +| `vcluster_private_node_hours` | node-hours | Whole nodes dedicated to a tenant cluster, per node | `sku` (SKU label or instance type), `capacity_type` | +| `vcluster_storage_gb_hours` | gib-hours | Bound PVCs synced from the tenant cluster | `sku` = storage class | +| `vcluster_instance_hours` | hours | Control plane hours while ready (not while provisioning or asleep) | `tenant_class` | +| `vcluster_lb_hours` | hours | LoadBalancer services with an assigned address | | +| `vcluster_network_egress_gb` | gib | Prometheus, evaluated at the window end (configurable PromQL) | | +| `vcluster_gpu_utilization` | util-hours | DCGM via Prometheus (`namespace` or `exported_namespace` labels; GPUs in MIG mode by instance engine activity), or your `GPU_UTIL_QUERY`. Informational: never sent to Stripe or Metronome | `gpu_type` | +| `vcluster_gpu_downtime_hours` | gpu-hours | Allocated GPU time on NotReady/unhealthy nodes, and GPUs a dedicated or private node has installed but cannot allocate (a driver restarting after a reboot or a MIG change). Informational: an audit trail of credits | `gpu_type`, `node` | +| custom (`CUSTOM_METRICS`) | yours | Ingest API: inference tokens, Slurm GPU-hours, ... | yours | + +Every event carries `tenant`, `region`, `project`, `sku`, `resource_id`, `tenant_cluster`, the window, and a stable ID. Schema: [`docs/schema/usage-event.v1.json`](docs/schema/usage-event.v1.json). + +```json +{ + "schema_version": "vbilling.usage/v1", + "id": "vb1_d26414a201f1c0e1a4c1b0f0b7e6c2aa", + "tenant": "acme", + "metric": "vcluster_gpu_hours", + "quantity": 0.027222222, + "unit": "gpu-hours", + "window_start": "2026-10-05T09:58:00Z", + "window_end": "2026-10-05T09:58:20Z", + "region": "ap-southeast-2", + "project": "training", + "sku": "NVIDIA-H100-80GB-HBM3", + "resource_id": "vcluster-team-a-team-a", + "dimensions": { + "tenant_cluster": "vcluster-team-a-team-a", "gpu_type": "NVIDIA-H100-80GB-HBM3", + "gpu_profile": "full", "capacity_type": "on-demand", "billing_mode": "shared", + "tenant_class": "public", "zone": "syd-1a" + }, + "properties": {"gpu_devices": 6, "pods": 3, "backfilled": true}, + "source": "collector", + "recorded_at": "2026-10-05T09:59:08Z" +} +``` -## What Gets Metered +*(That event is real output from the end-to-end test. A new 2-GPU pod started 11 seconds into the window while vBilling was down, and the backfill billed exactly its 9 seconds.)* -| Metric | Source | Granularity | -|-------------------------|-----------------------|--------------------| -| Node hours | Node watch | Per dedicated node | -| CPU core-hours | Node capacity | Full node capacity | -| Memory GB-hours | Node capacity | Full node capacity | -| GPU hours (by SKU) | Node labels | Per GPU SKU | -| GPU utilization | DCGM via Prometheus | Per GPU % | -| Storage GB-hours | PVC sizes | Per PVC | -| Network egress GB | CNI / Prometheus | Per tenant | -| LoadBalancer hours | Service count | Per LB service | -| Control plane hours | Tenant Cluster watch | 1 per cluster | +### Metering rules that protect tenants -**GPU SKU detection** reads from node labels: -- `nvidia.com/gpu.product` (NVIDIA GPU Operator) -- `cloud.google.com/gke-accelerator` (GKE) -- `k8s.amazonaws.com/accelerator` (EKS) +- **Billing starts when a container starts**, not when the pod is created, so image pulls and scheduling are free. It stops when the last container finishes. +- **Hardware downtime is not billed.** GPU time on a node that is NotReady, or carries a taint listed in `UNHEALTHY_NODE_TAINTS`, becomes `vcluster_gpu_downtime_hours` instead of `vcluster_gpu_hours`. So does a node billed whole whose GPUs are installed (GPU Feature Discovery's `nvidia.com/gpu.count`) but not allocatable, for example while the driver rebuilds after a reboot. +- **Provisioning and sleep are free.** Control plane hours accrue only while the tenant cluster has a ready replica. +- **No double billing.** Pods on a tenant's own dedicated nodes are covered by the node allocation. The tenant cluster's own control plane pods and data PVCs are covered by instance hours (set `METER_CONTROL_PLANE=true` to bill them as well). +- **No invented usage.** Backfilled windows include only what is knowable from object lifetimes (allocations, requests). Sampled usage (metrics-server CPU, egress) is never extrapolated. +- **Deleted is not free, and not overbilled.** A pod or node deleted between collections is billed until it was deleted. A deletion noticed only after a watch reconnects is dated to the last collection that still saw it, so an outage can only under-bill. -An H100 hour and a T4 hour are emitted as separate events so providers can price each SKU differently in their billing adapter. +## Tenancy models -## Quick Start +| Tenancy model | Where workloads run | How vBilling meters it | +|---|---|---| +| Shared nodes | Control plane cluster nodes, pods synced by vCluster | From the control plane cluster: pods, volumes, load balancers (`billing_mode=shared`) | +| Dedicated nodes | Control plane cluster nodes reserved for one tenant cluster | Whole node: node-hours, whole GPUs, CPU and memory capacity (`billing_mode=dedicated_node`) | +| Private Nodes, Auto Nodes | Machines joined to the tenant cluster itself; nothing is synced to the control plane cluster | Through the tenant cluster's own API: each private node billed whole (`billing_mode=private_node`), plus the tenant cluster's volumes and load balancers. `PRIVATE_NODE_BILLING=usage` bills the pods on them instead | +| Standalone, or any cluster elsewhere | No control plane cluster | Through a kubeconfig listed in `TENANT_CLUSTERS_FILE`, like private nodes | -vBilling ships with a **Lago adapter** today. Metronome, Stripe Meters, OpenMeter, and custom adapters are on the roadmap. The install pattern is the same regardless of adapter. The walkthrough below uses Lago. +A node the tenant cluster's API reports is billed as a private node only if the control plane cluster does not have it and it is not one of vCluster's placeholder nodes (`vcluster.loft.sh/fake-node`). Shared tenant clusters are therefore never billed twice, even with tenant API access configured. Label a node `vbilling.vcluster.com/billable=false` to exempt hardware the tenant owns. -### Prerequisites +**Least-privilege access.** vBilling reads one Secret per tenant cluster, `vbilling-kubeconfig` in its namespace, and its RBAC is pinned to that Secret name. vCluster writes it with a read-only service account: -- Kubernetes cluster with Tenant Clusters running (vCluster) -- [metrics-server](https://github.com/kubernetes-sigs/metrics-server) installed -- A billing adapter: Lago instance (see [Deploying Lago](#deploying-lago)) -- Optional: Prometheus with [DCGM Exporter](https://github.com/NVIDIA/dcgm-exporter) for GPU utilization +```yaml +# vcluster.yaml (vCluster 0.36+) +exportKubeConfig: + additionalSecrets: + - name: vbilling-kubeconfig + serviceAccount: + name: vbilling-reader + namespace: kube-system + clusterRole: vbilling-reader +experimental: + deploy: + vcluster: + manifests: |- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vbilling-reader + rules: + - apiGroups: [""] + resources: [nodes, pods, persistentvolumeclaims, services] + verbs: [get, list, watch] + - apiGroups: [resource.k8s.io] + resources: [resourceclaims, resourceslices] + verbs: [get, list] + - apiGroups: [metrics.k8s.io] + resources: [pods] + verbs: [get, list] +``` -### 1. Choose & deploy your billing adapter (Lago) +vCluster exports kubeconfigs for `localhost`; vBilling connects to the tenant cluster's Service instead and verifies TLS against `.`. `TENANT_ADMIN_FALLBACK=true` also accepts the admin kubeconfig (`vc-`), which requires giving vBilling read access to Secrets. -```bash -# Clone the vBilling repo (includes Lago docker-compose) -git clone https://github.com/vClusterLabs-Experiments/vbilling.git -cd vbilling/deploy/lago +**Outages.** If a tenant cluster's API cannot be read, the rest of the fleet is still metered, and the missed windows are filled as soon as it answers (up to `MAX_BACKFILL`). Fills are late ledger events deduplicated by ID, so a repeated fill never bills twice. -# Generate RSA key for Lago (required for JWT signing) -openssl genrsa 2048 > lago_rsa.key -openssl rsa -in lago_rsa.key -out lago_rsa.key -traditional 2>/dev/null +## Billing backends -# Create .env with Base64-encoded RSA key (Lago expects LAGO_RSA_PRIVATE_KEY) -echo "LAGO_RSA_PRIVATE_KEY=$(base64 -i lago_rsa.key | tr -d '\n')" > .env +Select one or more with `ADAPTERS` (Helm: `adapters`). -# Start Lago -docker compose --env-file .env up -d +| Adapter | What vBilling does | Pricing lives in | +|---|---|---| +| `stripe` | Creates [Billing Meters](https://docs.stripe.com/billing/subscriptions/usage-based) and posts meter events. Meters are split per SKU (`STRIPE_SPLIT_METERS_BY`) because Stripe meters cannot price on dimensions. Customers are created idempotently and never duplicated. Optionally subscribes tenants to prices tagged `metadata[vbilling_plan]`. Reconciles via meter event summaries. | Stripe prices on each meter | +| `metronome` | Creates customers with the tenant ID as ingest alias. Creates SUM billable metrics with one compound group key (`region, sku, capacity_type, billing_mode, tenant_cluster`). Ingests events and optionally creates a contract per tenant from a rate card. Reconciles via `/v1/usage`. | Metronome rate cards (pricing group keys) | +| `metronome` + `METRONOME_STRIPE_LINK=true` | Also creates the tenant's Stripe customer and links it in Metronome, so **Metronome rates and Stripe collects payment, invoices and tax**. | Metronome | +| `lago` | Customers, subscriptions and the default plan, plus events whose dimensions are properties for Lago charge filters. | Lago plans and charges | +| `webhook` | Signed CloudEvents 1.0 batches (`application/cloudevents-batch+json`), including tenant and catalog events. Use it to feed a data platform, Kafka bridge or homegrown rating engine. | wherever you send it | +| `noop` | Logs events (dry run). | n/a | -# Wait for API to be ready (~30s for database migrations) -# UI: http://localhost:8080 | API: http://localhost:3000 -``` +Stripe now steers new usage-based integrations toward Metronome, which Stripe acquired in January 2026. Both paths are supported here: Billing Meters for straightforward pay-as-you-go, Metronome for dimensional pricing, commits, credits and multi-currency rate cards. -Create an organization in Lago (first-time only): -```bash -curl -s -X POST http://localhost:3000/graphql \ - -H "Content-Type: application/json" \ - -d '{"query":"mutation { registerUser(input: { email: \"admin@example.com\", password: \"yourpassword\", organizationName: \"My Org\" }) { token } }"}' +## Quick start -# Get your API key -docker exec lago-db-1 psql -U lago -d lago -t -c "SELECT value FROM api_keys LIMIT 1;" -``` +Images for `linux/amd64` and `linux/arm64` are published to `ghcr.io/vclusterlabs-experiments/vbilling`, the chart's default. Keep billing keys in Secrets, never in `--set` flags or values files: -### 2. Install vBilling - -**Option A: Helm (production)** ```bash -# Build and push the image first -docker buildx build --platform linux/amd64,linux/arm64 \ - -t /vbilling:v0.1.0 --push . - -# Install via Helm, point vBilling at your adapter -helm upgrade --install vbilling deploy/helm/vbilling \ - --namespace vbilling-system --create-namespace \ - --set image.repository=/vbilling \ - --set image.tag=v0.1.0 \ - --set adapter=lago \ - --set lago.apiURL=http://lago-api.lago-system:3000 \ - --set lago.apiKey=YOUR_LAGO_API_KEY +git clone https://github.com/vClusterLabs-Experiments/vbilling.git && cd vbilling +kubectl create namespace vbilling-system ``` -**Option B: Run locally (development/testing)** +**Stripe (test mode)** + ```bash -make build -LAGO_API_KEY= LAGO_API_URL=http://localhost:3000 ./bin/vbilling +kubectl -n vbilling-system create secret generic vbilling-stripe --from-file=api-key=./stripe-test-key +helm upgrade --install vbilling deploy/helm/vbilling -n vbilling-system \ + --set adapters='{stripe}' --set stripe.existingSecret=vbilling-stripe \ + --set region=ap-southeast-2 --set clusterName=syd-1 ``` -### 3. Configure pricing in your adapter - -vBilling creates billable metrics and a skeleton plan with **$0 pricing**. You set your own prices in the adapter's UI or API: - -1. Open Lago UI → **Plans** → **vCluster Standard** -2. Edit each charge with your pricing: - - CPU Core-Hours: `$0.065` (your cost + margin) - - Memory GB-Hours: `$0.009` - - GPU Hours (H100): `$4.50`, or use Lago's graduated pricing for volume discounts - - Storage GB-Hours: `$0.0002` - - Network Egress GB: `$0.09` - - Node Hours: `$25.00` (for dedicated node billing) -3. Save. Pricing takes effect immediately for all tenants - -You can also create **multiple plans** (e.g., "GPU Premium", "Dev Tier") and assign different plans to different customers via the adapter's API. +vBilling creates one meter per metric (and per SKU as it sees them). Attach metered prices to the meters you want to charge for. To have vBilling subscribe each new tenant automatically, tag those prices with `metadata[vbilling_plan]=vcluster-standard` and set `stripe.autoSubscribe=true`. -### 4. Done +**Metronome rating, Stripe payments** -vBilling will: -- Auto-discover all Tenant Clusters (via StatefulSet labels or Platform API) -- Create a billing customer in your adapter for each Tenant Cluster -- Stream usage events every 60 seconds -- Your adapter generates invoices at the end of each billing period - -## How It Works - -### Discovery +```bash +kubectl -n vbilling-system create secret generic vbilling-metronome --from-file=api-token=./metronome-token +kubectl -n vbilling-system create secret generic vbilling-stripe --from-file=api-key=./stripe-live-key +kubectl -n vbilling-system create secret generic vbilling-webhook --from-file=secret=./webhook-secret +helm upgrade --install vbilling deploy/helm/vbilling -n vbilling-system \ + --set adapters='{metronome,webhook}' \ + --set metronome.existingSecret=vbilling-metronome --set metronome.rateCard=payg-aud \ + --set metronome.stripeLink=true --set stripe.existingSecret=vbilling-stripe \ + --set webhook.url=https://data.example.com/usage --set webhook.existingSecret=vbilling-webhook \ + --set region=ap-southeast-2 +``` -vBilling finds Tenant Clusters using two methods: +In Metronome, add products on the vBilling billable metrics with `pricing_group_key: ["region","sku","capacity_type"]` to price H100 differently from L40S, Sydney differently from Melbourne, and spot or preemptible at a flat discount. Rate cards carry the currency (AUD and NZD are supported natively). -1. **Label scanning** (works with OSS vCluster): Watches StatefulSets and Deployments with `app=vcluster` label -2. **Platform API** (works with vCluster Platform): Lists `VirtualClusterInstance` resources via the management API +**Shared and private GPU nodes on GKE**: [`examples/gke-gpu`](examples/gke-gpu) is a complete setup, from an empty project to Stripe invoices: a time-sliced L4 tenant cluster, an A100 joined as a vCluster private node, DCGM through Prometheus, and the manifests, values and commands for each step. Also on the [docs site](https://vclusterlabs-experiments.github.io/vbilling/example-gke.html). -### Metrics Collection Loop +**Lago (local demo)**: `deploy/lago/docker-compose.yml` runs Lago locally. See [docs/getting-started.html](docs/getting-started.html). -Every collection interval (default 60s): +Then: +```bash +kubectl -n vbilling-system port-forward svc/vbilling 8080 +# dashboard: http://localhost:8080/ +curl -s localhost:8080/api/v1/status | jq '.destinations' # delivery health +curl -s 'localhost:8080/api/v1/usage?group_by=tenant,sku' | jq # line items ``` -For each discovered Tenant Cluster: - 1. Read dedicated node capacity (labels: vcluster.loft.sh/managed-by=) - → Read full node capacity: CPU, memory, GPUs, storage - 2. Collect storage from PVCs in the namespace +![vBilling dashboard](docs/images/dashboard.jpg) - 3. Collect GPU allocation from pod nvidia.com/gpu requests - → Detect GPU SKU from the node's nvidia.com/gpu.product label +## Tenants, projects and metadata - 4. Count LoadBalancer services +By default every tenant cluster is its own billing customer, with ID `vcluster--` (the v0.1 format). Set labels or annotations on the tenant cluster's StatefulSet, its namespace, or its vCluster Platform `VirtualClusterInstance` (later sources win): - 5. Check spot vs on-demand node status for cost attribution +| Key (`vbilling.vcluster.com/...`) | Effect | +|---|---| +| `tenant` | Billing customer ID. Several tenant clusters can share one customer. | +| `display-name`, `email`, `currency` | Customer details sent to the backend | +| `project` | Cost attribution (defaults to the vCluster Platform project) | +| `tenant-class` | e.g. `public`, `enterprise`, `government`, `dev`. Stamped on events for pricing. | +| `plan` | Lago plan, Stripe plan tag, or Metronome rate card alias/ID | +| `stripe-customer-id`, `metronome-customer-id` | Pin an existing customer instead of creating one | +| `exclude` | `"true"` stops metering this tenant cluster | - 6. (If Prometheus configured) Query DCGM for GPU utilization - 7. (If Prometheus configured) Query network egress bytes +`TENANT_SOURCE=project` maps every tenant cluster in a vCluster Platform project to one customer (`project-`). `METER_BY_NAMESPACE=true` adds the namespace inside the tenant cluster as a dimension, so tenants can run their own internal chargeback. - 8. Convert all metrics to billing units: - CPU: cores × interval_hours = core-hours - Memory: GB × interval_hours = GB-hours - GPU: count × interval_hours = GPU-hours (tagged with GPU SKU) +On nodes, `vbilling.vcluster.com/capacity-type` (`on-demand`, `spot`, `preemptible`, `reserved`) and an optional SKU label (`SKU_LABEL`) take precedence over cloud-provider labels. Nodes become **dedicated** to a tenant cluster with `vbilling.vcluster.com/tenant-cluster=`, `vcluster.loft.sh/managed-by=`, or the `VCLUSTER_NODE_LABEL` pattern. - 9. Stream all events to the configured billing adapter in batch -``` +## Durability and correctness -### Billing Flow +| Guarantee | How | +|---|---| +| No usage lost on restart, upgrade or backend outage | Windows are committed to an fsynced, append-only ledger on a PVC before delivery. Each destination has a persisted cursor. On restart, missed windows (up to `MAX_BACKFILL`, default 6h) are backfilled from object lifetimes. | +| No duplicates | Event IDs are a hash of the event's identity (tenant, metric, window, dimensions), never of its quantity. Stripe dedupes on `identifier`, Metronome on `transaction_id` (34 days), Lago on `transaction_id`. A window can never be committed twice. | +| Half-written windows never escape | A window is a single write plus fsync ending in a commit record. A torn or uncommitted tail is truncated on startup and collected again. | +| Tamper evidence | Every ledger record is SHA-256 chained to the previous one. `GET /api/v1/ledger/verify` re-hashes the whole retained ledger. | +| Bad events don't block good ones | Payloads a backend rejects are isolated (partial-batch errors, or bisection) and parked as dead letters. Auth errors, rate limits and outages are retried with backoff; they are never dead-lettered. | +| Provable totals | `GET /api/v1/reconcile` recomputes each tenant's usage from the ledger and compares it with what Stripe or Metronome recorded. Dead letters are accounted for. | +| Tenant API outages do not lose usage | A tenant cluster whose own API is unreachable is recorded as a gap (persisted under `DATA_DIR`). Once it answers, exactly the missed windows are metered and appended as late events, deduplicated by ID. | +| A wrong customer is never billed | Customer search results must carry the tenant's exact metadata. Discovery errors never offboard tenants. A tenant must be gone for `OFFBOARD_GRACE` before `RemoveTenant` runs, and customers are never deleted. | -``` -Tenant Cluster created → Customer auto-created in adapter - → Subscription started (plan: vcluster-standard) - → Usage events every 60s - → Adapter aggregates over billing period - → Invoice generated (monthly) - → Webhook to payment provider (optional) - -Tenant Cluster deleted → Subscription terminated - → Final prorated invoice -``` +Running two replicas (each with its own PVC) is safe for delivery: both compute the same deterministic IDs, so backends keep the first copy. -## Configuration +## Regions and data sovereignty -### Environment Variables +Run one vBilling per control plane cluster and set `region` (for example `ap-southeast-2`, `syd-1`, `mel-1`). Every event, ledger record and billing state stays in that region's cluster. `GET /api/v1/status` reports the region, and each destination is a separately configured, documented data flow. Stripe and Metronome are hosted outside many jurisdictions (Metronome's public docs describe no regional hosting), so a sovereign region can choose to: -| Variable | Default | Description | -|----------|---------|-------------| -| `ADAPTER` | `lago` | Billing adapter to use (`lago` today; more adapters coming) | -| `LAGO_API_URL` | `http://localhost:3000` | Lago API endpoint (when `ADAPTER=lago`) | -| `LAGO_API_KEY` | (required for Lago) | Lago API key | -| `COLLECTION_INTERVAL` | `60s` | How often to scrape metrics | -| `RECONCILE_INTERVAL` | `30s` | How often to discover Tenant Clusters | -| `DEFAULT_PLAN_CODE` | `vcluster-standard` | Default plan code in the adapter | -| `BILLING_CURRENCY` | `USD` | Currency for billing | -| `PROMETHEUS_URL` | (empty) | Prometheus URL for DCGM/network | -| `SPOT_DISCOUNT_PERCENT` | `60` | Discount for pods on spot nodes | +- keep the full-fidelity ledger and the webhook feed in-region and send only billing quantities offshore +- pin customer IDs and keep names and emails out of usage events (they never contain PII; tenant IDs are yours to choose) +- run Lago or its own pipeline in-region instead -**Note:** Pricing is NOT configured via environment variables. Configure pricing in your billing adapter's UI or API. +## Billing-state enforcement -### Helm Values +Point Stripe webhooks at `/webhooks/stripe` (`STRIPE_WEBHOOK_SECRET`) and Metronome notifications at `/webhooks/metronome` (`METRONOME_WEBHOOK_SECRET`). Signatures are verified: Stripe's `t=…,v1=…` scheme, and Metronome's HMAC over `X-Metronome-Date` and the body. -```yaml -adapter: lago # billing adapter (lago today) +| Event | State | +|---|---| +| `invoice.payment_failed`, subscription `past_due` | `delinquent` | +| subscription `unpaid` / `canceled` / deleted | `suspended` | +| `billing.alert.triggered`, Metronome `alerts.*` (spend, usage, low credit or commit balance) | `warning` | +| `invoice.paid`, subscription `active` | `active` | -lago: - apiURL: "http://lago-api:3000" - apiKey: "" - existingSecret: "lago-credentials" # or use existing K8s secret +Override any mapping with `ENFORCEMENT_RULES`, for example `alerts.spend_threshold_reached=suspended` for a hard spend cap. -billing: - collectionInterval: "60s" - reconcileInterval: "30s" +`ENFORCEMENT_MODE` controls what happens: +- **`observe`** (default): record the state only. It appears in the API, metrics and dashboard. +- **`annotate`**: also annotate the tenant cluster, label its namespace `vbilling.vcluster.com/billing-state`, and emit a Kubernetes event. +- **`enforce`**: also label suspended namespaces `vbilling.vcluster.com/suspended=true`. With `enforcement.admissionPolicy.enabled=true` (Kubernetes 1.30+), a ValidatingAdmissionPolicy then denies **new** pods in those namespaces. Running workloads keep running, the tenant sees the reason as a sync error inside their tenant cluster, and a paid invoice lifts it automatically. -prometheus: - url: "http://prometheus.monitoring:9090" # optional -``` +Metronome does not emit webhooks for Stripe payment failures, so in Metronome + Stripe mode, send Stripe's webhooks to vBilling as well. -## Use Case: AI Cloud +## Custom usage: tokens, Slurm, storage, network -Each customer gets a Tenant Cluster with dedicated bare-metal GPU nodes. vBilling meters the full node allocation by GPU SKU and streams events into whichever billing adapter you run. +Declare metrics with `CUSTOM_METRICS="code:unit[:key|key],..."` and push closed windows with a bearer token: -``` -Customer signs up - → Platform provisions Tenant Cluster + dedicated nodes (8× H100) - → vBilling discovers Tenant Cluster, detects the dedicated nodes - → Streams events: 8 GPU-hours (H100) + 96 CPU-hours + 1 TB memory-hours per hour - → Adapter (Lago) invoices monthly at the provider's rates - → Customer pays via the adapter's payments integration (e.g., Stripe webhooks) +```bash +curl -X POST localhost:8080/api/v1/events -H "Authorization: Bearer $INGEST_TOKEN" -H "X-VBilling-Client: model-gateway" -d '{ + "events": [{"tenant": "acme", "metric": "inference_output_tokens", "quantity": 1250000, + "window_start": "2026-10-05T10:00:00Z", "window_end": "2026-10-05T10:01:00Z", + "dimensions": {"model": "llama-3.3-70b"}}]}' ``` -## Dashboard +Events are validated, stamped with the region, deduplicated (send your own `id`, or let vBilling derive one from the identity), committed to the ledger, and fanned out like collector events. Billing backends create meters and metrics for custom metrics at bootstrap. -A lightweight billing dashboard is included at `dashboard/index.html`. It queries the Lago API directly and shows per-tenant usage breakdowns. (Adapter-specific dashboards for Metronome and Stripe will ship alongside those adapters.) +### Metrics from Prometheus -```bash -# Serve the dashboard -cd dashboard -python3 -m http.server 9090 -# Open http://localhost:9090 +`PROMETHEUS_METRICS_FILE` (Helm: `prometheus.metrics`) defines metrics as PromQL, evaluated for every tenant cluster at each window end. Result labels listed in `dimensions` become event dimensions and pricing group keys: + +```yaml +- code: inference_output_tokens + name: Inference output tokens + unit: tokens + billable: true + dimensions: [model_name] + sku: model_name + query: sum by (model_name) (increase(vllm:generation_tokens_total{namespace="{{namespace}}"}[{{window}}])) ``` -Features: -- Per-tenant usage cards with metric breakdown -- Total spend across all tenants -- Auto-refresh every 30 seconds -- No framework dependencies +Templates: `{{namespace}}`, `{{vcluster}}`, `{{instance}}` (vCluster Platform instance), `{{project}}`, `{{tenant}}`, `{{tenant_cluster}}`, `{{window}}` (e.g. `60s`), `{{window_seconds}}` (e.g. `60`). -## Project Structure +Billing on DCGM fields takes two precautions. The exporter refreshes values every 30 seconds by default (`DCGM_EXPORTER_INTERVAL`), so `increase()` over a window that short can read zero: integrate a gauge instead. And device-level fields (power, energy, framebuffer) repeat on every MIG instance of a GPU, so keep one series per GPU (`UUID`) before summing. GPU energy per kWh, for GPUs a tenant has to itself: -``` -cmd/vbilling/main.go Entry point; blank-imports built-in adapters -internal/ - config/config.go Configuration from env vars - destinations/ - destination.go Destination interface, Tenant/UsageEvent types, - registry, canonical metric codes - lago/ - adapter.go Lago adapter (default) - client.go Lago HTTP API client - bootstrap.go Creates metrics + skeleton plan in Lago - noop/ - adapter.go Logs events; useful for dry-run / testing - discovery/discovery.go Tenant Cluster discovery (labels + Platform API) - metrics/collector.go All metrics: CPU, memory, GPU, storage, - network, DCGM, dedicated nodes, spot/on-demand - controller/controller.go Reconciliation + event-streaming loop; - produces canonical UsageEvent values -deploy/ - helm/vbilling/ Helm chart with RBAC - lago/ Docker Compose for Lago (demo only) -dashboard/index.html Billing dashboard -scripts/demo.sh End-to-end demo using vind -Dockerfile Multi-stage distroless build -Makefile Build targets +```yaml +- code: gpu_energy_kwh + name: GPU energy + unit: kWh + billable: true + dimensions: [modelName] + sku: modelName + query: sum by (modelName) (max by (modelName, UUID, gpu, Hostname, hostname) (avg_over_time(DCGM_FI_DEV_POWER_USAGE{vcluster="{{vcluster}}"}[{{window}}]))) * {{window_seconds}} / 3.6e6 ``` -### Adapter pattern +Tenant clusters with private nodes run DCGM inside the tenant cluster, so the control plane cluster's Prometheus never scrapes it. Ship it with a Prometheus agent (or OpenTelemetry collector) that remote-writes to your Prometheus with an external label naming the tenant cluster, as `vcluster` does above, and select on that label in `GPU_UTIL_QUERY`, `GPU_COUNT_QUERY` and your metrics. With vCluster Platform fleet observability, the preset below does this for you. -Adapters live under `internal/destinations//` and implement a small Go interface: +### vCluster Platform fleet observability -```go -type Destination interface { - Name() string - Bootstrap(ctx context.Context) error - EnsureTenant(ctx context.Context, t Tenant) error - RemoveTenant(ctx context.Context, externalID string) error - SendEvents(ctx context.Context, events []UsageEvent) error -} -``` +vCluster Platform 4.11+ collects metrics from every tenant cluster, private nodes included, with an OpenTelemetry collector that pushes through the Platform's write gateway. The gateway stamps each series with `vcluster_platform_instance` and `vcluster_platform_project` from the collector's access key, after removing any such labels a tenant sent. `PROMETHEUS_PRESET=vcluster-platform` points egress and DCGM GPU utilization at those labels; set `PROMETHEUS_URL` to the fleet Prometheus (or the query proxy, with a metrics access key in `PROMETHEUS_HEADERS`). -Each adapter package registers itself from `init()`: +## HTTP API -```go -destinations.Register("lago", func(cfg *config.Config) (destinations.Destination, error) { - return &Adapter{ /* ... */ }, nil -}) -``` +| Endpoint | Purpose | +|---|---| +| `GET /` | Dashboard (embedded; no external assets, works air-gapped) | +| `GET /healthz`, `GET /readyz`, `GET /metrics` | Probes and Prometheus metrics | +| `GET /api/v1/status` | Region, destinations (cursor, lag, failures, dead letters), ledger stats | +| `GET /api/v1/usage?from&to&group_by=tenant,sku,region,day&format=csv` | Unrated line items (filters: `tenant`, `metric`, `project`, `region`, `sku`, `cluster`) | +| `GET /api/v1/events?from&to&format=jsonl\|csv` | Raw event export | +| `POST /api/v1/events` | Ingest custom usage (`INGEST_TOKEN`) | +| `GET /api/v1/reconcile?from&to&destination` | Ledger vs. backend totals (default: previous UTC day) | +| `GET /api/v1/ledger/verify` | Hash-chain verification | +| `GET /api/v1/destinations`, `GET …/{name}/dead-letters`, `POST …/{name}/dead-letters/replay`, `POST …/{name}/cursor` | Delivery operations | +| `GET /api/v1/tenants`, `GET /api/v1/billing-states`, `PUT /api/v1/billing-states/{tenant}` | Tenants and billing state | +| `POST /webhooks/stripe`, `POST /webhooks/metronome` | Billing signals | -`cmd/vbilling/main.go` blank-imports the adapters it ships with so the `init()` functions run, and `destinations.New(cfg.Adapter, cfg)` returns the one selected via `ADAPTER`. To add a new backend (Metronome, Stripe Meters, OpenMeter, custom), drop a new package under `internal/destinations/`, register a factory, blank-import it from `main.go`, and rebuild. +`API_TOKEN` protects everything under `/api` (bearer). Keep the Service internal, or put it behind your portal. -Built-in adapters today: **`lago`** (default), **`noop`** (dry-run / testing). +## Observability -## Building +Notable metrics on `/metrics`: +- **Delivery:** `vbilling_events_delivered_total{destination}`, `vbilling_delivery_failures_total{destination}`, `vbilling_events_dead_lettered_total{destination}`, `vbilling_destination_lag_records`, `vbilling_destination_lag_seconds`, `vbilling_destination_ready` +- **Metering:** `vbilling_events_metered_total`, `vbilling_backfilled_windows_total`, `vbilling_last_window_end_timestamp_seconds`, `vbilling_nodes_down` +- **Tenants:** `vbilling_tenant_billing_state{tenant,state}` +- **Ledger:** `vbilling_ledger_bytes` -```bash -make build # Build binary (local OS/arch) -make docker-build # Build Docker image (local arch) -make test # Run tests -make helm-install # Install via Helm -make tidy # go mod tidy -``` +Alert on rising lag, dead letters, or a stale last window. -### Multi-Arch Docker Image +## Configuration -For production K8s clusters (linux/amd64) and Apple Silicon (linux/arm64): +| Variable (Helm value) | Default | | +|---|---|---| +| `ADAPTERS` (`adapters`) | `lago` | Comma-separated destinations. `ADAPTER` (v0.1) still works. | +| `REGION` (`region`), `CLUSTER_NAME` (`clusterName`) | `default`, `control-plane` | Stamped on every event | +| `DATA_DIR` | `/var/lib/vbilling` | Ledger and adapter state (PVC in the chart) | +| `RETENTION` (`persistence.retention`) | `168h` | Ledger retention once every destination has consumed it | +| `COLLECTION_INTERVAL` | `60s` | Window size; must divide an hour | +| `MAX_BACKFILL`, `OFFBOARD_GRACE` | `6h`, `1h` | | +| `CPU_MEMORY_BASIS` | `usage` | `usage`, `requests` or `max` | +| `METER_CONTROL_PLANE`, `METER_BY_NAMESPACE` | `false` | | +| `GPU_RESOURCES` | `nvidia.com/gpu,amd.com/gpu` | MIG (`nvidia.com/mig-*`) and `nvidia.com/gpu.shared` are always detected | +| `SKU_LABEL`, `CAPACITY_TYPE_LABEL` | `""`, `vbilling.vcluster.com/capacity-type` | | +| `UNHEALTHY_NODE_TAINTS` | `node.kubernetes.io/not-ready,node.kubernetes.io/unreachable` | Add your GPU health taints | +| `TENANT_SOURCE`, `DEFAULT_TENANT_CLASS`, `PLATFORM_CLUSTER` | `cluster` | | +| `PROMETHEUS_URL`, `EGRESS_QUERY` | | Any Prometheus HTTP API (Prometheus, Thanos, Mimir, VictoriaMetrics). Egress and GPU utilization. Templates use `{{namespace}}`, `{{vcluster}}`, `{{window}}` | +| `GPU_UTIL_QUERY`, `GPU_COUNT_QUERY` | DCGM | Replace the utilization queries (same templates, results grouped by `modelName` or `gpu_type`); `none` disables utilization | +| `PROMETHEUS_HEADERS`, `PROMETHEUS_BEARER_TOKEN_FILE` | | Query API auth: `Name=value,...` headers (e.g. `X-Scope-OrgID`, `Authorization`) and a token file re-read per query. Basic auth also works in the URL | +| `PROMETHEUS_PRESET` | | `vcluster-platform`: vCluster Platform fleet observability label selectors | +| `PROMETHEUS_METRICS_FILE` | | Metrics defined by PromQL (see Metrics from Prometheus) | +| `TENANT_API`, `TENANT_KUBECONFIG_SECRET`, `TENANT_ADMIN_FALLBACK` | `true`, `vbilling-kubeconfig`, `false` | Meter tenant clusters with their own nodes through their own API | +| `TENANT_CLUSTERS_FILE` | | YAML list of external tenant clusters (`name`, `kubeconfig`, `tenant`, `displayName`, `project`, `metadata`) | +| `PRIVATE_NODE_BILLING`, `TENANT_EXCLUDE_NAMESPACES` | `node`, `kube-system` | `node` bills private nodes whole, `usage` bills the pods on them | +| `WATCH_DELETIONS`, `DRA_GPU_DRIVERS` | `true`, `gpu.nvidia.com,gpu.amd.com` | Bill pods and nodes deleted between windows; DRA drivers whose devices are GPUs | +| `CUSTOM_METRICS` | | e.g. `inference_output_tokens:tokens:model\|region` | +| `API_TOKEN`, `INGEST_TOKEN`, `LISTEN_ADDR` | `:8080` | | +| `ENFORCEMENT_MODE`, `ENFORCEMENT_RULES` | `observe` | | +| `STRIPE_API_KEY`, `STRIPE_WEBHOOK_SECRET`, `STRIPE_SPLIT_METERS_BY` (`sku`), `STRIPE_AUTO_SUBSCRIBE`, `STRIPE_MAX_RPS`, `STRIPE_API_VERSION` (`2026-09-30.endive`), `STRIPE_API_BASE` | | | +| `METRONOME_API_TOKEN`, `METRONOME_WEBHOOK_SECRET`, `METRONOME_RATE_CARD`, `METRONOME_STRIPE_LINK`, `METRONOME_STRIPE_COLLECTION_METHOD` | | | +| `LAGO_API_URL`, `LAGO_API_KEY`, `DEFAULT_PLAN_CODE`, `BILLING_CURRENCY` | | | +| `WEBHOOK_URL`, `WEBHOOK_SECRET`, `WEBHOOK_HEADERS` | | | + +## Upgrading from v0.1 + +- **Workload:** the chart now deploys a **StatefulSet** with a PVC (the ledger). The unused `pricing.*` values were removed. +- **Discounts:** quantities are no longer discounted for spot nodes (`SPOT_DISCOUNT_PERCENT` is ignored). Price `capacity_type=spot|preemptible` in your backend instead, so metered quantities stay physical and reconcilable. +- **Double counting fixed:** v0.1 billed GPUs and CPU/memory on dedicated nodes twice (pod usage plus node capacity). It also over-counted egress about 5x with 60s windows. +- **Timestamps:** events use the window start, and identical windows now produce identical IDs. +- **Dashboard:** it is served by vBilling itself and reads its API (the Lago-only `dashboard/index.html` remains for reference). + +## Development ```bash -# Build and push multi-arch image -docker buildx create --use --name vbilling-builder 2>/dev/null || true -docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -t /vbilling:v0.1.0 \ - --push . +make test # unit tests (race detector) +make build ``` -The Dockerfile uses multi-stage build with `gcr.io/distroless/static:nonroot` as the final image (~10MB). - -## Deploying Lago - -### Docker Compose (Development/Demo) +The Stripe adapter is also tested against [stripe-mock](https://github.com/stripe/stripe-mock), which validates every request against Stripe's OpenAPI spec. Metronome request bodies are validated against Metronome's published OpenAPI schemas in `internal/destinations/metronome/testdata`, and webhook verification against Metronome's documented test vector. -```bash -cd deploy/lago -docker compose --env-file .env up -d +``` +cmd/vbilling/ wiring +internal/usage/ canonical schema, metric catalog, deterministic IDs +internal/discovery/ tenant clusters (+ vCluster Platform enrichment) +internal/metrics/ window metering: lifetimes, fractional GPUs, fairness, Prometheus +internal/spool/ hash-chained ledger, window commits, cursors, dead letters +internal/pipeline/ per-destination delivery, retries, poison isolation, tenant sync +internal/destinations/ stripe, metronome, lago, webhook, noop +internal/enforcement/ billing webhooks -> tenant billing state -> cluster labels +internal/api/ HTTP API and dashboard ``` -### Kubernetes (Production) - -Deploy Lago as Kubernetes workloads. Key components: PostgreSQL, Redis, API (Rails), Sidekiq worker, Clock, Frontend. See [Lago docs](https://getlago.com/docs) for production guidance. +To add a destination, implement `destinations.Destination` (and optionally `Reconciler`, `CustomerResolver`, `EventFilter`), then register it in `init()`. ## Roadmap -- [x] Source / Destination plugin refactor (adapter pattern) -- [ ] Metronome adapter -- [ ] Stripe Meters adapter -- [ ] OpenMeter adapter (native CloudEvents) -- [ ] Custom adapter developer guide -- [ ] MIG (Multi-Instance GPU) partition tracking -- [ ] Grafana dashboard integration -- [ ] Budget alerts per Tenant Cluster -- [ ] Reserved capacity / commitment pricing -- [ ] Auto Nodes billing (dynamic node provisioning events) -- [ ] Netris network isolation billing integration +- Hourly usage rollups so long-range usage queries stay fast on large fleets (today the usage API scans the ledger: about 6.6 µs per event) +- Slurm accounting (`sacct`) exporter +- Shared DRA devices (consumable capacity) and per-namespace showback of private nodes +- OpenMeter and Kafka destinations, S3/Parquet export +- Reseller and partner attribution, and pre-provisioning cost estimates +- Leader election for single-writer HA ## License diff --git a/docs/configuration.html b/docs/configuration.html index 8817972..00d3341 100644 --- a/docs/configuration.html +++ b/docs/configuration.html @@ -228,7 +228,8 @@ Getting Started Configuration Metrics Reference - Private Nodes + Dedicated and Private Nodes + Example: GKE with GPUs Troubleshooting
@@ -241,275 +242,546 @@

Configuration

-

All the knobs for tuning vBilling behavior: environment variables, Helm chart values, and Lago pricing setup.

+

Every setting for vBilling v0.2: environment variables, Helm chart values, and pricing setup in your billing backend.

Environment Variables

-

When running vBilling as a binary or in a container, the following environment variables control its behavior. All have sensible defaults. LAGO_API_KEY is required when ADAPTER=lago (the default).

- +

vBilling reads its configuration from environment variables; the Helm chart sets them from values.yaml. Credentials come from Kubernetes Secrets (see existingSecret in each section of the chart).

+

Destinations

+ + + - - - + + + + + + + + + +
VariableDescriptionDefault
VariableDescriptionDefaultADAPTERSComma-separated destinations usage fans out to: stripe, metronome, lago, webhook, noop. Each destination reads the ledger through its own cursor.lago
ADAPTERDeprecated v0.1 single-adapter form, used only when ADAPTERS is empty.(none)
+
+

Identity and region

+
+ + + - - - + + + - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariableDescriptionDefault
ADAPTERBilling adapter to use. Built-in: lago, noop. More adapters (Metronome, Stripe Meters, OpenMeter) are on the roadmap; custom adapters can be added by implementing the Destination interface.lagoREGIONRegion stamped on every event (for example ap-southeast-2). Run one vBilling per control plane cluster.default
LAGO_API_URLBase URL of the Lago API server (used when ADAPTER=lago)http://localhost:3000CLUSTER_NAMEName of this control plane cluster; used as the CloudEvents source.control-plane
LAGO_API_KEYRequired when ADAPTER=lago. API key for authenticating with LagoZONEDefault zone when nodes carry no topology.kubernetes.io/zone label. (none)
REGION_FROM_NODEUse the node's topology.kubernetes.io/region label instead of REGION.false
DEFAULT_TENANT_CLASSTenant class stamped on events when a tenant cluster has none (public, enterprise, government, dev, ...).(none)
TENANT_SOURCEcluster: one customer per tenant cluster. project: one per vCluster Platform project. The vbilling.vcluster.com/tenant label always wins.cluster
PLATFORM_CLUSTERvCluster Platform cluster name of this control plane cluster, so Platform enrichment only applies to local tenant clusters.(none)
WATCH_NAMESPACESComma-separated namespaces to discover tenant clusters in. Empty watches all namespaces.(all)
+
+

Ledger

+
+ + + + + + + + + + + + + + + + +
VariableDescriptionDefault
DATA_DIRDirectory for the ledger and adapter state. Keep it on a persistent volume./var/lib/vbilling
RETENTIONHow long closed ledger segments are kept once every destination has consumed them.168h
+
+

Metering

+
+ + + + + - + - + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariableDescriptionDefault
COLLECTION_INTERVALHow often to scrape metrics and send usage events to Lago. Accepts Go duration strings (e.g. 60s, 5m)Metering window size, aligned to the wall clock. Must be 10s to 1h and divide an hour. 60s
RECONCILE_INTERVALHow often to scan for new or removed vClusters and sync billing entities. Accepts Go duration stringsHow often tenant clusters are rediscovered. 30s
DEFAULT_PLAN_CODELago plan code to assign to newly discovered vClustersvcluster-standardMAX_BACKFILLWindows missed during downtime are backfilled up to this age.6h
BILLING_CURRENCYISO 4217 currency code for all billing operationsUSDOFFBOARD_GRACEA tenant must have no tenant clusters for this long before destinations offboard it.1h
PROMETHEUS_URLPrometheus server URL for DCGM GPU utilization and network egress metrics. Leave empty to disable(empty)CPU_MEMORY_BASISusage (metrics-server), requests, or max of both.usage
SPOT_DISCOUNT_PERCENTPercentage discount applied to workloads running on spot/preemptible nodes. A value of 60 means spot nodes cost 40% of on-demand60METER_CONTROL_PLANEAlso bill the tenant cluster's own control plane pods and data PVCs (normally covered by instance hours).false
WATCH_NAMESPACESComma-separated list of namespaces to monitor for vClusters. Empty means watch all namespaces(all namespaces)METER_BY_NAMESPACEAdd the namespace inside the tenant cluster as a dimension, for tenant-side chargeback.false
GPU_RESOURCESFull-GPU resource names. MIG (nvidia.com/mig-*) and nvidia.com/gpu.shared are always detected.nvidia.com/gpu,amd.com/gpu
SKU_LABELNode label whose value is the SKU (for example vbilling.vcluster.com/sku).(none)
CAPACITY_TYPE_LABELNode label for on-demand, spot, preemptible or reserved; beats cloud-provider labels.vbilling.vcluster.com/capacity-type
UNHEALTHY_NODE_TAINTSTaints that mark a node down: its GPU time is recorded as downtime, not billed. Add your GPU health taints.node.kubernetes.io/not-ready,node.kubernetes.io/unreachable
VCLUSTER_NODE_LABELCustom node label selector for private node detection. Format: key=value-pattern where %s is replaced with the vCluster name. See Private NodesExtra dedicated-node selector, key=%s where %s is the tenant cluster name. See Dedicated nodes.(none)
CUSTOM_METRICSOperator metrics accepted by the ingest API, code:unit[:key|key],...(none)
PROMETHEUS_URLAny Prometheus HTTP API (Prometheus, Thanos, Mimir, VictoriaMetrics). Enables egress and GPU utilization metrics.(none)
EGRESS_QUERYPromQL template for egress bytes. Placeholders: {{namespace}}, {{vcluster}}, {{instance}}, {{project}}, {{tenant}}, {{tenant_cluster}}, {{window}} (e.g. 60s), {{window_seconds}} (e.g. 60).container_network_transmit_bytes_total increase
GPU_UTIL_QUERY, GPU_COUNT_QUERYReplace the GPU utilization queries (same placeholders, results grouped by modelName or gpu_type). none disables utilization.DCGM, namespace or exported_namespace
PROMETHEUS_HEADERSExtra query headers, Name=value,..., e.g. X-Scope-OrgID for Mimir or Authorization. Basic auth also works in the URL.(none)
PROMETHEUS_PRESETvcluster-platform reads vCluster Platform fleet observability: egress and GPU utilization selected by the vcluster_platform_instance and vcluster_platform_project labels.(none)
PROMETHEUS_METRICS_FILEMetrics defined by PromQL, evaluated per tenant cluster at every window end (billable ones are created in the billing backends).(none)
PROMETHEUS_BEARER_TOKEN_FILEBearer token file, re-read on every query (for example a service account token for an OpenShift Thanos querier). (none)
- -
-
i
-
COLLECTION_INTERVAL directly affects billing granularity and accuracy. Shorter intervals produce more events but more precise billing. The default of 60 seconds is a good balance.
+

Tenant clusters with their own nodes

+
+ + + + + + + + + + + + +
VariableDescriptionDefault
TENANT_APIMeter tenant clusters with their own nodes (Private Nodes, Auto Nodes, Standalone) through their own API.true
TENANT_KUBECONFIG_SECRETSecret in each tenant cluster's namespace holding a read-only kubeconfig (vCluster exportKubeConfig.additionalSecrets).vbilling-kubeconfig
TENANT_ADMIN_FALLBACKAlso accept the admin kubeconfig vc-<name> (needs Secret read access).false
TENANT_CLUSTERS_FILEYAML list of external tenant clusters: name, kubeconfig, tenant, displayName, project, metadata.(none)
PRIVATE_NODE_BILLINGnode bills each private node whole; usage bills the pods on them.node
TENANT_EXCLUDE_NAMESPACESUsage mode: namespaces never billed.kube-system
WATCH_DELETIONSBill pods and nodes deleted between windows until the moment they were deleted.true
DRA_GPU_DRIVERSDRA drivers whose devices are GPUs (ResourceClaim metering).gpu.nvidia.com,gpu.amd.com
- - -

Helm Chart Values

-

When deploying via Helm, all configuration is specified through values.yaml or --set flags. The Helm chart translates these into the environment variables shown above.

- -

Lago connection

+

API and enforcement

- + - - - + + + - - - + + + - - - + + + + + + + + + + + + +
ValueDescriptionDefault
VariableDescriptionDefault
lago.apiURLLago API URL (in-cluster service address)http://lago-api.lago-system.svc.cluster.local:3000LISTEN_ADDRHTTP listen address for the API, dashboard, metrics and webhooks.:8080
lago.apiKeyLago API key (plaintext, use --set)(empty)API_TOKENBearer token for /api/*. Empty leaves the API unauthenticated (keep the Service internal).(none)
lago.existingSecretName of an existing Kubernetes Secret containing the API key (key: api-key). Preferred over lago.apiKey for production(empty)INGEST_TOKENBearer token for POST /api/v1/events; defaults to API_TOKEN. Ingest is disabled without a token.(none)
ENFORCEMENT_MODEobserve, annotate or enforce. See the README.observe
ENFORCEMENT_RULESOverrides, event.type=state,... (for example alerts.spend_threshold_reached=suspended).(none)
- -

Billing settings

+

Stripe

- + - - - + + + - - - + + + - - - + + + - - - + + + + + + + + + + + + + + + + + + + + + + +
ValueDescriptionDefault
VariableDescriptionDefault
billing.defaultPlanCodeLago plan code for new vClustersvcluster-standardSTRIPE_API_KEYSecret or restricted key. Test-mode keys default to 20 requests/s.(none)
billing.currencyBilling currency (ISO 4217)USDSTRIPE_WEBHOOK_SECRETEnables /webhooks/stripe (dunning and usage alerts).(none)
billing.collectionIntervalMetrics scrape interval60sSTRIPE_SPLIT_METERS_BYEvent fields that get their own meter (Stripe meters cannot price on dimensions).sku
billing.reconcileIntervalvCluster discovery interval30sSTRIPE_AUTO_SUBSCRIBESubscribe tenants to active metered prices tagged metadata[vbilling_plan]=<plan>.false
STRIPE_MAX_RPSRequest rate cap; 0 picks 20 (test) or 200 (live).0
STRIPE_CANCEL_ON_REMOVECancel the tenant's subscription at period end when it is offboarded.false
STRIPE_API_VERSIONPinned Stripe API version.2026-09-30.endive
STRIPE_API_BASEAPI base URL (stripe-mock for tests).https://api.stripe.com
- -

Pricing defaults

+

Metronome

- + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + + +
ValueDescriptionDefault
VariableDescriptionDefault
pricing.cpuPerCoreHourPrice per CPU core-hour0.05METRONOME_API_TOKENBearer token.(none)
pricing.memoryPerGBHourPrice per memory GB-hour0.01METRONOME_WEBHOOK_SECRETEnables /webhooks/metronome (spend, usage and credit alerts).(none)
pricing.storagePerGBHourPrice per storage GB-hour0.001METRONOME_RATE_CARDRate card alias or ID; creates a contract per tenant. A tenant's plan annotation overrides it.(none)
pricing.instancePerHourFlat hourly charge per vCluster instance0.10METRONOME_STRIPE_LINKCreate each tenant's Stripe customer and link it for invoicing (needs STRIPE_API_KEY).false
pricing.gpuDefaultPerHourDefault GPU price per hour (when type not in gpuPrices)2.50METRONOME_STRIPE_COLLECTION_METHODcharge_automatically or send_invoice.charge_automatically
pricing.networkEgressPerGBPrice per GB of network egress0.05METRONOME_API_BASEAPI base URL.https://api.metronome.com
+
+

Lago

+
+ + + + + - - - + + + - - - + + + - - - + + + + + + + +
VariableDescriptionDefault
pricing.lbPerHourPrice per LoadBalancer service-hour0.025LAGO_API_URLLago API base URL.http://localhost:3000
pricing.gpuPricesComma-separated GPU type-specific pricing in TYPE:PRICE formatNVIDIA-A100-SXM4-80GB:3.50,NVIDIA-H100-80GB-HBM3:5.00,NVIDIA-H100-SXM5-80GB:5.50,NVIDIA-L40S:1.80,NVIDIA-A10G:1.20,NVIDIA-T4:0.75LAGO_API_KEYRequired when the lago adapter is enabled.(none)
pricing.spotDiscountPercentSpot instance discount percentage60DEFAULT_PLAN_CODEPlan code for new tenants (Lago plan; Stripe vbilling_plan tag).vcluster-standard
BILLING_CURRENCYISO 4217 currency for customers and plans.USD
- -

Optional integrations

+

Webhook

- + - - - + + + + + + + + - - - + + + + + +
ValueDescriptionDefault
VariableDescriptionDefault
prometheus.urlPrometheus URL for DCGM and network metrics(empty)WEBHOOK_URLEndpoint receiving CloudEvents 1.0 batches.(none)
WEBHOOK_SECRETHMAC secret for the X-VBilling-Signature header (Stripe-compatible scheme).(none)
watchNamespacesList of namespaces to watch (YAML array)[] (all)WEBHOOK_HEADERSExtra headers, k=v,k=v.(none)
+
+

Deprecated

+
+ + + + + + + + +
VariableDescriptionDefault
SPOT_DISCOUNT_PERCENTIgnored since v0.2. Price capacity_type in your billing backend instead, so quantities stay physical and reconcilable.(none)
-

Deployment settings

+ +

Helm Chart Values

+

The chart deploys a StatefulSet with a persistent ledger volume, a Service, RBAC (enforcement verbs only outside observe mode), and optionally a ValidatingAdmissionPolicy and ServiceMonitor.

- + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescriptionDefault
ValueSetsDefault
image.repositoryContainer image repositoryghcr.io/vclusterlabs-experiments/vbillingadaptersADAPTERS[lago]
region / clusterNameREGION / CLUSTER_NAMEdefault / control-plane
tenantClass / tenantSourceDEFAULT_TENANT_CLASS / TENANT_SOURCE"" / cluster
billing.collectionInterval / reconcileInterval / maxBackfill / offboardGraceCOLLECTION_INTERVAL, RECONCILE_INTERVAL, MAX_BACKFILL, OFFBOARD_GRACE60s / 30s / 6h / 1h
billing.defaultPlanCode / currencyDEFAULT_PLAN_CODE / BILLING_CURRENCYvcluster-standard / USD
metering.*CPU_MEMORY_BASIS, METER_*, GPU_RESOURCES, SKU_LABEL, CAPACITY_TYPE_LABEL, UNHEALTHY_NODE_TAINTS, VCLUSTER_NODE_LABEL, REGION_FROM_NODE, CUSTOM_METRICSsee above
prometheus.url / egressQuery / gpuUtilQuery / gpuCountQuery / headers / existingSecret / bearerTokenFile / preset / metricsPROMETHEUS_URL / EGRESS_QUERY / GPU_UTIL_QUERY / GPU_COUNT_QUERY / PROMETHEUS_HEADERS (chart Secret) / PROMETHEUS_BEARER_TOKEN_FILE / PROMETHEUS_PRESET / PROMETHEUS_METRICS_FILE (ConfigMap)(none)
tenantAPI.enabled / secretName / adminFallback / privateNodeBilling / excludeNamespaces / externalClustersSecretTENANT_API / TENANT_KUBECONFIG_SECRET / TENANT_ADMIN_FALLBACK / PRIVATE_NODE_BILLING / TENANT_EXCLUDE_NAMESPACES / TENANT_CLUSTERS_FILE (Secret mounted at /etc/vbilling/tenants)true / vbilling-kubeconfig / false / node / [kube-system] / ""
image.tagContainer image tagv0.1.0metering.watchDeletions / draGPUDriversWATCH_DELETIONS / DRA_GPU_DRIVERStrue / [gpu.nvidia.com, gpu.amd.com]
image.pullPolicyImage pull policyIfNotPresentplatform.clusterPLATFORM_CLUSTER(none)
replicaCountNumber of replicas (only 1 recommended)1lago.apiURL / apiKey / existingSecret (key api-key)LAGO_API_URL / LAGO_API_KEY(none)
serviceAccount.createCreate a ServiceAccount for the deploymenttruestripe.apiKey / webhookSecret / existingSecret (keys api-key, webhook-secret)STRIPE_API_KEY / STRIPE_WEBHOOK_SECRET(none)
resources.requests.cpuCPU request50mstripe.splitMetersBy / autoSubscribe / maxRPS / cancelOnRemove / apiVersion / apiBaseSTRIPE_*[sku] / false / 0 / false
resources.requests.memoryMemory request64Mimetronome.apiToken / webhookSecret / existingSecret (keys api-token, webhook-secret)METRONOME_API_TOKEN / METRONOME_WEBHOOK_SECRET(none)
resources.limits.cpuCPU limit200mmetronome.rateCard / stripeLink / stripeCollectionMethod / apiBaseMETRONOME_*(none)
resources.limits.memoryMemory limit128Miwebhook.url / secret / headers / existingSecret (key secret)WEBHOOK_*(none)
api.port / token / ingestToken / existingSecret (keys api-token, ingest-token)LISTEN_ADDR / API_TOKEN / INGEST_TOKEN8080
api.service.type / portServiceClusterIP / 8080
enforcement.mode / rulesENFORCEMENT_MODE / ENFORCEMENT_RULESobserve
enforcement.admissionPolicy.enabledValidatingAdmissionPolicy blocking new pods in suspended namespaces (Kubernetes 1.30+)false
persistence.enabled / size / storageClass / retentionLedger PVC (StatefulSet volumeClaimTemplate) / RETENTIONtrue / 5Gi / "" / 168h
serviceMonitor.enabled / intervalPrometheus Operator ServiceMonitorfalse / 30s
image.repository / tag / pullPolicyContainer imageghcr.io/vclusterlabs-experiments/vbilling / v0.2.0
@@ -517,14 +789,14 @@

Deployment settings

Example Helm install

helm upgrade --install vbilling deploy/helm/vbilling \ - --namespace vbilling-system --create-namespace \ - --set lago.existingSecret="vbilling-lago-secret" \ - --set billing.collectionInterval="120s" \ - --set prometheus.url="http://prometheus.monitoring.svc.cluster.local:9090" \ - --set pricing.cpuPerCoreHour="0.08" \ - --set pricing.gpuDefaultPerHour="3.00"
+ -n vbilling-system --create-namespace \ + --set adapters='{metronome,webhook}' \ + --set metronome.existingSecret=metronome-credentials \ + --set metronome.rateCard=payg-aud --set metronome.stripeLink=true \ + --set stripe.existingSecret=stripe-credentials \ + --set webhook.url=https://data.example.com/usage \ + --set region=ap-southeast-2 --set api.existingSecret=vbilling-api
-

Lago Pricing Configuration

vBilling bootstraps 9 billable metrics and a vcluster-standard plan on startup. All charges default to $0.00 per unit. You must configure pricing in the Lago UI or API to actually generate non-zero invoices.

@@ -595,9 +867,9 @@

Charges to configure

-

Example: Neocloud pricing

-

For a GPU-focused neocloud selling compute to AI teams, you might set higher GPU prices and include a premium for dedicated infrastructure:

-
# Neocloud pricing example +

Example: AI Cloud pricing

+

For a GPU-focused AI Cloud selling compute to AI teams, you might set higher GPU prices and include a premium for dedicated infrastructure:

+
# AI Cloud pricing example CPU Core-Hours: $0.08 # higher due to bare-metal premium Memory GB-Hours: $0.015 # ECC memory cost Storage GB-Hours: $0.002 # NVMe SSD storage @@ -625,6 +897,13 @@

Example: Internal platform team pricing

If you see usage events in Lago but invoices show $0.00, it means the charge amounts are still set to their defaults ($0). Edit the plan charges in the Lago UI.
+

To price H100 hours differently from L40S hours in Lago, add charge filters on the sku (or gpu_type) event property. vBilling sends every dimension as an event property: sku, region, capacity_type, billing_mode, tenant_class and tenant_cluster. Spot and preemptible discounts are a filter on capacity_type.

+ +

Stripe and Metronome Pricing

+

Stripe: vBilling creates a Billing Meter per metric and, because Stripe meters cannot price on dimensions, a meter per SKU as new SKUs appear (for example vcluster_gpu_hours__nvidia_h100_80gb_hbm3). Create a metered price on each meter you charge for. To subscribe new tenants automatically, tag the prices metadata[vbilling_plan]=vcluster-standard (or the tenant's plan annotation) and enable stripe.autoSubscribe.

+

Metronome: vBilling creates SUM billable metrics named <Metric> [<code>], each with one compound group key: region, sku, capacity_type, billing_mode, tenant_cluster for GPU hours. On your rate card, add a usage product per metric with pricing_group_key set to the dimensions you price on (for example [region, sku, capacity_type]) and presentation_group_key: [tenant_cluster] for invoice lines per tenant cluster. Rate cards carry the currency, so use one rate card per currency (for example AUD and NZD). Point metronome.rateCard or a tenant's plan annotation at it.

+ +
diff --git a/docs/example-gke.html b/docs/example-gke.html new file mode 100644 index 0000000..4fc206b --- /dev/null +++ b/docs/example-gke.html @@ -0,0 +1,418 @@ + + + + + + Example: GKE with GPUs - vBilling Docs + + + + + + + + + + +
+ + +
+

Example: GKE with shared and private GPU nodes

+

Both vCluster tenancy models on GPUs, metered by vBilling and invoiced by Stripe test mode, from an empty Google Cloud project. The manifests, values files and commands are the ones from a recorded run, with every key, token and account ID left out.

+ +

What you build

+
    +
  • GKE as the control plane cluster, with a spot L4 node pool time-shared four ways.
    • +
  • shared-l4, a tenant cluster on shared nodes: its pods run on the L4 and are billed per pod, a quarter GPU each.
    • +
  • private-a100, a tenant cluster with private nodes: an A100 VM joins it directly and is billed whole, read through the tenant cluster's own API with a read-only kubeconfig.
    • +
  • Prometheus reading DCGM: GKE's exporter on shared nodes, and a Prometheus agent that remote-writes from the private node.
    • +
  • vBilling sending usage to Stripe: customers, one meter per metric and SKU, subscriptions, invoices.
    • +
+
+ + + + + + + + + + + + +
File (examples/gke-gpu)What it is
01-prometheus.yamlCentral Prometheus: scrapes GKE's DCGM exporter, accepts remote writes
02-tenant-shared.yamlvcluster.yaml for the shared-nodes tenant cluster
03-tenant-private.yamlvcluster.yaml with Private Nodes and the read-only vbilling-kubeconfig export
04-node-startup.shVM startup script that joins the GPU machine as a private node
05-gpu-operator-values.yamlNVIDIA GPU Operator inside the private tenant cluster
06-prometheus-agent.yamlPrometheus agent in the private tenant cluster, remote-writing DCGM
07-vbilling-values.yamlvBilling Helm values: Stripe, DCGM queries, a PromQL billable metric
workloads/Sample GPU workloads for each tenant cluster
+
+
+
!
+
No file in the example contains a key, token or account ID. Keep yours out of Git too: the Stripe key goes into a Kubernetes Secret from a local file, and the node join token only into the VM's metadata, removed after the join.
+
+ +

Prerequisites

+
    +
  • gcloud with a project selected, kubectl, helm, and the vcluster CLI (0.36 or later).
    • +
  • vCluster Platform for the Private Nodes license. The CLI must be logged in with a key that can register tenant clusters (a scoped key fails with denied by loft access key scope). vCluster 0.36.2 and later need Platform 4.11.3 or later.
    • +
  • A Stripe test-mode secret key saved in a local file, for example ~/.stripe-test-key.
    • +
  • GPU quota in the zone for one L4 and one A100 (spot capacity is enough).
    • +
+
export ZONE=us-central1-a CLUSTER=vbilling-demo +git clone https://github.com/vClusterLabs-Experiments/vbilling && cd vbilling/examples/gke-gpu
+ +

1. Control plane cluster

+
gcloud container clusters create $CLUSTER --zone $ZONE --release-channel regular \ + --machine-type e2-standard-4 --num-nodes 2 +gcloud container node-pools create l4-spot --cluster $CLUSTER --zone $ZONE \ + --machine-type g2-standard-8 --spot --num-nodes 1 \ + --accelerator type=nvidia-l4,count=1,gpu-driver-version=default,gpu-sharing-strategy=time-sharing,max-shared-clients-per-gpu=4 \ + --node-labels vbilling.vcluster.com/capacity-type=spot +gcloud container clusters get-credentials $CLUSTER --zone $ZONE
+ +

2. Prometheus

+

One Prometheus for the control plane cluster. GKE (1.32 and later) runs a managed DCGM exporter on GPU nodes; scraped with the exporter's own target labels, the workload's namespace arrives as exported_namespace, which vBilling's default queries match. An internal load balancer accepts remote writes from private nodes.

+
kubectl apply -f 01-prometheus.yaml +INGEST_IP=$(kubectl -n monitoring get svc prometheus-ingest -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+ +

3. Tenant clusters

+
vcluster platform login https://<your-platform-host> # browser login, no key on the command line +vcluster create shared-l4 -n shared-l4 -f 02-tenant-shared.yaml --connect=false +vcluster create private-a100 -n private-a100 -f 03-tenant-private.yaml --connect=false
+

When the CLI is logged in, vcluster create registers each tenant cluster with the Platform, which is what licenses Private Nodes. A tenant cluster created before logging in is registered with vcluster platform add vcluster private-a100 -n private-a100 --project default. The private tenant cluster's exportKubeConfig.additionalSecrets writes the read-only Secret vbilling-kubeconfig that vBilling reads (see Private Nodes).

+

Tell vBilling who the tenants are (the emails use the reserved .example domain):

+
kubectl annotate ns shared-l4 vbilling.vcluster.com/tenant=acme-ai "vbilling.vcluster.com/display-name=Acme AI" \ + vbilling.vcluster.com/email=billing@acme-ai.example vbilling.vcluster.com/plan=gpu-cloud +kubectl annotate ns private-a100 vbilling.vcluster.com/tenant=initech "vbilling.vcluster.com/display-name=Initech" \ + vbilling.vcluster.com/email=billing@initech.example vbilling.vcluster.com/plan=gpu-cloud
+ +

4. A private GPU node

+

Create a join token, put the join command it prints into the startup script, and boot the machine. The VM must reach the tenant cluster's internal load balancer, so keep it in the same VPC and region.

+
vcluster connect private-a100 -n private-a100 -- vcluster token create --expires 2h | grep node/join > join.txt +awk -v cmd="$(cat join.txt)" '$0 == "JOIN_COMMAND" { print cmd; next } { print }' 04-node-startup.sh > startup.sh +gcloud compute instances create gpu-node-1 --zone $ZONE --machine-type a2-highgpu-1g \ + --provisioning-model SPOT --instance-termination-action STOP --maintenance-policy TERMINATE \ + --image-family ubuntu-2204-lts --image-project ubuntu-os-cloud --boot-disk-size 150GB \ + --no-service-account --no-scopes --metadata-from-file startup-script=startup.sh
+

Once gpu-node-1 is Ready in the tenant cluster, remove the token from the instance and from disk, and label the node so it gets an SKU:

+
gcloud compute instances remove-metadata gpu-node-1 --zone $ZONE --keys startup-script +rm join.txt startup.sh +vcluster connect private-a100 -n private-a100 -- kubectl label node gpu-node-1 \ + node.kubernetes.io/instance-type=a2-highgpu-1g vbilling.vcluster.com/capacity-type=spot
+ +

5. GPU Operator and DCGM in the private tenant cluster

+
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update +vcluster connect private-a100 -n private-a100 -- \ + helm install gpu-operator nvidia/gpu-operator -n gpu-operator --create-namespace -f 05-gpu-operator-values.yaml +sed "s/PROMETHEUS_INGEST_IP/$INGEST_IP/" 06-prometheus-agent.yaml > agent.yaml +vcluster connect private-a100 -n private-a100 -- kubectl apply -f agent.yaml
+

The driver container compiles for the node's kernel, about five minutes. The agent adds the external label vcluster=private-a100 to every series it sends, which is what 07-vbilling-values.yaml selects with {{vcluster}}.

+
+
i
+
MIG on Compute Engine. Label the node nvidia.com/mig.config=all-1g.5gb to split the A100 into seven slices. The GPU cannot be reset from inside a Compute Engine VM, so the MIG manager reports failed until the VM is rebooted (gcloud compute instances reset). A stop and start can move the VM to a GPU without MIG mode. Under MIG, vBilling still bills one whole A100, and DCGM utilization comes from each instance's engine activity.
+
+ +

6. vBilling

+
kubectl create namespace vbilling +kubectl -n vbilling create secret generic vbilling-stripe --from-file=api-key=$HOME/.stripe-test-key +helm install vbilling ../../deploy/helm/vbilling -n vbilling -f 07-vbilling-values.yaml +vcluster connect shared-l4 -n shared-l4 -- kubectl apply -f workloads/shared-inference.yaml +vcluster connect private-a100 -n private-a100 -- kubectl apply -f workloads/private-training.yaml
+

vBilling creates the Stripe customers and meters on the first windows. Give the meters prices tagged with the plan, then restart vBilling so it subscribes the tenants:

+
export STRIPE_API_KEY="$(cat ~/.stripe-test-key)" +curl -s https://api.stripe.com/v1/billing/meters -u "$STRIPE_API_KEY:" | grep -E '"(id|event_name)"' +curl -s https://api.stripe.com/v1/prices -u "$STRIPE_API_KEY:" \ + -d currency=usd -d unit_amount_decimal=220 \ + -d "recurring[interval]=month" -d "recurring[usage_type]=metered" -d "recurring[meter]=<meter id>" \ + -d "product_data[name]=A100 40GB GPU-hour" -d "metadata[vbilling_plan]=gpu-cloud" +kubectl -n vbilling rollout restart statefulset/vbilling
+ +

7. Check it

+
kubectl -n vbilling port-forward svc/vbilling 8080:8080 & +# dashboard: http://localhost:8080/ +curl -s "localhost:8080/api/v1/events?from=$(date -u +%Y-%m-%dT%H:00:00Z)" | head +curl -s "localhost:8080/api/v1/reconcile?from=<start>&to=<end>" # ledger vs Stripe, per tenant and metric
+

One 30 second window:

+
+ + + + + + + + + +
TenantMetricQuantityWhy
acme-aivcluster_gpu_hours (sku nvidia-l4-timeslice-4)0.0041666672 pods × ¼ GPU × 30 s
initechvcluster_gpu_hours (sku NVIDIA-A100-SXM4-40GB)0.008333333The whole A100, also under MIG
initechvcluster_private_node_hours (sku a2-highgpu-1g)0.008333333The node, billed whole
initechvcluster_cpu_core_hours0.1All 12 cores
initechgpu_energy_kwh≈ 0.0015≈ 177 W for 30 s, from DCGM
+
+

vBilling dashboard for the GKE example: GPU-hours by SKU, usage per tenant, Stripe delivering with no lag and no dead letters, verified ledger

+

If the spot A100 is reclaimed, the window in which the node becomes unreachable bills only the seconds before it, and the rest becomes vcluster_gpu_downtime_hours until the node and its GPUs are back.

+ +

Clean up

+
gcloud compute instances delete gpu-node-1 --zone $ZONE --quiet +vcluster delete shared-l4 -n shared-l4 --delete-namespace +vcluster delete private-a100 -n private-a100 --delete-namespace +kubectl delete namespace monitoring vbilling # removes the load balancers and volumes GKE created +gcloud container clusters delete $CLUSTER --zone $ZONE --quiet
+
+
+ + + + + + diff --git a/docs/getting-started.html b/docs/getting-started.html index 31688e2..62952c0 100644 --- a/docs/getting-started.html +++ b/docs/getting-started.html @@ -264,7 +264,8 @@ Getting Started Configuration Metrics Reference - Private Nodes + Dedicated and Private Nodes + Example: GKE with GPUs Troubleshooting
@@ -294,7 +295,7 @@

Prerequisites

i
-
vBilling auto-discovers vClusters by scanning for StatefulSets/Deployments with the app=vcluster label, or via the vCluster Platform API if available. Make sure you have at least one vCluster running before starting.
+
vBilling auto-discovers tenant clusters by scanning for StatefulSets/Deployments with the app=vcluster label, or via the vCluster Platform API if available. Make sure you have at least one vCluster running before starting.
@@ -352,7 +353,7 @@

Get the API key from the database

# (1 row) # Save it for later -export LAGO_API_KEY="a1b2c3d4-e5f6-7890-abcd-ef1234567890"
+export LAGO_API_KEY="<your-lago-api-key>"
!
@@ -365,11 +366,11 @@

2Install vBilling

Option A: Helm chart (recommended)

-

First, build and push the container image (if not using a pre-built image):

+

The chart defaults to the published image ghcr.io/vclusterlabs-experiments/vbilling:v0.2.0 (linux/amd64 and linux/arm64). Build your own only to run local changes:

# Build multi-arch image docker buildx build \ --platform linux/amd64,linux/arm64 \ - -t ghcr.io/vclusterlabs-experiments/vbilling:latest \ + -t <your-registry>/vbilling:dev \ --push . # Or single-arch for local testing: @@ -412,7 +413,7 @@

Option B: Run locally (development)

# Run with environment variables export LAGO_API_URL="http://localhost:3000" -export LAGO_API_KEY="a1b2c3d4-e5f6-7890-abcd-ef1234567890" +export LAGO_API_KEY="<your-lago-api-key>" ./bin/vbilling # Expected output: @@ -432,7 +433,7 @@

Option B: Run locally (development)

i
-
When running locally, vBilling uses your ~/.kube/config to connect to the cluster. Make sure your kubeconfig context points to the cluster with your vClusters.
+
When running locally, vBilling uses your ~/.kube/config to connect to the cluster. Make sure your kubeconfig context points to the cluster with your tenant clusters.
@@ -467,7 +468,7 @@

Edit the plan charges

4Verify It Works

-

Once vBilling is running, it will auto-discover your vClusters and start sending usage events within one reconcile cycle (default: 30 seconds).

+

Once vBilling is running, it will auto-discover your tenant clusters and start sending usage events within one reconcile cycle (default: 30 seconds).

Check vBilling logs

# If running via Helm: @@ -516,8 +517,8 @@

Check current usage

Try It Locally with vind

You can test the full pipeline on your laptop using vind (vCluster in Docker) -- no cloud cluster needed.

-

Create a vind host cluster

-
# Set Docker driver and create a host cluster +

Create a vind control plane cluster

+
# Set Docker driver and create a control plane cluster vcluster use driver docker vcluster create vbilling-host --connect=true @@ -526,7 +527,7 @@

Create a vind host cluster

kubectl patch deployment metrics-server -n kube-system --type=json \ -p='[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]'
-

Create nested vClusters

+

Create nested tenant clusters

vcluster use driver helm vcluster create team-alpha --namespace vcluster-team-alpha --connect=false vcluster create team-beta --namespace vcluster-team-beta --connect=false @@ -555,20 +556,19 @@

Deploy vBilling as a pod in the vind cluster

--set lago.existingSecret=lago-credentials

Verify

-
kubectl logs -n vbilling-system deployment/vbilling +
kubectl logs -n vbilling-system statefulset/vbilling -# Expected output: +# Expected output (v0.2): # Using in-cluster Kubernetes config -# [discovery] found 3 vCluster(s) -# [controller] new vCluster discovered: vcluster-team-alpha/team-alpha -# [controller] ensured customer vcluster-vcluster-team-alpha-team-alpha -# [metrics] vcluster-team-alpha: CPU=0.048 cores, Memory=0.33 GB -# [metrics] vcluster-team-alpha: Storage=5.00 GB (1 PVCs) -# [controller] sent 12 billing events to Lago
+# [controller] starting (adapters=[lago], window=1m0s, reconcile=30s, region=default) +# API listening on :8080 +# [bootstrap] setting up Lago billing configuration... +# [controller] window 10:18:20: 17 events for 3 tenant cluster(s) (11 pods, 5 GPU pods, 0 nodes down) +# [lago] subscribed tenant acme to plan vcluster-standard
✓
-
This is the exact output from a verified test. vBilling runs as a real Kubernetes Deployment with in-cluster ServiceAccount, discovers vClusters via RBAC, collects metrics, and sends events to Lago.
+
This is output from a verified end-to-end test. vBilling runs as a StatefulSet with a persistent ledger and an in-cluster ServiceAccount, discovers tenant clusters via RBAC, and delivers events to Lago. Check delivery with kubectl -n vbilling-system port-forward svc/vbilling 8080 and curl localhost:8080/api/v1/status.

Clean up

@@ -580,7 +580,7 @@

Next Steps

diff --git a/docs/images/dashboard-gke.png b/docs/images/dashboard-gke.png new file mode 100644 index 0000000000000000000000000000000000000000..9ffc79de91277931257d9a0ff73166d217f82d99 GIT binary patch literal 171809 zcmZsCb6}m%wswQYY|@xbW7}@f*tTukX&X1TjosL`ZQHiLo%DCkJ@=k(|F_@$?wK_+ z3(tDinx8UKBJeO+FdsgAfd4M~P42^ok6%B00P}}}06xjhLqz`Y0qMi{Z~O`_AV;Ym zbw2l^1-&-gwlBYbn!9tU9j&!B3nnHe1bjEtmfoH+BN4ppq#MxQ_MrSmVvn3(9#6e@ zI-O@UHgckt!u#m|#eV)$0YNOeR3K6$ufldm3+u{iY)BQhGe)`2VKalIR2!j4tyW&9 zh-$m!oS32tib5{$yUdE-wSQ8zCwqZFm=k|m;_;z;jahO=PwB%)`cluT2IkFYb zD^G^})~&Cl2kGZT<}9a#71<=^|}=OtR2ImX}dQSEN=nYE-j0xnh!*mC*eZToydZ zpIeqLWQd3T73N`pw-W|?5`qSRIhfNAxW0`fG~Ujj*C#c1jVIh?bWPw9$<1hxbTh+@ zZRV3Scbg%so_H+Kmh;-Cz@ajpG%ChWXf?Gig>^c9Rztex%`25@)|5Fx@{Hmze%XNc zyNEZ*aF~ks8<^V*kr3C6z9{{E81MHh-r9=Wc)qCAzeh3wlgagft29JTOHQhSYljin zQGLg-S-zMfX+1DTYUpAv{9)J)Qn7GyT&*TC&!9VwKPJBf>x`7qrkNhM0d+SF|8uNF3ati#;A2BiN$`!Oib$rx@i87 zI+COKK|(HNbw!W75wqDmYk}}{8HaW30A;fiV0Qkc|#8te|@zvJglT@{}O2nW{^Io)>B=5T!0*{+@y42goQ4zhq z!BoeT;$lV~H)Es+2Wx-n6^Z|xBfvjt&Q?6;_FHs^BLlORr%2?MRxLpNfbDMc}N!X1BP((%zauA|kRn zn64`HFS3qT3o=q$^Ln^PN>O4H5`YKr8Gxrf+-*eDdI#%bm}s;nNhOKxJMwxzB4}YR zpa?!V_gH81@SSMPJEY|2a|%beAzFv-Wjl1t++FTQ$kZsv=13-EJwMjWnl4nE`LIwQ z>~BWIjLbFJ{yBQsTF%6v&w=z(s3?}Mow+-RRvv3{6igkF-4DiZVn%wEf6!aSvpjo7 z2l%@S-<&QZCo<&8?}`Sl`!^67hf@{Em7^x4ojWiW#dV(YHbuNNFK!MiKdhzGr*}?v zrpoS3)s({=hNQQ8qSF@NMXX@L9rT7)6*wUt%tEhv+$4$3LTP(2Y3#RXVh~F~L4gqJ z5X+R>sXkp@tM3&*beuM?;9IxONFRnuM3U>f5qtA=QTn^DDH3mb$yNld=Vg1bK0G?p zTkt72@no^`xEe$8csbLF#ggSj%Q&Yp-H5~CPr(yGT#s?+K2(`HXkVk78CnAwpk3kFn40Y=9l+A#(BhslMVs&-s+QFn~dH`V>Nv#Xn=tzC~y zwFw+d*yGo!SDKGr;N=(V6C4}GinC&A?aru8<8kvg_r3EA&N`cYtd^HmQ4Ha4)s+DL zG>v)sr!d+v=f2=N=T+F!r0MP{RdKqFQoz}|arU}Nd0Y4VJdeDE=Wl5!C|?6fAK`w2 z=?=4jf_$fp%+cOnL|Vj0jAqD|h)HAGjp=r7jKD7+FFWM-T|6PZ3$FX%)o-$7pAYIB zA8CXsG$VcrKXV{w_eyQ*Vkpyj)4+Lbdt;mp{4>1Py$L$$`8l~~d6ZYlh#pQGu zWX2(pF&FHRStj-NwtP~y!GO?{2hp&~LZ%i*%X^oz>EedLaj92!U*AY=&t(;J-nzqFs(-YAD3~R(X0p~mi5-mZ+9Z`e z=OwoG^zh8468QwQc>-`yFfe3P4Dec{`2=!(wLg=|l*TNjhhaKtf&gYUEk$*Hy^cMezJ6-&~FejVc5ffrH|0Erp!BC_WHqHD?gu-sC z!MR_Z-7!P~V8#1PCO21fture~FoE7%9Vx+IMo83@y#lc6FsInXyualiAT|Z^$ zN+z}NoeoPTO{LwrPvG1=-0x>3UMlTzvzjW-7s|Wp+!s;n%sVVP=e&fZfDjRNbn_*fTzXtWzmV+6@I~H;0v2-VyNTXo5+F6P#r$ascqeT3C!f5Y?J4F!c7Jzz z1A{D9?BIdN`212$BIcGkqsea~b+pjOXGV{w5rU>7hcTMtR(&N7sFd%*xA80;mi!rreuD{ip>3^7zQ?qYJ97bwww z0RlB}F;wyoopf|n-sr^|TZfbdqwIz4C|C&A%MIq#gVNXjrFf1yQybd%AcutM`z^D+WU)?u_Bg<8B_X$NMA?hkH+K zS1^9P!IJ8jhe8VUDo8)XK^4~UxLk5pwT&=>s!clS#xk^+xr-wi>xO>-gr5UxTOLtT zGW!fCja>M2eB|)_fdPo9G$3i+BOg(u;dB)-jhoT!h$+uIj> z%O5I6+1OvX?F)8e!9;uE2xx``P+lc$!5FYZRhhRzx_hco+Fp;O4-t2$_@3w37h40r zer<~dNKtqZp7}xd?W16W0HtYhB+@MPHl~_#$J!+sRVi^4J@^f1exzP`*b{Ln^hF{i z6**e42k%^c5~IcEa^3l)g*$_i_tFbWRdhM*DWP$k^m#2e)NhJEm=gCY8NWBU-wGp+ zP8Y83c8!pWYgu6$<#8ARoPNQ zm^&Xjo&CZqQ!}V{AXaTeDW7 zRU6elO1vCWUhOR~3`{OJ+v7_~ArvSTh8_$V^)3==hciSZuJupmKl~AjUeF;;o*ccm zTXEJtMGTRXFjt~Q%)Kl`-dmGulA?M;gB$65FkGE=-n;wGk3g8j*_X%-3$r~K#_RP0 zUdXk4#+z(75tRuI1=V#mKL`YWx}g|zxJ&{{g)&&Y-nR&8MN!Aw#jHYc+!J++Vl!He zUh3W$Sr+|S1xodBL5K%IsqP4!i;vIz-A-S@7`SD7{|3>tKzh zf~4_}bjQ9#?b~N6DR1tv?W+u5|F;7-YOL8FAaeC_Dm~cTN(^x+;24&$KiXmk!m!W7 z*wX@eaQkPS@z&(GB-vv*UXBt^m-eVMk;?my5EVuP9}qxoXV!_Vty6;F1<-!!1dJ!4S*5F64sU8+Ud^~GiK)FVto>Px>9=r1i11PB9*(?D6@b>EB$uY*) zej!vSG<-_7MqYaxvb_MHR4gZz-)egpzA^~JdcHA28KUh?4YU%N-$fF?-5X#8O%U@6 zk65V6C|<{sKHn23u@q#kTg+8qhaD(_7*aTnq_?^%2PY?6dtpS#5yR>q)LF+oqJzree@JD9qO{d??S*R!x%yI+;JkxvC$bnaGW;;XdM^H80?$eUVS3ns_+#zUk zGIhsizx3S0DhtXD{&hXP;7&(8Py^`_n3LP&H=IM*nk4oP=pFY|J5qF@CTuwCKuajvs z*@2z5PU8~=UxTiW-%OrHJF+{U^0oz#!^#w^u}`1B3Sjerf%(U%AR^CEC`K|Q6s5jg zUSD^juZ9b4i{;WEr(2IV+uPeXMMW^A9Dx7DzjH_hn$0-E2&aK7%j%T%p8_Iy9wh-L z^wFU&_7pP~bAc_~i)**|yfQs4d`+*d+KyKh)9QaLclkZe&uxZM?zq$L?sT@?SNGYh zj7BtlFART?`x~*k#iGK2ZfD*UNM?i`!N85{lB2|z&HeCvj97nWecZf^2--aTd8u$< z?9Kbk1I=o^+UaiH5(-2*g>JkdnSM7+I|`CfM?mr;A!3)MFJZd9uE{YhkGTN-ds za@89=e1Wzsd-1Cgf`@U})h%k3YOWr4_a-)LKTNxzK?o!5pk&aP6@r8YL?*9~$Aw4z zj=1;0&h%Azy)MzTlrQXGepjnBP^k0_Jz2A}SS^R}aI0;GNDJ%qq~v0$WlSu)*sNbD z!eL~FhI)^7#jn1Gty_Eb>jeDx5{k$9`4KTy=0Qg~^ktn}qtz9XXyr~IKeH)o5^w$; zX?-C)6M*|&xeMqL?>rKbxvMxM6r(?6U*mQ-zGcay%)}^&p_%dAYFyp$)_xrkJob3@ zxNe-bTCLAV9r+OY;0W5#`b6W_@`PZLakR86dDi?nLU5(iXtDkII)YrswSk~owbmB0 zq4|o1El|_d_U6fO+Z(nSq6Khcz3KC*3iT@VpGYMQ>M@GhsxSx4IrccWh#U@Y(aM4` zh+J2wId9D8ZxtN#_9OaA)mq`3W2m;3weKCflviJR9{7WPT+n$e9ObHoU(7jZJ~jt_ zHm@1-r?A;; z^(JB;n0ZSkejZBvR;oh`QBhq}-v=$lazk!~%1t zaAk6w`GiNIc}D&8atr4<1i!jG=q6*mUD0dpjR)A)l_o}?wU`U1@m$qoIA#kmsVg#P zb=2}oF}i-RRAV_5tv+|VgA&w!)d+TU-^V>mu?YUvK`W;Hyce?SQh*yTdeKySx&k(d z!(|7$9P4qCyJbvEtSueXBrZ=H_l8`_gZO5>tWB3AFWMadg)NLAT%HI1FGZuv=53E+ zr0&>^yEMvf>SuiPkMpgs%$kGZH`(Yk>VQFZ=W~WizthF%vr_)K3Qv3le9k~6$wTsa zup@34jb6wC-z01A2a&;6JTX#_)9CclhMl4M!Ly062nUXTG>zy@)1nlAR2q>H5Q(E` z;fQ#%nc-+@DCjC238YVo80~)xl-vN~(I2Fs8b3X+(+{My*w-BmuJ-+PHWw1Cjkgm!33i|Q_;wHn<#A1HGb&9WrTIUu-v(!fh zYMNj$k4zjQ#(RB!Eidfn^P@rpYosZPBvKR@KjpYU@U^wSea$YYv>hX1%K0vj?pN%T z7y5`L-|HO`Zi&5d`rUr5c%VX3{9eQuo1_;w6_oGZCs7B8{yim)31U4^IcDHE@813G zH;}(yM1O; zK?dJS5g=%Rw&`cpHg-A~`Z(9J^QC2!avS#DU;N-_*w!#NEqqtZw<(d_L4V8td+s-N z1q#Xbg#eZ+rJ15ks_5~{yo^gU*!8j2$b)e z9pP_d<{ZU;KYoK^|Ng>vK}tZ43QA7|d^6$v_td|C_}5#$UHp7zgy4U7pWx>wB!7cX z%{FqD|6b}^H&SHf|DFBJ11RCW$J5tZ;AGH(d?R1}8R37}}s8|GcgFPt&%=|9+7m!N=k~#rpd=QUDrwc7a+hrZw+dllv*s33#Lg z|7ZHP5Mtnh(Fb{RQ#2RI|Id&AEd8rD=!5_v5;Z986hR`Pr<%j*(kDSkhu$CU+~;=b z;RVW#FjR5qYNaN=(cj)C_r3eNrc<8rZQRdkq`RA4DW6KU1nJ?*Bt#d?UB~4|NvV3V zUFZOX1|6-Nv$cOFSZovC1nl#plR#xp_4?5kYPQ|i@;8~+T%m!S=R&{mfLKs>&eD^s zJ&?@7QW8mJYPb*iJYwza#i~5JXB6uoK!67=I{vTi_>Ja^u6I8eP05KiiWvgiJG%~8 z<>__SZV3eNx`VZPyn>(&-=io?TX(d@H?+046Pk|4EWDrFh7Vtq!2KWRMUSI7w#AX@ zA}vDUT-Fk0ct~fet{eEuvq$OlwME*ME>^+{r}L%qI#1h1Jg8q5NCk$I4Qvg@itTi35zy`!tm5%cn1?WG}$XZIq2FY-NC^hL>E(LnqCe3-tpL&s2QK|R(` z|A`KZiOm&5{VA*Ya<}b$XWe71G~4t&mFyCoA`()4cBrjv*#3Bgx6XT(>{1N$e;uMb zD6~~%iB>L7_>rn#Lp9~%Y*)|j{(d-qt1J3tURd1xZx)R$8KB}2ursc~rN_HDx1~{E z$`ELm?V24>3cBgO4AJ-De`hZ%qj4iJdA{hkadN8QOxMN0xSr-rI6sx{OQlTKOeqZp z6Co10>OPUQkEAs2+PApudp{$BJaDq?AjA{j<3nRY>ysn;VLTGgo1?_zjvA{~^IgPa z)5R=b7J)b##jg&iaGg*nBq0Afr*}(=LirKS&v5Zc4++S;D3z$N@hsu+^YiO=U{4k2 zadzo``TA}skwZ`h{VB+UyAhp$Y#xu3I9s&Q`1Ei%c_|$oBp`>=84ypaj?>&wQx3cP z>GaI#*@f53Z_)WAkepO7$RUA#Ut~XFQ5H1&T?yqkvpq z^X?8X5qeAX5j*P~Oy%`Ee~~9J=*BcWS=QvjbI*4IY|YxKOz z2K`wORH|u=!it#Q_|+E>US99UAZN4Lt`SRN;GU6QUT!LGc28Gp2N~uf@p}oyR}T8rOL!)X00G#n zR*Sjh@iYBfO^jo`p>C{Xc?7WCfJUr;7R~48P<1nh!4rku5 z$=)53QmjN4xX!Bqt#{8u1wGf&OV5Y-`tsy1T(X!*Qk}24k_mW5LjrRP4siQ7{dMLh#Rh9*&RXmdjZ02sk(EC?T#;? zRXaiT7!lt!dAtH06gX5Dx$mHI84<8zbvJJ0L(D?Dw~`yHSyMdg57~1JD(K24qfn*6 z=y8Kn!LAUs@gB%>FhEV^@;i3%Td{QA7i-Nz2R@67V9+9ekN26J;5Uytho91j^mR1CR10R;??hW75q;nN{V= zn%&{pN3Yv^btisw8ZF=2TYdAnnd8TjUakVw<}c1pC&jPLcsu%RgUVJF;zWOSR0HYm zYH(GiLXMIcKOH`$t45p9f_t-99!}G&oumc9KTCuF!B%j9z``)!RC2aWS-^ zO^rF-@wBmI#amtq-U&O@nW(eAYMgR3fMX&4^|`AP+r8Z82A{r<_&YF?iBf@o8Y1Y2 zu-}O&)mJgOUFY;a_8ZZz)7wRVvtt1Q_D^JTHRL?cb`A-c!M5WwDf5*XZn1VIDgZ36 zmlIE%xfY~qvjuDn>}rqx69uBxd#o3h)5M`9VxqE?g&(hZuR-*@-oHxC^Ypvc(wtm{ zpSkdWEPpjDN(7Q9CU{Ql`S%Q&{T__#K^M-%fZe;@{dKG1iQ2G2|)S#5$u9 zbq#F90HaG+@7?C<{+<4l!}G|+Z(^AC_I40vEQihpNzStcb?q$Y*V=3f5{V{Vj0Ucd zh2WntAKn}gw=Xy_wHMMKs5^}!t$lVMY`I?t6jXd~0&WX=6d!~6Wo$9)MyZf>*il4m z$V>ZEqW)N`H7ROe`k))l$Q`+AIdNH#+NA-o#`sTPK!@k%j%7^I5>{|TT2-E8G|6Wy z<^tc9<;4t#!YnlqgPhw$8u)r!n?z5OzXh!!Lc~Yp8*SRr5WS?a$6wm;PZH;I<~~qqk7Z*!l21a z9ooT6QAy{;&IqvL>>rkrLfG$6Z*MHps&zb`FT#1Y8u_wJXMQnivQU7QM3KD{P_Dh% zSsP3T13)4^4F(130@6r9Ffo#J?qQ-=p8k0#cZ-+zk#Ex*|5i4VEshdyr1L z8}CJHDBlE>H7KUlO>7L4*AW7AV`B%4S=8QJ+Rj6whxyM-nUeY?>Nm6tRTdfXQ*%N= z7hC<&V6Trh;Ao@wCXQ$I&_tcSYsTDO{F`>MtMG0*MwF_qA)g&W}NYhI_&23-#BuBc% zoAxAuYnhCW)4@@tTthC@RHS2m*EcXGo|^k9m86n8^A@Uny9C#+om=uMdacg%MGNLCe z9UM#*0n0D=R@bDdV1;i98M&{I5PZTGVwAm+yR^vJizuVk&EM&y+V*U)y_U<`3ZKEu z6;prRw>!59wS5Bb4Z%BmrIfES5>|tiflT*&#s*@i>!WUkG`s@YQcx2ajdCMKh}I>c z+z)qAe=t$Jznz{~R<}y^QHigOby}Ybxi4?)F zx>{jCXr!c`4Lakm0UIG|dK&BN2^`I&&yrD5@`gIgQH;+X=L#defg*m@1dT&ibQ~F) zZ*ogY+pts*CLWy47K}h%lM`69VV5!j&tV6?)`SFld&1im$}T?k$8eQy_Y{jk{@IcF zVThQ+2$bR%x0V}9>(e=aMd0D#UUz;VY=Oe@%1gu`QgVmHlx51s6EUHL}(vH3`4K5ZN3MyYu)g*B@98inVEP?u+&+$xF!cWeue@v6biZiI1h5gkSk7| zpiUZ73(xWZh_Y@I=ZVqYBOD)80HI8c1AUeCP|3bi$KUxzgaM+}aXmIGR)o7ZMMEj0b}7 zCyaV~BatY-dmY(}CjCVcj^=-C=F<{EBE297&U`hv$8G|3MY?x#07xhAr|vPdUpViH z>KDQ`0Jhz|E(x*numN+iVq2@Xx4ENzewgr0N#d}{A&+r*2_2mK$Lo>iM~<}9>v2dz zY(zxo9Pwn1F5}k0RLVxp4`C@FAinP&i~NTV6EiZ3$zp}C<}r!1w$8s$MB%41_1`cf z<_zk^()tE%xAQ(9%w7|1^T3~|wJP^khH-!0?(8N$H!YSZ6 zLL?7A1r}G+UOf`0;^3BB9!t!Pq&PS^bv#-l_}ypOr^;HgWl3h|aK!Mw=POxX==5x) z8z0V|^_@WePW@FEv_+h2y@8BVW5(E*MYxAw{jH&7!|<>zqW27p#VBgFY*nBfX4qzH zkG!tFeq&T}~(QZ7+K)>7IwxorkyT+>YZA+#jyUwef#u_eyUT3`3f_csy;X zrnZ=;M_>Lhcq^XT5hrDwL>im>y=K+I?cGHlkPru#Lgc{>HRUwJGq_Kcy`1c z+ko^&^+kS#>KY-y9Ly`#V&I1@8JiD=Kgf)?vBz|FIccO_flC#7diRH@k{RCVQhl*R z$woRD5H0sl@Sa1%z|urrB?lHQYe|Y9W3cl2QPlKYlM$ANre?`G*P8g?E-eC!sMo`` z3G;#Ft5jMFd*#Mddu(04a5gQDc5q|7^Tu;6Bx7+Z_T?=TXX{^-@YC0)TaQ1gRB6co>OKP#)#leJ&((BT?w_LK0~fI3Kdz;xVAc5KFS5RQfmv9FI{af^FLBc@~) z`02BHhVG!3TU=haLh=elnZmYGfkH7+-jGJlAnth&I)(7do4T2Sf0`mNEuW)BlrZ~~ z)ABb~2msUJHRq3s*S2;}42b;1$0yB@O5P3#Mbn2x@f&p1pWAOVgUth@ywrEMwtoCD zDy3B~2g&xj?`5g~Yp0(+`od)@?aGX3);TcCPEluxiPpX0^Y_FZnBOtJp;Z#&$O)Er z4HNh@SLFy;-Y?T`HQc4u$b)9FSTIFLOWW<2$KN7XLlg;5tq57E(8s?(D~>?PADIsf z-qr2(8as;^&mBEn7b4B2D=Y?%-yic~HPC#q3Jfx0(QRAh&D#;YIrg{Az!&;!0e{8! z*5wOBQlx_ zO%CwJBU+*5jgF5el~{vO?~DG3w8n`mI5CPdP1iBYp;J`&M(`f&qf* z`)N!A`8rS{8UHPRz?VMFp zKo#M0fC9z{EGB^@t5;urC@820Xg!&x#o9O{)dr9^3PcU|F0o@4W#+7$o{48=SV;{dZQjq4&{GrpdLB$zgTvp zAfR}i1W|8U%Wr~S_% z|7ZL+Lb@YHoS+^4_lT4fnQ95a#r8;4N#SgvEV8Seq4bPpOb>AXkyyb~aB#$@qC$-|k-xe6!zvc;Up{Ylqi)-4CD^$G->e|l&4MWH~0bjjvbA*ZNa zTjC_mh$Ihw7-CN*l<}uh^d(YP?uXfEm&%zEpAH<}`Q6)RI##;T!3jQY>TRyA`8ZKy z`&6*g3`8>o@G(J)Rv3J1TONit6hq?=H|b+KND_kC7CL~sYMV;K6FuHH6#>TWt-)xp zTm}4<;puLgk(tg*d4$Cv?o-=wf%~yKuj^gvo+MuBG7A$8K+-h+GMJtDYz5Dk4YXyu z4)NI{(^S#xi?fTb4k`;wg-ElrESZZ0yW%NtsG@X3=wh=vlUo+xWawbA2-OMc;2#~W zU#AEx(bE+vQlMCA2z&(gX266UD1=Ru1Vdqc_=gMF2lhJzUhT{EQUDSQMccUwl#46& zW8X}xKgK7UK~#=MIB!d6+F{3Xl4~QJAuMf~gGD6$q*PY1e}*-t=q;tK%(`h^Dyl~(RUcsN4C-i8qDH+P-<$lv-$I{Tqf z7Jl2BJgHyggB(4M&w1EfO_?t9t7c21jZD6by2*? ztrYbbz&#lXQc|seKN;93lBwF6gmGbK4D4&366?_`_W@%DHY<-eWW7NnH#m7+Ts78j zooe{_Q7J!B`6ENVG8l;lX9enOr8OqQUTlqk zU|DQtU^xjzCN?$}YpUp7)-^U39?B4uD~caP!$Eyb;QuX9=DN8sSN@139FZEN{gMC! zY_C=z*QU@2Rb@2(HiDzt(S;cV5*G;d!d?`$1C_P5wS_7hLo-QtB+_KRI}`*erW-~l zFB(t?73`v4Z7*|;uv`Xg4mTxccJK|-ZEGjo5Et2muz0EG;9R@|68kv>px>Jc%V>}z z1cF)6N`BqmqIw1p2vrW^?dqh+pDR1y6u3O-63$G=`=HM*b%-0eSs)4IN7L~BEZM6m}RrJ zHBbL(%e60s9jxFo5F(OM9j#gHe04wM>=Iv|&U68Ed6X)FHa?D&n}b^;&gM@EXCR`j_>S2u7%%$J zju3T~pDLJoIFtH#r`mroIBONaUM_@b-w9syVZf#nDjj6e6l1-&utfI-m>eD_$kVcf zo?TY*1R=|&fIy0OIf3&S!*8l;8Iy`MUVfS4ogq?)bP41Oh5F{f+Eb8V{F8`dBf@00 zU4O1ZKV53$a2L}1^~2UktmLC5D|O6zg~fhWUpg!4id!xqyg~(z$y5f+x^)53I(*L` zcnyF;Xaw^~Ia^E>?qC)bH~3VcEJ&GE*<6EXs}#z7&-ge9*n>k`A5y)T(5yr{ue21m z-%CT$<`3UxnX)U!`OAgKw7xjv=tR_Gb!G*ki+09wAerv-(!5|^&+MV2EF-%`#&wx+ z!3n@htucxN13C;ja9^`nF>~&q3!^(9tG;AT_s~VnN-_4P%LQ?<)dPQoQf%LccvUD) z>#5cPUO8U##jLcre50oNe7tUsG2Q}e5e0xoT<`)g(0tbTn8R&0sp)u97+ukn2XC&n zZ5c5_UG*Yde>e!^hPwEBsSdskIGnyF7@XX zk$u1Cu>vb*&k?GD51A-^ZyjQ&LO#!J&5+@yM1fN01!i>aicHuIn9=Bi;qy$#)w*%#WLRU!?fUMaV?ekYL9?@ZqIc%v&mTKv0`@0bhmn z2E|N?x>w#zg!5Z|P1y8hCZG&GU9f!?-3GW%d8*t^JXN(k&gh1$nVvzI+}z8KczMR2 zl|{zCqN7pVoGrtm29!DuRX75`Cb5Hj#kR#ahLIj&9G`)78l6}Q(hqu&#}$*%N)g^p zVDERiQ73|BfI%6syhmv`#_t@tU+XwHYti1`NoxDDp=bjhY?~{N3c|Tv31okv<#+t$ zJJgvAmn z1}s@qWFGt)*UvwEP^7}Airk@6$sIk-4PX%Q7Z}B0fCEEidWx##u~&TP_-L2y7se;p z#|s$(hNFNh*kn_sqJt51GgGRH4D3IL5LxPd=0pT`v`>o@vRbYNTXG_--K4s^ZKeFI zZz{0s2J!?BxIL0du*%BDRaSTtXq}*l#ANss`kkv3C*fGjSu*tw_Z%&chM+7K^I;TX zCP8VAU3T4Xtza%tGCvDTA?ONpUhW=S*&Mb|lbDNgKO+q!H<-!jC1ODvFEJBe7Lg^| zh=eVKr$41DQF-6n7sVN7syiGU2_12|Xh32I$$cG&Z39;37DmGw8NMkg=mLv=79E)A zY2v91DC*8fy|iXkDKIvdB-IWTaQhP(`isJB-s>=4@^+51D?}y=0E!{!lRul*1t27; zDwa4@F~J9X&$3bC*&Hj$G3yRTyp^$XxIaj)i}ith_IEF{KiTY_;7phlzi+?Y2SF7e zusl#$Be+h2=R9wm*eqMED>c-UNJ^&`7ghkMp<&0;bbaP{DbgQSkiZ2HMz67X*gV zT0wn;DJjY!ZWj&M@kv;DgK(qt%wXDl? z%r^eqzAe14pwgt7?y~0Ks;KL|pOp=x7A<+5-QAcgD-qQQjUut?`9>c-@vO+Kohg?c zs~AS`cpdS8J+&vpsEyDV5AXQ4J+>u zL*mLHjngVLgjwXl6xb__+f(i-+l|3DG4RqV*;K9=*K&$_(>IMx_34iho3wy^ZvM=N zPJ6R-iHv%V^8;mcmN#<1i&5a1j5$JsB#sW_CGuq&s`QjwO~C672R|i3nWlQO(j-?_ zq|a`<>NA>iJjGjRCpCG|jfXVzNX-fj9wJUBO8{ku-r_TQQ+q$_Jg!OVQncSLV0A_M z5C`pVeyvYFxU(}F-lk-=*{e4$|A{L<6$+5EaLBt-Z{0__Hdx*tT`cb*JGA8&b6cs$!BJ`HRT<~}3A=5fUYO!DPxqfOt9o(=l$?qOrp+ekM8%U3F$1wK55Eqgnn@ptkE zMF;}(I5^RK2;I1{g#cjTgFy6|)i5{xH`voL(Qof*f z*ZB&`g%2Mft>{^3lPB*lAu*zQ8WHom*O&N|RQiwSEr0GBM3&n2L}>WLa=p7Ywp247 zH+JT(`(oXaM`s*&0+Xk1jA55FD7hlgn$h{!T4n{Gi+)y$jbN z4|~XZI)OW@!Ct91-C)TP0s!!qkrSEGPuos|mR=KkZmb zu?lpv7;-~=8ZM!8&Fy9$6{h#$WEv zRJwY?`Rl6Yp|r7>f{5$csTv0HC3~cw-|**1>EP~XcifEm|OW`|6P4e)y# zWUkH;aPlaJH+grv)h{nxT*vnZr?itI+3z^?zLjL8-Is_oQVdgTo|F|1Y@g_krfX_m zeC_!TTB?};Rb(I`exXx7eiU&RM_yPaq#FbgpL-@~P455`7;t7jonD#$E0%1Sr~KrW3Vl})cneKxlSR80It>ICK&68 za>#NlS9tWwHrM-TwO(<(U}K6;$wP?5{npXsVCyt7C_A^m6_LQ|t)S@mN9BsDXDE>& zL$erSA(mVbRe{Dd)7WdIZ7VIj=iA9X4i42ClO;{^$y@_+kwV2Os(gkVh@l#uy>-E* zEq-D)5`sdH0By3=C0H)gYWNn24mGXHj@jwKHK&anc^nf9!qFlyWpx;{oQ`6_?xf=* zGT|U@lZ1$!>fBlvv}QK@+!PZGIym&HAT%oFJ72a@zEfCx?jrnw!ZhMXM#-#I@sELg zFkja)AT-<@ED&hgPv-KF)rw`6I-$_8aM)DqjVNJ0_xWpSs{4wI!4z*M=uni!d~T`o z=sqYdipFWO2tGZM%tuX;T)Z{=$XsoFw#WLJ+rEwHik-}T=ntt_CUXRNJ|4yP;FZ0! z>nO-qJdtg-4+hCbD_PSzyDjedXc+8Oq!a;{E8>e+c2vM5`{K*m!A6$$_E@X5`yV2U zMlR|G>g@>ur*O;3$A>%B<|CJjYot7{?%a8=fk+ccq!AEKKUS+1{(A-?{bc43|KbAL z=V>Ao^RGH`i*0Ni#tIK-3$v~j`m^)JV%pM-tHlYMuE@KZi~Rq=WqGg7$7MXwf}~e-#wgmO{tnFksa>)Z4NdYB-z*= zfvH5Mjx(ra9i)q)yTWXW*3r@&frXQw?!UF9uCThmzh9feJrc1IkiS~?<6A2{9E<_| z0=MUr!YMES$6K+MsDI2WR;+Fc-|X!(u`twh@L2#hp<^30d&Jr*eB9?9Q1L{na0T8g z%=LE`?yU(M%aT1iNRK3I#2w7@or24uVxbDcqO%9t_Pl54E*brR1(q1c$es9dH7~wo z+6zMIiPTOagGo3{=FD9<;HSn+p)x2HPg2bw%+c1ltVzo0{PJtX{_<{LzPRl4|WBS=bs|6J73hE}5~%lO_lZ*>m3@|wwnmRT(W zMd%N?e8mqu{xt?+uRh($8E)hpI9|3K7~L=UTrS`tpEy(*KS~c9`Bj;LCyV#GS{#}X zINCW?K^P7t%(xwuz*XU3u7~{CzuVk>BugL=wH&&X=W}u81~vX!@$8^aI@G}{|1vq) zpeWbKyOaIl(Rje)iOUFPhgLP@*Y+=>B4qDIfj7#O&kv;!*jQ6xIG7VpCU5loe9C2x zixV?pbF`%zs(5i4yh`FLE`v1=4_dJSj}>Q`DV7}(?3Gf@9*m*N(as;Q zw?vTvelr*1`$XPNK%ju=Ha(mmtuz-<@(cda8;3t!?pHQF z(cwRI1cfY9^xC$$9}#hs#f+KRIC!G-kqx4C-nb%Ykddv-lsnz4uc4T%VB$&^p|*n~ zDa}x2BlL`NJhH0{&xkQz)%v4^oVfNEL)}IW0q!GqL&=p;h40`JU%akd~U$$SA)eo&&nb0pj5`=OuRs=B+yXC1t0 zvu4iJA)oO4F-N~sYVhxE99cjFHjd^SEXn4&ho^HPZj!zR2a9Q81^G6Mk;1YHg8NQc zSDHUNfp$zpzff z=jI0oYwbMs-l@8gvuLmPJki@qdwKw`WL6xqoK(3&47G>44px^*KI#?feJmCF4VXsh zZU<&XZHF~{*N3LC8JoomXw=$>BT4zqKEeW(7}AK&&zjH=n4`1Q=Ls~DSkjuKKaA#i zm4dVpX6UIkISlW%(njduQWS%vFWofjTomCkSim32L(xgW^#ll1vVVRL-xXcVmba&d z0v161H0CSK;)%@jY{a)hdvWF&{j&!o+HR%Ms3IY|VYX*e$3dDobxdAqLVryyx8gtX zqz~;+5;i{&P6ITFC$4IuRwD5Id6$5R-5kWBJ`A{&?#W6$I|C(1Zv-3M-X!vTEImKz zpLvZ2JFr059Yw!!ObtEFCQ%S`3&fEJ%o*`8v${h3a(}aA|LbNe9M829;>)LyILk4{ zK#CahvI+Jqk^DFemW<+RszoqkQe$rSUzGy5RA`z!o6XJXcBH^7Oupa52=$Bq#|7+-5t`1ba%IO$Dvacld2BU>B09|AxHW}oc(O77}E0}R~ZDj^j5cHpoan8)7<2@g@J735h&KOJNkluSIC zlEtRLCy4+;&c#N8ty4qTgBFJzT5-}AcBD=Bz6QV?$jXh;{?+w_Ywi$I9xS`wTSdR)NdEQ>ncHKf0<5Oh>RndJQg>`r=jCxs zY8Z)TR_QbJ!_lM!Su<}OE^_=-t z?>S44H(4A6?dI^o>q8Aa-ttB<4>v<2N~t|SS&!gUbGCa`ly4E9$YA|ddD-oxz#H*~ zw4^}bEvePJP8rs;-A}+JB!F&A6h$nz>b>lKZh?0sJ*9-$b4b(q98GgWZ+}5oj937K zYRS3##B|?I(@tOh)%~q$P1vgR+cVnnu6esPKM(F_S+r`l-%M*&x0Y&QIBe(o^!Glb zJA#dx5RV+3{O&O&zpt6S)g#}!TEPvN;dY+A0+eN73F1W#W<7Aal==L}KEBN11>CUl zgA7k=ak(nn1$3<%c$a>am-i{<@`(NoIQ(Tv9A*1Fu5Vi$L+x|N#V`Hg(b`VNe6D6u z=(MU40g-}nrOY=*%LiW^aWO}uOgB%DP`uG}7I|lcI${IVn~U!5VHi_%IRI|PbMdue zl;@haPxV^d#Cr0*%$9OUbn!=F2tFeqBgd%%XNw;J0}aDP_Vi_}lE&|Qi^%Kk30k?F zPG4F~<@r*aS#B}Sp};$B4aHN!OJDWz#WDoU)eBmBsguqnEUzwKPyd`Ds30N`%Jp+9L+52mq`Th@Fm9TZJyul?tiGjS(++^oIA3b-qWz*d? znb=)rVb(`ulW;Jl+K4N+=Nd% zB1zc79j7#4muh#V$8&GrgPU&948W--c#cCS30?C^ilHk*r*m+cE$h#Odew4xJV*@h z;+7os@y4F!T^}VFI}oR4`o&ce%%BnFBRf|K-LhT@G{Gib?Gj;dqzaM;UMci&(C--h zOt+ETTr3XWq%L+yBqGFfZm+=Yb)g~R^M>PVsDW)sYMa>`+hj0)35i3onlfg{1iNpI zWwIy4Jzv2r>?aJVaC{h4hvjd-v?G0Vh1*;>)iPv);+<+W|Rh z%pG7hgCYe;&>f-UfcU*hn0ybBE8ovrfd`Bdq*!GSbA&av-Mg8zIIRa2;*i?+3D0NGz5YBsN{c(wgEpgdWP>qkMrsdfuZW6caOiUO zSV%31U^qiMTU)c)Bp7czHCq6$lZ`HrSaglfKGhWP%G@;wR?W28)(Ju4IR$)E=N9aJjQ_A%RokBzLHukW0jSeEaOyOb-%t(D&=E{3P6xBuWdbiWke{)*9d zUUATWanC}C5|$Zj(`&?yN>R>BG*+xxE7NBK9`lrWv6^tPlG&xH=CgH&b3x2oWl4FW zaKKjp_szB6?=HKLW5*${{TLSu=dyp6({zPKrBI>v8I zQuj7+AV!B~{P-jkDp_PO{FTEbg=v)?$9kn3Cu5eTgCgKNy&!?YBmDk$XzJi9^z)Z8 zH%0Q_2N-5?ifE+7#w3#B|9~c5K{14`7Uj0KE@!(U|HizQk+;tHY1ddm2&AvJ(IdX* zyo5k6sMR$8fB1a};l3Y#BUON-x<2GN{bdJkpvuRVz7;gn=eGP43qXkQ<@(<@kLXfG z#CPTqfV#j3gDhp-l9^dW!RGV;7OAz7d4O{JcNL$W7$>*1_=FTf>R1hl5_)XrJD`p$S{WSg+&~87M~HvRyoU@ z2tGeMgAw!nMk5VjRRh$4P~Ygxr}y{wn)x9z3wO7-Yt>lgP?TvWGJ~FfJC_?>q4K81t(F*PKI zw5X2xD)Dz25x&9Eq_=l>sapIM$Z@dcP=e(zGEs2~p_M5%vQfdiTX~+x zkK2Vf5v&>OT3`coEDAF>eNdcG)Kk7gcLY;PBm%;3DNi%*zowqdk(b}1j8U{ExxL+Alq*)G*BgDR+ z_Yuib9+yb@b0%b%`~{umyDetu7%Pv(YxAqiVXd{*lMgWv=xw7NfTemB8xcUimVA=dWlIr6~ z@)xw>f!)Wieu4YPMe?I_6zlU1K6{4HmMDe+ZABFGtr0>k8deogGLRl5AoV9h^1kfG z=asNP!{$D$pVuNEro@|lbkItiPYvS?97^Ykr?-y*NteIPQU9__|qj`=; z$$*9@WNV9o%~A|{a#KIPoj2!3O(^%YqmMW=YFuHaC1((ikuvh_<1)1u_dSke39b>E zMKDrwDngFpeJwD(() zocoInrE8g)-W^YO(78i@FMb0=GAjgQjP=ZhsPKz@K0zzd8Y4}kRR7IiZ`^DqNpg(E z-62*<&=Vuw%8+1NyfaQ2q1Do0-20u(8KBsR5 zP~m95rKJhjEHnpGh7-4sO(1S7es7!Tknk~&>$6AJ7$Kb7c@5bKK0k%(CFmuB4}Qup zC@a3RUGP4d)iaI11Aw(Judd?PjJrNN0{A7zN2r%2B=p28I#k3!|Kx$GE9tof#yre{4 zgVZJjsg3(tPs0Tf+16+0wbADYap7zciz5}0XD_c@S%vnIR_Kz~iR#Q_dA=y{JDfi^ z=GYOP5vQEm{q9ryT2JI6%H!2z{e;HDeC!D_DfBc=3>m*f_=D(THZ$g77>|k4t+K2c z)%o4G`K`5kTe#T8JB>TC=pgg)(F_Lrj;lg;>VG~%YzE}RH(^2KsF4gZ^B0Uz`GAiL z{pX9vU;X|Naf>L96paZ5IDUnF)S2{NNwOi~j@E1za!(2oCe3FAC7a z+13c*|CSVyj3BP{_v_$5EN7HYq|3-fQT@o2=7T)7$V3-}S z|JyvjHCe&gKZgQ{5BTmK@YI(rLvfXV+Uxc_V)DPgdF%s0bztm}``*2c;(*G3^pgM2 zkO05=J!zQyfQbl@z+TIMmAey2_$uBHelF5rA<>%b|Mk&t9>xK!o9`85(L???gx-e& z8;N2LOF`BUnTILwm(mI0 z#s#s0&BiD4DivIUr9K_I^&0A^Q~saoBRmB6p?=pbxbKJUh$fGnDJ!}V!i@)F2V0FV zq>5dzo@n;_Pw$!&4&;SIq^73!BH;RZprtJzDd}B4g?;^s2_0~G%c)h5AA9x1Q>grp zj}s#wBb`AsC8Qn~#0IvI^Mnyx2<0XKeaD^R zEMSxIk-S0$)69KQ!m#@J6Q-bQp-}+uUF2&kh)(1x@F~rw^<=!gpVs5=i4II4!IaQ? zd=MAdVSM}7k<$v+0AL1+V0M0Re@c@7eOLn6?HCa?5TMpi1o?j|AOO`*bC9UQNywq0 z0jQsRlt?+RJ0lO9p1-CO6rg#CARn0dRw5;=UgxdKA2XRTNW0052r>7MlL@jihQxe< z*L?$a8s9Ti(hNa~A|l$iyRaMU?DiGZh$6(_ih4x&1JFqvDDx6@@hsT9K83_Ao6QWT z#>*0lp%n)A_nt;^$A2vofE<#l79J8QG&<4oG&Pmz@c3LPy^)w@k+CFs`3gZCazgxP zmRmqWoC$@l%o1!+*mSo33@`jphht~S%O3K-I=T#&Y^?)5T~=%%jGGA5lra8!7ES`V z@#2=D;dQ6$>`r(RSM~SG+l9@ORs_m4#Su+bn__ww9^#m%n2zpx1e*o-=eg659n)Kp zBOKy6Krl|=LGW;g`-j$F{Q^0Sd~R`s6eE_xSTghj@XCgsPZ>+O{VCz`6&T~5fOsLU z;y0~mGh8y1cHg8z+SJQyky>qu-(EaFY!*4>Cjd^(x1I>Z-~PElBPxKz{r$7=URjC% z7AE+N0$quZMH8-PJfE+cnS}{Tz%cqlX$G6lUyTBS1|U9~=<&N>1<-WC#udADG)ley z@UYiJ1T`UnjZsQLop5eqP|H>og(jQ?z{tM5e)=w#_Yb+{ca0Gr5#={C_p#Mvfa3sb zNpG~nuMpptkDSU2u~a9Vg|O%x6=>@X0?Mfk$cD$NdML z1nLWLR7!gS;s<*>4QXk`33aig)zbZ!yAe_MSg9DcicBV(czr#WKVwcv!7Jc6|0T{> zS^D1>1sLC8?{O%j80zGobpFPlFvx}e?I*xCct9)uLZTF02u$ zhpexf6q;(~&jY{^yF$VKI*Wlk$A2K94%g?t=s@-#!i?bPtKUtje(We5gl9noIKLwl zS+LNfkhAAU!QRJbmQQNpjLV8P>BWx!Bf><6^4Jhjl{qSuCI1FIndV9q+UEzI-&Yrhbr1mFH%TaUks7>1a$;h*cCe_rW{0nnaDllZ@a#mga2 z88UiFTGdr){5uT)HH;8z`y~aiwiq{illIwo!Ctc=hO_Fbt60+a5G75+4i&@Zb*Y?htEj~7r>ovNojI{>h`nmeD2cNL*Vmy z`JA{`?=P>E;6%Xl3-j`T63QNcAb4p$`x8f_(p^O7q9C@f)9s9akg(kS2zI`y5kr^r zymjYVcffgMS73Lr+OW-)u-RwMv*ED3#XG0njUFI>m1^qdc7Usr15yJo0bBZ!SN$eUi)Z4C`jSw* zNzW38AAR$sY95XAxug~Ey(h7sG3xkptJni#LA;ISYTLg&>~?-|SZS~l678>4_9gKI zPFCmpjCoxhSpqjT?O!?bi|5puG~M<9jfg+nDwzv3qD4f10tunBZEEndb;;I=j+x%L z&XU)kP7Ds>-14+DVo+V2oka_jiIxS_u(7jb_r9?88V|+crp$77G;X)F5#X+UrdbuH zWeMmQzQT#80R18YiM)PrvFm~!9tM_qmZb%vftDP#s7DO*Oe0om>i$=^`l@^ow+-C+ zP6&~X*7hO$POZ`US9)Yo3*%(eW{%WZTN`z(_U*O$MCJE?h-9yfFv{MphsocWd8&6*nzG|tY%D3Ed_lw^1@LYebHYB1HCNUR$TgoU70Fayd1bGE!`WR2VgyVU6L{ z?A^pY9Xp%>G9u)6(s!}cWECp4v5w$T1M-Gg{-Ar`r-$JdJkgcVlC#Z(7d=L0RDS+( zmaRaIFtgcPERg7p`?IF&mhAA~a>VG+ZLseFe@XBZ;bX2C)^k3^9b5Yhu^Jm)z4(dm z!j5$`Dd#DXcSr4N2k8J?%77ul0B38giQY{7Mrhj&0M%7WjTL&}>!>DSp+wMIQg|Lt z>WAQrw=>ANriNK3{Zcbd*qO^rDenldLgRqSOwHsrc(N$O)6Ic~&>L14Y+Z`I^KoPzLPI-fdV^r3?BiSSGTD#Y zzE-4B%zJ;aLR-IIR2!X{DXWwv&#!~s&Q!J32*)ruIBVN*S3Ad(hk1zL4>oCABH1 zen3kuX=u^6C(hm}g@^>7ch{e(il*HZQzT7UXwL@ zuSstLQM+=>`%^UHmAhKF4DD^h-!de?cKqrJy^Ks*2}tg6O7|x906-*|P1-z>O>6qx zr)H}UE)q%LwI&(VYy2>7uGl7lJC-o$w8@~|dGFsT5faN}EFPH*3EIEB?Ys6VPM0 z^W6~sYRnw6pKH4`wvyS5$T50E($B_bWuwbVC{W;v_mq$hHNB!5w1-^0ChxW)e0`iB z3h)bAf`f+bVNbVG$uEz`rkJ+Y87@8gvDH~z=(vJrOH_ot?neYX5g1l|uf?(*f9UD$ z*wGNvbni1(4jf00wRonSyN3@5VYZqj^qjG29T5WngBiC%@HsSQ5Hoe02Xq0{2 zb}_!~OPd7Yzc$d1?A@!^e8-Y3W2EMGtHy56-Pdx7Lx$^@YkOOrR;D#QPXX=>cnPyI z=k|j+hPSW3KTKiHS=U0+<}Vjx-%JwQ;Xa*m{d1iDueYrIJ_k+A;_+u=E(Zn|Ip1C7 zaAV-+aGnh~)p(>DeH#_i1CKp#$@jYiY+W1qdt=#itHg4z1-cL=(eL*F8fSfgplGS# zOkfQe_tJK*B7=YPLXRK+RyMX*I8b3~f1(=Y^1#0~aI5Sf_Q>)g?z?254-O9@w6G@IUC1ov;I+{cK=KT1 z@)x$7Iu2FS9o=)kID<0bP^QvT47C6C6eeY&`flTd6EvOG($dzrojGw;pYcs{szwBc zV5SKt8D|tQQJ|8x0zD4QSIv2l)y^slD6VTb!3AtH&_y2;S&dUCS$YHU^QfH3|U@d^chGT_rHwvBANW zEc>8|u01VIRncya>x7^~TOU~{F=Bcwq|S#Ua=?U^tQyu+XT=0MZZfVeuV4T|Ycl8eMfD%0_MW$(b1$i(m2^MKRz?+1e}^jG8=JQ%~v2bH@DH5zg$-gKZ@Qc6W`;m zwJo^lh*^mM<(jT(=>>e}IKeJ4iE&=?KBAapNcmVhp65nrT$RG@_pUF6p~^Ag8S&d<`5G-0|I^XZ8!V9s)k+uR@QYD}$9N_T`^X zDZUvLcqMi6{#?ow5c~4$X?0k==ej7JMvX2pc7Gba(gSYx*iR>=A~UFAL$Rit-76{P zff6-~GnGyj6F@tFR`iaA=)GY%&gD++(3vkU<;yAy-82b7!rN#@?~Z6nxkwUW?pvOH z9;=xev{j1!$)kBXy=sXYV*b;dDe#96ADHZZis3Nbk^%_L1-wDrkEeDOzB8r!{x0kS@0d-Lw z&oT^}z+0mkxTw48=r5iDozd=2umF%?SiZ@H*EpA$PL*YTGab@-+aBjNk}jf0z(-Uh zr~*jCr)^9aG@aeAw2M(GWRNf3UBa`O-$kj0;3yQ<-$fw?h;%ibtWW`}Y|kckTto`S zhje9|4qBP}^{VhXdO3vq&S6F}5Aea-&*O2>(LZyf1blOA@axw5<~Kn#Q>-KfWO;xA zAl;LRw5n9u7usJ3VE{PNZS7^^QKW&B>7WiE;CU z3HzEPwtS_G&n!+X{d%}zwgGh2xq6U0_uoeebTrr~`^e|pk@#w|)ng?2NGJF zzMcxk)~Af*PZg4otQ>HkyFOjGY5Kw)>OsR7?|AbBkN;$C(2u*Cc1M~`9khEzQ-6y4 z1)ywz%K$es-R$#VKFqi820Vq-DM{`Zg2r3HCLER(fGT5VV1ccvN?MIh;K~@IPy4hz zD~MN(*j|+~1yw!?23}6TH$()G*h-n)6l*+s1Ms6iOf6pZ&Mua(nwklqgwRN3`4nQy zy@N82F#e{tcnKXIn-SeLfHx&nj2A1|)NGz~j~o=~aXb!xCB^X;(A3Z$^r3aq#nW)3 zcXzTxksZ}vK8#FTnW$|R^xyE>nh0anIuk;Z-n!t(b^0_7pLDK2|=(G8niAfwjjmJmifNx&s ztl4bE(|yS9;faq|qcF9IvelNkd@hX(v$NxbcvhnM%^*yj^IhDz_gPCMSA{8}ETQHX zya}4+))kBb8>Ap77as%Ujha#;cI=S3 zZ+vsc%sFO4zaiT&VnngS$GK5qO(Zt8H|vN%DVGv#bIi2OHGMBwjyqh+YLmJY1V44I zf4NLF>7?Dv3+j9;k8QeK3gnys)PoF8zM501R3JDX@bxdNe`lAX_-3)bj9b_ec7J( zh}_)OOre6qAN)|ILsDzCE}GJy_(^fP%#g)KXF7lcreOy+34qLS%X0i)lo6+hfc(Jm z^pSo?K)2OWM?$W;m#YawpwE3uAJrZgsT}b~D*-cG$LYwlX?>xZEtqR=z^K9HH5@)gS_l3$)8`%D z)5tptCyU^9T$4GY8*Sg(i%RkVAXk`RDBMDs{)k1{g17hmHDQD&{Lr>=;WGHex$#s+ zU*+|8XJNMwZ;^{wH|SLOw?@PmikCfta1O6Hjp~qA&0Y;A@Y4_K)_wFN<$qf&$`?Uy z3%a$)M3^kXa#p1ht9PL_4u7gztc~S=4O+>Vu)jj&yRv#Ib8TH}B>1^4e&yHyT)P+v zz_nwsJ^DC8>h;#dm>ipLG>ZA80UT;{7)E;V?0WH{8?<2eNwZmy9WGAz^kAbs$i`bq+X5kQ+?QbjR_flfH zV<&Qs{bDUsY-4y2(KZ^;r(WwLEHZW2AOR`nXw?{!0;~PKPIjf!IvktzoLxG0Jk;igZ)!Hm<(gEZ=OwdCbG7fvl#d4Q&jHr%`X&p zWuPS|CzpBMNdsaH;$lsAZ7!EMgoK2x6EdiArU7qR zZJo|dh(p(_P~A99z{;umOa52H{O(1k-U90%OzRw+X-%a89!ik0zaD0XQ{^x*%(1<1xWCd!Aw~7KWDDAED+nM$cV4Gxt z8dV-g)p>l$NQ2n~u4cW+ZNl-bndZF{`}I@N`R|Pw0zk?u|Fbvo`H>QPU;GE;T4ctv zfw)lqU^ODXs&w06sN;A$5rT4NkXFqBl-Rogm*fzF5p2qH`u2Q(S`fbtGS0ln7I-Q@ zh)-+7PY-Rr%7F$@K!wKJPw)#GxP#Z_MV`V9$cXo!uP;L}4KFGW3zAK2ZvA?{WPWm1 z{w4@-CMwpN=X|?LnSYQ_$A6$O*I=r1X1UAS>EmpXW`l-x`+Z7-9j>t6jkK72Tl(v4 zJ`2`G6IHr7cOhK91MOKqmnDSz*ThSoPJnEI`7!?hcc5Uiz5u=QPI6fU94Vz{8&qY( z{6+eu``TULbJfJ_(~O1SQ3TkOMo*BLE#22Uc#8Wg_OJuJ zFrB0g!~2k>*L0jU&u^FeQzIl$vZb(~KqG*y$uzvuhndPB+qB=k#I>RjpG z?@({?tiXlMCejKQCRV1>f*uzJlXV!c@T zFtBO+W9OQva24JLi(V%fd{#gEeKyn`oIOZqF;W4_Q>~>X>NpmTUZos_Q?o8F%Eu5j z*u2*Giht(>6Y$hWx=Y)ub0#Et*zSHyGX!TgTH>1%z>JlaD@i@_TIv9Vk@e|Wz zvFy!dwK6Z(Ket;FNstH|fau7HCmBl@wPFMEwvmNzcZMbT;X1}2Gl}lsq;(nfhJ2dj z_56GyWrw`{on~S;uVq@)N1LlT`qM$edktCHu;qSfq7&_69%MLy&%zIB1$^^k>)n8) z2L}Xy(uAP+a3%zsGXa}o z(y#F$g+!{>F>hq{fZs{+@=H!GE|sFTD*j_ywFeXT=L=%$YS0r(Fk6Ik zoD(Z(jYQj+GP4*11CtEWgCBZa;|M7!0+tRe(KNBm9YCO!O_eoi3}DFfErN5TfQ zVN1|~*EMNa8iKdA%2;PLv)ippdP$$VTHZ+8*Zm)#XeW(IT93;NY(-yoWAj>0oGMY) zf@Ec7-8;tGNGZ{0kK9?7P$ql;wg)UAz#n1q39%NXF54U43$8Jf(gP>J?^g$s1|`3h zLIAjb3ZU)fD4_t#$wg z4a)E=DL&)^LXZwYJnpARYt-<>8^8r(SuY7NT>om+KW(nw=)}kE?DKR6Y4QjDfsbYJ zU`@)+*)cqW-{syA_2l7a)PTlgLsEk_0#q7JWwaw!OUhvJqj4gLC19PZ?HMi{L-Y2q_U&Hwy zL~?OCj(rM3Nh3;Ky^8b7Wq@qi!`lH(_ldIYWnvcN8t?&DnjnIY+MjmPBUuE!udmEr zQq9Pxkyce;L=h}pH7g9O){MEtNj z*^q=7scxsXib9Qr7P4AB(b-gvwJs3z@XXco&W?^NnSXZRdB_(Kev9lDG^ymSOGr2* zb9Sv0+rj|ITguvg3@g3Jun~9J>!i1=e{1}8>vqHU5;$SR+Nfh_pWp5WF0H~CZ_#R& zP}e123@+SEdg|OXhW*G+T81$m{xwyf&r@kt56L=^Tc}zZ%y5&PL2M?#na|5xSBZyK zO*tN*=Wg;wq>poq8Jj_8Vv|uNpWMCaHg0kzsTM!4&x9L1E3KYwLy4(|P1w-%p?2{G z>q?Ssxc=ZQX=qlf#GNNL2lZKuS=O2J99Nu>T9y^7|-;x4}exn>Nj)y9t#}jeF%;vIiP$i>Ud^Xm$6uYc-Uli#D|av~k;Ij{-DRd>PG_ z_w=JvLuhV66T(ddy(k04ey|&Vy~F+XE?= zl-`<#l`-?a#?E>TQFH}N@Y@@k_SVSnm|zmT=y-E@I5@w}%#))d6>_)-KF_t{_Wbmx zfr9ua2{wbbnCX7jtCI@|*7C@T!sXJe=3vi>&Vl7FH+8u7I#Z2)VFAK1)_y1L`vIpp z=e@4Hk%#5Z0Um=vgPir$0FgLHJ!)$Lw{?f@?aKH(&)VlWIE^nqZcoyse$MoWYh^-q zFE_j5ZVIvC`p1E;Y7 zqq0OSDkpfe-#%ix%%k39gI}jT_f_zBK0L@&^}|6IUzA9@z7#y|UQYC}&lj6rWan>g z=JGm+<`VSTIp@n3QylY-`_Sh+l-d%q6uV;fYy9CJiuO zPW=`T)@BPb4pB$chf6xK4UGy)F*~{o)EGn7s zsiWWU

DsXaPjQ#W_kjz1FGBPQ$51;g!^Oj;7yAuK>j0F}0Qin$mHB{gL6*2o7tD zg`uL^7f=3)1>}8+-ZYpld>HG3sDmxPK@eAkY`edKSQ0O&^lIu-tzI24$VJ7=(fotZ zl86zE;t;1$*kEj&CG=MYiKdvrVi@8}!x z`0iBoEcV@B+ZO;@k8540ow5fMiIZC{cVTFsL|J(lDjx0eTo|p!Eim`IHvbpt;o0ka;iTg=qd? zz@2zAY?kq#rdk0dj>ZN2Ga$b^_6j{B?ti$R$BF(<0)i?#t`N@#I$yjVHf!Tg0lxK?=|@WU{^eyYT>8l^BZpZ{&bI zq-J{wrQSa>ujnqYS!90?;O~kn$Rd{EB!CPDr+bTVq;R0FL%)ULS}pv#Ixel1jowI1 zVZHqiB#bVd%kkV9df_!xYV~>SsqjF?`arrw*2CdJkogpBb)i}rl9c#R*}cvYz=<9L z_2Q}Y8OSe%4XKDavG8!IA5PMGDZJ7f+#>TaFb6JA$D4+q4&Y59*P;T7HtiAd+@xN! zY~0OWg#l(`jj8+`oIoaupUd7Uhe7om!Z$8L(Qxejd3NXa_~UstDuA3Mo93sDfmlL_ zpq~m&T#)MZC6MRF#o5cJj*F+gNNpl0d-XP^QP)~)JiJUhxI+*gBtuP0#d>{V6As*| z&t@MiHNZ%_G3V)tvE}O#qN1Wsvh8V3iUBf1#oj0BIoHLY(2Z*5ldLSS9>b?c*JX?4 z7A!N(Z@$S@o!|vP@ncg(UprY6?s=BWB=n96FD%^+H>>#n!&m?SE7unm;gUe|+c%E2 zVFU7|s@Y5{_Rp8MwtN9~zfXY;TG}tMO^xfQy}q)Hb<9wA8Y|YCEDl=ZcX8Qjo7yQ4 zSx%B$#z_z9IOYHPWOz3Z2p92TUw;^C-YgMkL$Y&rP`pqj-f04O;!@f1o6n*fjCKN4 z`CKDLP2mgg6g`1$vQhgt@Bk$~1dxG-q=v<#g%J*y zrPsfgisr^lCTyqC-yY2g$PH{rq-(o4i7)CXx}r_nX=H2VqB2@>#Qesh6~pp&RW+1m zJ%+JS00t}gDHf@aE!59b;++J|1vDTFk+Nt`8Hiplhe|Q`{9|-IMjrFJ^ID4zuJA=CCT)}+{wwoSqqXcp&N6_hOTgvy zVG$dQ-2P+-^=znU>Rz}jmeJ;+yOURLD=I~WGT0yEDbiYJm=1ioQwPQ6^<{=yyeb$E ztA;`@je#8+#@|l6Cpb91B*UrahpV+X+Hk_aJ2rkdcUNGpGxLIIT8yY{Km?@XkYCuJ*zk*UXM&Q zTpQNy&RCPn`eOCkT+l+f_N$Ee&%mNEZ|{Ol(9irI;XtT{@LsDC>m-%{SwQ6W)5cJ= zF#f&OP^&WzoYs4Myb~kv%WKUJ-on8)_hEB6Xe+#_!XRH?K_EB4x1l)E?aCG0gs}o4 z7jV6_6ZrHg5KB|w!RrC8g(ka@8(fxxC8|=Dg>d)7Q@Mi4X*qxOMCWcz6%zJR-F-%^ z*_}pH!#0hPhY3Gz0TMX>Nfnrm@Ui+Zul4fUa-?*}<<>KDWgf zPvj|LC4)9_U*tr6x=~}&vtcXH!d#cc{A9qZG#!N18;IQdB%aknm5()Da50&?MI@WL zaL^bB&SAa3V23#JgRwgDOX>t{kfrL|&eI`Lx#!2>dzN{FI^WFNA|jS`Mt)pH(Dzj) zS`1pm#P0#pl7eu&=T0tDthnlM;{rMP>N$0(9lh>_>1Ge*_unf`E2Yv{#jxI2;T}~7 zBOd`V0s_j|V-BNz2S5n|cg~skCm{AB?A}rAa1}+8|6{*9IkT@rnTL>W$eW_Eq+jaO zSMB2f$A=iT-5u&wKs*^m%T8R}JzE>AKVC{tj^;BUR45`Zk3K)Ioetf09+eT8gs&dR z%}(=!tv-%HSDaNk7x}R4s_W7swlBSzGGXk_FHOtz2h}^=OG{nR*@Z~UP@ZI)}Q+H>G8&|jCx9?9m|2^0yP?# zxD3+q0K(Z_4OVj)l7ryW=h}L@!3E%X_6fmv+@SFMqU-RfX`vpbUYJEW?qL%pZQL&4 z;$;O8Az1&ER9MJ9*)fBQIRPqBZiaQWE`APfmFel4o#@@UiEMGf%zEHj?O^&~99zjk z7f;>e%;U$xcuP77!w`S3h_yOtFgXpwoeTf3)U?t5J4z1N!#745dIt$ai6p@H4Y zvUri788Oojh@6mdKvNgFW~BzUlSgH$uYh&opkp?QxXHQk;BWJ(8J_I&=gm0$RN3 zZ|oO_!Cd!V_=?%)$V~TIOhc_yl^jrZ=9?!Qr>5!y1VaJMV!?JY3`MwHiIivA?A&Mmx8Pf(8(iZ7-7=uq;hm$CLzf z&xp{3=Z>~@M-RF43rkVo1rNQ%%zkjy26Cv6s}2eA(>kTkW~mKTJAmENL{Z!w)>XI8 z^T;%@sWN)CY8)_{Mw~4<_kMy=c&bhm{Hp+&lL=r0D}t+eGd&PXRygiEHyg@hw=qvH zomh>7&7eJk!JC&UU)tb$#Vr{K9(&jDA_|hF5#lEa5xnb**E}aO_~UHRV6RG1cA-^bJ`@WzA&@cpFJ=@2l~{#6B8nqX|HOw!A zy0EFKwzd!K_%)eygYEWzVOl2aLE+ZgEQCVY%bx*3^*a zkprkxq$~CtO)m8hc@#$0>jlxk3`sT3zAiUJY z(o|SyJ{W?Nj4a7|xvd6w)+)s}_pL^8(#;Fdav}2Ufa**4v$Ja%oLyFu*6D-t;`vsl zc#Vr4d7>PLl-nzRl<@t!0twLu8p!B`6zW~2{W;m0hRc?ZW z1*BE>hCe4nVqF9z6ZHo_Q7_QSa`aT0jGS(H1wBZoRzN2tKJaB=6grtcWo$!E`N5zbwlEb}*i2 zcLs8Lvb9xGcvn$iABg_0wf@(2p_H;hO zQYnJ%)R~TXZ6B_C&H9a+b27g~F_-2oaT$wVJLThAff8A=VM0%U7;x~*UKr;Dyuu{~ zIr4+AR2oWDYr6ZB8Mxe4HlJUE@68t)iLTGg+O1~S`&0ivw$3^(tEOGs(x9XuNP~2D zwU3(nP7r!#6xhA^7Jt#p|W=Nb6!82+!aIc@}-<`oFz|AS}Imr&9!8P|8 z#6s<1PGW08INycc321-5IeA`?o)YV??_(e3H~GW~c25k7#2m?Q$@Qi6(l#=r*U9oR zSEFNB_S>eXt6ZhOq?Dq{))~sT!mL!j(cEIskMa#>4PK^*W}&zFwp^Y+@_l-v65z>>JKbsVMZu1h2W?+0q~Ikn9z0x%muzHIMJ;rp&tNg&LUs#}bF2 zjLmLdo{Elz{UIK#aXB^=E}7}uT^Z)4o@nPx;hfQs}Ycep@r}S;{InCZhk=$rUx(S zyfXzcpc=5tC5(pRrA5$6uzIbJf{dJ@_!1r6M|@#{SNB(9o&g^9`;h0G0GE%&_)f-l znuxD^FU&npmYXjzmobLPA9alZmvm1 zZycw7n&Eh>ZC)K;f}V^Jdtw+r>H3_ktCAhJ2d=eMPD>w8{^Fs%N0t{UPmb@(CThCp zAOsa$>`lACvbSE-sXofYmr6am*b@r0i+mj29EVDOGBE2bHOO5CkguAtooq3&b_}8U770#=&ae=b{F^B_Z zAnHfn8~_;ct$^kB3o>&ZTpI*rG-%?Djdf|P)U5je!(fi4E#!UZ#%W6Ca9}UU>0MsG zQFa!9-YGE_G@5uhgdQd8^s&!SF6AoE`NQz>r1jieFC%6{0`bLe(~(NJE75t)M_Ju3 zV^Y^9TBa^%YwATxnRCdGYPjN|$=tnd%z!K*6oLaHVpH!FI6sENJxRFDbGEyHCflJi z;2wQAnr*rfoG$kgGA-JmN9zdK{HW$!;(hOcNBKeb8^AO*6;09qa!;6W%~Rpmp}|P^ zcpXafn#27~iFVU>>>n1pl-RukGFZ!Gg<~x*nGJt0;`#?rw#}#MOu7 zfc3)aW=Y&hQ4<5ODY>BiUL^#9nv78g;-5pBJni3ebp^m&f8HA?^Wo66neiCaItHdC zr}w%zIzEeKcr4y&UL;5P!j5P&Q;h?VfJZ7!uRR2DWf3SHa*(IWt%$v^9cW#xo|2do z3Nm{gWa8?DnO@8S{u9Q7y6q%AS4<+2F8lZm+OF>d$W0beA;~L zARC@umWD_Cy5-c)7~UjiH#nWsUte!}r@!7BL50sU^V&RQy{D@$1b4RD@*LaZ3_u17 z<9071@?`_Lwyp2ZZ$EkE45}A^68E+gh|ad=2-vs`03(v^wKl6{1rGBIoz^5rbp%*-C|;PWmXl6 zaS%2_(1tW&czL{^kLSg)x`+X=x=S81cDT4RvXu|t&u-MMvQFfPVfC&Y5uYA-5nEy= zNKnnVyfkh!4%1?uW4ZcHyZ~-K|GWaF;j}$lcIjMQ zp`%_lj6yE2>s3}2br4?jD%(K<-r2+UW%tdr)5#bh6dY3RVeXIS^4$2j5L>FF1_LZb zjE?kLMx0*XPqHT=ZpQb2haQ`ZXJpidH#!4n!aIn@b~_wHHUzqBU*MFQ`PJnIvB@Mo z*b^$L9Q6uht3eL)h)p39WF#H9!u|SYD28n z_?&kFUe~zFG4GAV2QBBAkgV3BAtMu^Y((Im>a0TSzF~Yfvz#vhi#Myo>^0hE#oJZ) zDgXkuRuU@$UeC2k$N+#^;kE~f)6@VQ=HX*5>v@Hau-zN(>O}Bu$aax3`TaQf=VP}X+k6wJY`xmsOt zd&AJ+)Zn;b%V{=F4GP;ao5pi8tt8~LtCmfsG2k6ymRhCX74gdh9Bn=sid>JpkJg_f zMjLaBkMnd5J?5@l<#~Pa&S{`Fchx3tvft4Xr)?X^RtW(JwWDkV@MrsoF2f zYW2!zV1@e5(A3X_Jd~bAyqR;kANISoqe7Z$3VVbHT40^Kct$Cmb~Y1LWGTg!H{+;4 zS6!Ml0xl(%8pgalJUf>;zFW#qb|1_0#Qf1w`YQx1;vf1Ud|yA3l%Wcdyn3srnHb0Ziq&F@bvPQ${Dy&34x9 zxUjgeFv-bAcgg*nrD})Hfx2KRiV4dw6QD!){X%gh(VbCf?$bO0cZKkgu0`Y*5V|vO zBZfJo_S*G#XT(wQm$}JYeKX3#W2&Am!=fz%EOc5nA3ktA%Yw<60^EX$IsxP@fxe<3 zQy!W_TGGQ%2px}Xe#l|W5wEevEo)GrJ>Rr`Zjh>=t=C@ypd@@2hopHr}!6X$R zHVC15+bt#dj^Xkkb{n7$R4^tb34`BwRLk7pMhQq8i-~WIWJ?z#tc>KDP@meVpsc&c zhpk_mMGLl>JU_R@wqH78^CQ1~3Z$M!i?A@S#h2^v62)}}=Z237kvyYskZK<}I?0e- z&pS&tvC3KKD{pf!8?QoE9*jR)NudfhL7Q+VV_F9cOOu-?;!0r*fvnZvI~Y*^h!+>Y zdIphojv>jtHlPOtN4`sfkq?~F(C)ISb-|IjjCu{M3-XT}oMPncV^=P=-s4VAi@~Z~~>8t=PNF@xjBR@1pM}`Ci{I# z;{ojwAx0$;+|XJ}{|>f}_md17)kJ&pN4*hR&{bBGB8B#^b#Rr>aAjBVFV&)3VlMIL z&l78wckwMf^^*Rl`Z{eS9E0QZZgJ6Mn*SvpbTW0xz34r ze+D*?-Z#5wZ>8l*>g@Q-lJ~fuT|#c1Ob;n|wxQJBz=S#hy?YsEZzbT8Yvyxab4_^cGFZ8H z6=!C7>0c~Pc(&2|T!~XKbX|XVAuc zCjQGcd7Ih+jZgp%`2MhccN&cONpB&{j%K-z|D%N$)A6dGP2ejJxyBxlkAEb6j1w-v;n-f7DYb z?@>KmVf(y2lJEQKK$MTM^kBXia9bLLL)GHrL1@b5cjWC690*=p(pZLe8!9$%xbF!Y z+J197i|3@|R^O(2!A6Yml`tB$nRCcyBDr%$VQ3eIy3M#t1lc*%(ouHS!K1dJB_oB) zb6{v#?!IG~WzVTXAUSeGT=_%Mjo|6X>pm3}na#8m6(~-4k=e~3(^%r;Iyu<}gnhRi z0&DZbo*NZfyc^4G@&bWzT70w4Th9`Igs{CpM%r{bL*6`BE@2!9q!~nyV2+sI2egW$ z;QK;_zO{V*)N>wWva2=zTwJFAP!5njkeuYY5lr|8BF8$~IwwVrvV9E<`~cV`vd zjVRv$;)oU53lIfx`vo{1AsJ;2BaLs&{Fs%Becn2Q!snzNe=&7j z;qIvYUi<+k17kQ1+pYD;pi~PC<<0KjgRL|aCjB_?fwlz~`yv;hFND_7C^Iv;VAKcW zj%>RGFshd+FeU3>16hDW{A7Jtv@0v?yEk|$`SMaU8~J(z2guQVJK&<3W{(2ZcgrrDS)#efSp(<$6J5dpCHe0C^4{JTt-A8} zalOh6b(U1E?}opnDSB)U2=JeHe*((^*$dAXCINn(d(5|MKIThE&ud@@i;5uXZIk7= zF_;|`zl&U3x$qaAAK%pXVJP#}jh)YPI&GMLV*!nI4%Eh>(P&)}Jei#GOzL$Ci>bMTTCgVkTM~&s zYCiSJBE_aAwnXVx6~s*HOwc6mN&{brPbX^HUm*Q-z zNx6c%2RXm`?R->`_xHv#s!Os0)+|eD->mRdpeWcFE<~ImBu6gRI9R>`gs&fcp4qCi zm$J=IBqP}F{hGENz!xE9L1t7dK-3y{S}jD!W?1xr36Rr}PGf0hM!ru#W(zwUEK*^( zsyC%85B#>ZkaL{w8K1FJLqtDFQ_6kSiYrX383l^kRiKQ3eVYCvLJ0<3c?&5`W)VfQ ziL1&Z7{Oij^+nK-p;fIGz4t=zM?pAt4e42z_N_PqMA$@TvYqIesy?jV$GB43J$=n4 zu_)l1v+B-0Gv>%G)3$6CJ8 zJrb<}geGajM@sdE7|8fTuYeTxkG1Sqgu27|(65Tmwnsnr%JC?LuIf#HBNFHKr5Dti^?BZpuYUv>`>I9x_AV|G zC(p3M#}+%9k9Gw=cLBY_)r-1^%9g@veei2}PKUzAo-?M{!R;;VzbO~Xc^58|GB`O% z40G%JZus-{kT4CVHXmmNUL$&Uld#h#T3bMy3*W?yn}R_WvG*}EIL+5rWGjn1np*Lh za!=maA(9&tu?g`>`$7_-aCWZZ^-B0F1aHZ3CLt!x0_u~so{*v*QwJ5AMt*S`K)r@Q z!9Parn)NlWD>9dfT2z6?D3~9ODciCjP2$m1zDTPKi<>T*NpW)$)x*UGh0$f9p3T(^ zVg;2usIAaLe)f{U8TJYyJbTp@!lUU2Kg777_?h_KcEQDfYzT&N-BZT->Mop2yF9|L zPZegnf&!SaOgT(pOkDowhyI-XyPw&b-q42hT6j2_WxH*t`YYj)5xW7@|J9#t;{B{$ zcWc|HOX6O&$~7^p@2X9;dU^4~D@c2zY5X^SnEMhFEf@7tqL^>(HM~6_ge0Xjfn(jP z{OZP3sSvB4mTBCDU!SuNw)2DQD`7#Wg~|b6l56GFThC9tvc1jGGv#U~s0$5c<6nLq zt`XN%F#*x5Llib?@_02Gh3H~vN|2+c&+pv?lR}B?^uCI5dH?dU&lQUF*$DHU3ZH4dWd0IWrSrhHEB1nesP2s;u~deYsqSxx7Q zN~+p=L-JpKHyugiwd6}ShQjAM6azMJ$GtQ$lfuWUM3s)5pf_aj*tA_eP|LxrwGAop zcsh+wc-F9=uvas$@Eq%1EK7`(HUXbJ2LVKjiNm2+>BxhzTnIf_Ou;~e*msbg5W}xO z-XT^YUCT~O6($=E3?}Q7Ob#aoSXIoWvK8Ve_Ax5%^1$|2bE%t)l2aq#$sV%WA&!E5 zg4@Y0Zq&6_0FRodFj$~c{P|%Bb@Zz7F{NGaOZoQY<|m*V@wYT>^(1zAsJA0QM|Br z()N}dHUq|uouT&i`pIK@SZ8#$2Z$uXq^uF=phuRZh zB_VJ*>4WQgu=&brT6}(PcRaf8j<|Y?gj$D5TpNFUc~Q zJG%c0C=Y3ENL)SdJFJuoKWYqGYu11A8|duEBqr|Du7xMy*l;Udtj`n+I$RlU>kRIg zzeTG|mj<#Xm4oZF;t!!dp})!#>KOc7!+Hm(iL`0BpjtYr!eT>5O@-m2%<|cfU}%(| z?-tZ=D2YX?pQ<(!P=IsAOs!%mt|R8bw|xcki3b4i{zd$=4Dz z<1PpAgFc=GGdG84U)?*D8I1UyY=7$A%o$Bg7r0Z-WLrq_Frqnk0S0R&y_v)9LL|b( z4mK&)HLO?;)fU+&6qO1~ckLE&ycXX>24b0Tlx;+sLS+%AETLgwobdZd2JTkqEBAXo zwljAuKKX*hZc# zqG>PV1T>Ak9?R(Q(*2@X;yMKj99kQFp(vS4TysCIdtQFLDNT3WSb6?pw-S3!v$wK8*`q z#^717X$&#uS7$-HG@b$s7Ru~;iGDf^xMY>BcCb^XX47#k4~0hpZp|%@x5}NVhhGi7 zOT`zKyt>uTC{*}CrG2;TYnQ*!_KOyNqTCUVT=lm=L-2ey(d3%|Mkr5CLvC=5%OOIq zl(W0K%dDDN+%^D(c}d|*u0``OK}2&;r{<1ay!&w)Z$^~3nMS|i1~cw3jhs2Rd2Gia-0Ahsdido0 z==4-HS}NNqLfZq);QVI4tMn|T$NcBn+}GYXR*SNb)s02mpmUjlKq)nQCp&T1Do1EU11`|#zA6gXWJ|kxmnT0%t%clzia9%)x=7s z9zuhLD^}NYHwI`w3i8gQ>Rza>-eoV13lglQ8))#_WDCud7`vZbeHTPnecOOvz|-mZfgk!~ zBDsB987XZp^4cgVCv`tR^&10}`@rF;dIO;=mN$Y*J%uKZtmnx30^H5*u%0>(C!JXL z_33~uUHg~(N0!%~ImbSQwbgX@xwRv7`I@%dkn6@=>yGj`#y6X~JN0+4LJI)esyV(?!>;5Ue!_$3x*AtC@_8-<$7_gTxM$(Oa+`?lw1ASlByeJB(?PKm3wiD~A z1#Ozag*<#Ch&38X6VB{CTs79)XnZBbqtC^bK`_{AX8BrZUFe9enq7E?7R8g9Etps| z4P$jVlWLH)AoCQb)D-4LGz@LDvY=_7yu#8X%CzKW**2g{3!ouAY-DRQjl(TPCI`vJ z{dkNif)o1n({){p5Nz_Ousiu0w>jMKjTJ>@xBRd{juoT;1d;N$I z;1kdv*_p$a$x8(2rrH3X@1aT0GS=6_LwX$n3pR80Y9P3Txvn!}r_v-Pf4m$ZxJ{Rd z8M-Y?BT6zTM|5`D_Ld)~uIOXKt41!sJMvea?R@=*26jwFlM4kgyBH@?-1>fJZLs%f zLg?l2SLqhwG5w{Cb^PW5l+f)f>#QcK{$Vi?M7l>g0gyxR*o!%7(yL%4vwSA}o%&_`Hk1L{ldjO4la4X>FFz&^#Hfa~V6BUjD z5}$xgM}<~Fu&q}daC>N+wJ#fJH6yv>4Hm z0gok@TY{VAWznn*$52~6V1|Rgr={RMm6J{|h0Xf9`G^DGMX%^aV3a{2nI{tW$2<@h zFnalOw8#D<1QpRlG`?@wRO}dHFl$D@+*Z3-z3T(1`Pd0~s1cN6{fe zM};*q*E}FXe5l;NA4Tx8->$1hp3PmLp_c@4vdYir+jdg8iG@YkQWy|Le9T)OV8{2L3Zr+!Lt}iSVv)J+ZW>jD z)_o2!Rj}Jc`5v2}B@bTUa_iVJpA)bN^xmuC+INSY^LQO(;KKE&&6+G)XE&6m7z4KP zgIf+iAB^Mny*E?ZG>DiK-GlQ(7s_#;t?Ds25wQ(A8T}$CErHr?-Mwax@;1xibGw1Q zYGk<|(=|;phS*8zrVki&*frVh6790B{)&H5Kah0RTno6BEuAle$H5M0KpOZ^y_tZ~ z8fRro+_7VyjgIsV3UVf2*4eM2ON=%v!oxx!e6lSfKWn=a4(tfOm~T>Wr~6U1Q8CU* z7Wu51b1+sB`DVG<(sALgB;#-t_b-b*#$r};*X9@ikvx34nZK&GakRemPoXUZGM#aERNK@B8QD!PRQZMIdO3kQKU=DK@C&Z0L54=j(Avv^3T2 z5Ka-v{ea~u+Aqub(_ZLO{577~T zrickG5Ag$^tU>89sIOWT& zPD`ov_ug_cx}Z|#_?IU$SzRNNe-jjZuPA}!3u;7%I;7|b>Ilg*2lS;8348C5H{^v+ zIShb_QuAlVcG!f+TVK- zp!rXk{@x^P4<*9>$A6#v_rCv>iyAn8gcP_~mP2t?&fnAiY_*hOgx&n>Sx7pbTGPPr zw^askr{*9^lFff?HvV-K5a7~&N>Hhvlr>Kof?fObWdC2mz6hTHv7BHR;K*Q?U;ih` z@>`(Kuiper;hv`wMmZE`XaB2&h#!RS{`KoqbGAIyUQ2i{(*OO$za^7B6k+zr5j2;0 z(!KxZVOa#wBZ{QJR6yV<9q2}y{1q1Tf8FPMs?XS80p?R>4pRW7+lq*A{Cm_(VyOJ^ z|FqxVCnxE~0C<-X4Tg%#IC8?~fB*dV75vjIZF_p3ZyEqD%VOW1l@~0uwpFp7L61{H!{H~8A@$G+=?x~|&Uw#X&`(wjzI1YN@~fWSfzck2dO(uur%Jb^egK9UAa&0Xo>qtQv5Oos+a zW@IuMUZih6KCu5P1<6y^Q!sScyT5;vcn+Ht0^IQ3hvx2Xj1PFKofe0lLEfT5t=YYk zLtrI`hG5x{kgKzM(zx?QDP*_%7iWZbq%+k;q5O5_)7hAz|E>bzJ*1DHQdR7q`(nun zKl`8FEcs11dFR#A`4*D(Tw|ui{bEK{o_xs{UV+??Gk;Xei&QdQi$_1E#EPETm(wQv5!*V|GeWXi9V4O z=&9I3n2WjLCy}JSK(2es_%z1*D20(vECBbC%yPa=uBP(+>P+582<#Jl@p!Ows7T?% zldAwvxsljJ-?kl~(MkRfaEBa8BKxaw`5SrCWTu-9@M{w*sr9YU$rps35Ua^6UpcN;Kzn@{xKe`TR(|_$B z@^CY%mYz&|de4QBrdsu!Hwm9hP0cgA6>#od@VGYuur2eP)FZ1vlGjRZ=_35Dbt1D; zhvU^5nAEjS4#DDQKtp1|pv9HPS?<;1t!Ox-mAM7MBHvkegXqy@Y3kDKpx7Iwj$tLOUQ;_i>mneO97-b6p|f9xXE1nnOTXTS2>*J@BjQNGvt zS`mq9`54Wlq!qxGr_|2L-D)5%Etp#R zHf*NFZxx%223yS~rVDv7AuhI+KF2ed<&ue%9Dw9Au!JzN@Kn5XdY+d_x+RZk@ofk4b`%Cj5aax(GQGY~;+2)`!mHyN&~gTH2H-)Py;_TVr)W}JU%xad zl(Ta6-Qj8jRLa8rb`l7FuMNb%0w$jUO%L&Yd(*5mWSh3h-cPpc=u%|FFu-PtZ(mPy zod`=tQ`#r3Tz5Ahodb`)``0G}MDj>Ae@8Vd$x!410cYGdN0k=p#Mb~nVZM(%h3H^w z1j2$%uN{h{w2`S(ECnd|l0tG=F7LVEn{{dK00-%oJ~ujfAU`DdW(B3tE*^gShvwy{ zg;ox3+bgV_3rjyBS!(<4&j{=L`lSSEH{lQAC^`Hxg_8<2U6H|(;xeaM9ZLYId++Yj zT$2jM@Rz-mWjOZ5V{h<2<$S_eNyShH!&P)_Ir5a7w1l8k$V+|oI%-O-gs`qIrpgyd zKoK$}W=ZOG2ciIoL8ZovgBsV%$kgTYAx*9|upFr6$xr*ZdlYBre!b@SAy+7KFb(ae zdzA7|GVSbjpkl}7lK5Qv?Qu_2t;AS;eK63e&DH!hn4UXAxV(#spX z_IJnyDvVY?=#a3058ZUmutu(CR)8Df6p#2k& z+Fsq4;D) z6?BQiX*HV}xj*fK6%d-zy4!YZ+NN_#S;ew(h`aP+$_cv4&e;xrI$!iYmQGytiOB=# zHo*;59pMHhwt!rDb_p@`ly=oF5&Qf&lr?IGa;GFXL#<@KHQ{`=%wdq!a^CG3keE9p znZQ}(>(eakdD_OkHxI&^b>8dLB)<1~tZ*UrQBVRs>jX`HNZ$ir!i!XQh2?iGcrYQX zynX*(1oko}o#ag#GJ-4t2;)5lRDAj6f``lF;$f8wIOgq(%51QA^jbt?EYNm3_T7gO zXDivg5iPK0R+OW+Hf^fvW#(Yuo;LPf^o54Ut;V#E=)%vSV9{$?d{M0ur-`3?AiviC zESD)WwB1-)UvvEG1wQM~LRf?ViCb&5o+6&b9sw(ap2WisIJ&yJG>Xhk_e4z;ciAKOn8_ReKQRKpg z$M}6~Ybax35?dGqXviq*~nBvca0xW6MWJ= zE(&U*W1C`&?e13U)Smsx5nupSAoXA~XfzwssO8iGa|FH#q(?Ug4CbBo%i{2ULw6^0 zm2l%9yMclS#V$z?jYmvu(I7+;pJ&jBb8*6u#yX?#JX}L*BSx6=8b;njK#ptgs}}&Y zJeNSwQ|95ushVKsI+p%wN0i6yB>0QJf1uz(Asc&i-jr?NWkYUbvC>n;O~qhB((UEyWwIl)}T{X;)nUGY(3w`2r3unYvY!T zk8UNLFEKDk@zDF?SlV8x2VxB{`T9cHAI!mBFWP{lR~yWweVUs3Cu_YK%ofYb_B9qq zhh#@n7Lcyukyf_nLvN_77uNGeS9k36IiF9MqeNyRk>Ke!x^#TaOfLL^)=~5 z#sGX}zI5%g`YPn{Lgk;M*ms9=ZWgUF-3c5O05a)qj_hkPl%0v6W;3Iggvq>4Vg#rG zcMUr|O;@$X_r~4HY>8L5*VFuPef4LuipgqxUTB6ul7!(xtWfXQpjgLCIz$TnDY_a?A6?L5GO-G9eR?$K8Kz2 zw9LKb=j~`()sM6>42$cZcjNJ1`T=dd#mrr-nLN$LUvL)vYrZ!1Ultjn?$!+9r#Q_6!?$#Ix+M~4Ep-Z zD~E@hNdf~@nZKLrpPoPx_SzQe0{MiNd@FhjHsm>z&rKLYN7|4y7rV-p&5k~-O+l&78C>*og zW|0&L@dy(I+B~7ACKhSpy0;wkII2LSOr2_RuYsNXI*zni5OO!eN8IZb8XV2kbgcq^ zW?d**-H@rO(w)jwV>weUeSkkTt>ZZTWou!&0%z~;5^aa}hWnhzz-#AhdP^Ate9LB_ z2M>gag6}ONMB-T`pep#iWM;6mpNo6sNnY$JxI{YcyM`8NH@z=VVPIPC!NSxfCT1fr zs&&}Z$ya9Cm``o*`#RquD!zAfu_r)X=U`ORMkRw@rTK_4;&CKlfJ<{AC)OW#THI*% z`R8D0s_D+H(D0?pO#BvUSlF}erx%6*3QTv>7w@1rypOiMO>D>KaG{ly^d>j6i8w%r zMMp645-gLz+WDZTQQ=N8d^mu2&fBGV_@QRe_p+M!%}w6nh*4qbM^wz;gY!Sbq%g@b zbXlhF_oPGPitVqhQ)*PwB?Ri9z~b_;xAUP*K|1Wf{8#!xE8z&Kscjh@MGO^xMEin} z`_1eJTbbC^Jx|X@3bhhT`^QXx$3Hb%ed{zVbfI9{&vsdux|aN%&5Fye$1<5*^u{Fy zkiAGTVRG*ZT$ybPE=hJL@F*1tSsSp4UADHieXwOvd@hc_elFYF^-ZE8c-8*m4vq(O zkl|=OIk&;6QyFIZP%(Vi?&&f@-(bzqCf6h;kaIUsNVa}J=`wRSGs>GEkGh+j3=kwD zxEAi;bu{uToS}^Z{_4t+pPX(4dlM+PW^Pj8Pi}5U4YIclR`KYqJZubx8Ko!s6bqCU z@)hffz66NZ_GB7qTy+XLFMuh3Y7|L`4Bwp}tyO0h*}B9zr+{lHhJZQs zP3KeX*NnXn+9r>2{o(VVpwiONIWEuOE#i>ThTBa~Nb`Uw8t}ape&%~dB1{N;0e#od zYbl^4i@!mHAC7YufBNzhx;VRYwN6+ykTVUQ)&=$4z{cZ&&XR2m%G#?%e=Eu` z)nSegZw%nikN3WU=wpuug~%fpyk+a1to0WKaIK-E6Luq~7o~QwQ4Du(nYw~9iK0|2 zICs|yopt*j)!m(}Xhtc2MA97o2&W?6W?H;EYUqF5kd~Wj-YsxSeAmlibb~VjQ6c+& z|IHyQiQh@=ZR;DE6ht1UouXtl4~@N4koKFw5UM$kh!_|4SRLo9yo; zcfhBb5lH=wz|n~}JY~E#iT0vDSnXalyg^tc2QAE<__0j9yBFmYO7vOZT_t8acT+Hy z^H4b;9#PTpxHYJA}fvr^vd#U zkhDWbP<`+zl3v6pV@4woq~*18N$M<0!#6KljqA zj~gtx7whs1{ekHP1(M?L$cmtK4l&t5D(C_Z_04$W^%BCLAFzi0e zS}=0YDd~9clEt%Z@=a#RZQhd@$nYd%3f^5g;Idl=_oZBd&+vJr9zLFwvPohmE6n@%|r{%k2QEaQmyS@n)I+4EJPB)q(ntqaMq7_py1JH z-~M1WBGj**McigP!&L`cSI{Lt)hdY*>_=&hm)!2tj|uJL#J}uuz@q-h93hs}a52@R~%4b;Z z3$?AncZ?`kISjvoNOLd|0w3gmzRFKO$Yq}z@H?IC%z)FyoSXXHC2gJK^~tMOb*|gx z^w%a;N!+h6^BUeLB_V0L9E76`hYEH|N=0WTRv$#ETnzMoqwC3aaVVMn`QCYN0iW*& z%kO?+Evz>Z(uy>Y-O?eq=}lG9bVaJFlT38!6;arKIoZdvm_U5|#$bIsTsrpidN!9} z)cJ6z%Pu?b@3WtVj$JGgK4@!bSn_%1*YAFJH{US^TUv5qcLpXpwoCY&gU^aj$CE@O z$|L_iPcicuRQFh|3N?y*UgAwrWg$P*2w)74%DPsf826 z=l`jNr}~&gCn0e|A&5YiD2Aqb+qfpdtZfOyf0Tye>WG|7U)d4OAT6hq%wrpv^RWM#%oFV*a@TwnT=H zA#Dli`~Ur*tx}R=RY0)wrR5;ETV+_P)8Q+Kev{oYaaPq zS%0b~LI-%zk;L*}iI{rUe^2|*QyqCina72|Axh+|{*W90cO8*{k>quMA8!o?ovjdQ~mjo!X%6`nhHG0SxtX_83LkAzOyo<*uw`EX6Bg{$$X_R4G>i_$d;AvmT4if z#masD)J6s9Y3TpdEaDt16m&#QaLOw(+f`^dm_f;}eVtp(X(&VRBOmZhqi)0IbnR>I zv}_{J810uk(9&u=W&itKfF(D3s-SD=F?3*zms+|$nawhfVGb?8jQ-n&fa)d@KqA)^ zW~D)2bUUXp8~N$n805ZL6|Pt*0RJzf#U< zhui)@(&n2Zvw@ub;;^6djEJc**^F@(VG8yBc#x+uj!Gq7>OTHlG2e50?IU~d?*U*b z2aXquH+G4Y==Gd7HsjSh_{)9I{>Es!sBmJ)oiJtNb<$dQG@I&z3@*$%crrDb$plOX zce?&Px&!e~OT9xH5(gTvM2$oRfjpL8n_;&4@JEfT?}PWs>OKR;^76@LIfwm5-?i(h zw#6r5pVX_w_1ew+k0n|Bcfw4wb#kG6TGftrZU%RVlWXG^T-~KXG4&mh2Et5%<;E!G zK{^fIg+NYUA1am8;X#Mf`uBj_Zc)*OztgQSJUDY>KKK4l_vgEoZtLwBiiPe*Zl~kn zC%rYJEB%ibg>t>RFHEGL=3{b$&gEwP(Ay*FU;Aephm-A1H+4r!T@cI`ic%zR6@VbC z#V9Zve-29x{kI#28N(^klf+XvP?}9_p>`+J1t9N@iU96GM#IoBY!bJsKiZdVr|g)R z0xH>7y}cp!XH!T>s7Pg_RW6$CtgKlelWu}2NpUM2A`DUlG{R_R%(yWmyvp6N>N+5^ zSFs$(9gloD4`=i|%0vt~21h5wv)`@U5H+GC98e|v2N(AgBvtL-^ME7a=0w!stqjRm~Jz#VD;147(co^Ee! zCXFGPLy{jmla;G>9s!{QvC^@O4V1AADt&7mibgvn78@rh*b6OSJ*l0;5!AJr)i|IjN^)Ipf z`MR`gv1%1uZZyl-t#&6%qnwj|P3joRZeyR@&$T^2Y=5$5)~%b7V#p+~6hbIOa|k&u zT4P^)WlS4Vxv1*>E_&$j*0)&=PFTeHS8fdORZ*tGj9UoggHqziE>ax z3iwDw4-GMI4^_uU(Ws26?hPZ{c44Aj#)gH-$Lle|1DJ2qT#HhVO%IAlB+HGinD+~y zdovooa$<@4$p;Jbqm?CC^kJQ{c$`^@{1@`Xt2;6SOz+mz~S zhxHytz`P`^Vsk)R!Zhz)9I?o`N2Tz?oU!$vSi{qhOJ4Wvg$x%o4R)Agpy0*XE~|bI z!S%}SGsTG8h?jZb;5w_|2nYX6|go-{1~Rd z1YvT3+(}Y(&Th{qIPEG&=5{vH0ie8asuZubbLz{j@XOva^~JdJfeK?D2gC3YilC|H zY=Av=4TOP=fEP*T?9FZ_u5P@1Ve-2_%QbeYur|*{i}Z6oH6ghq?qXua2Gv)$+j~{k z!$a^yx`bSB%Z-sDGTuM$sT~gOS#3j@It@;JxXi_3NYbT*oHsTepD0DJi97RI`1|A$ zlkFouy&%mnuz7z*C>n;a45;p)SjI$>h4-4B*>n@%Yiin1LHq^~N7K=Fjg5_2Z|fq; z*HSs0>nmbus560}Y*)9E$sV~E`^>$a5#R6`5NDa2ErQ5+fr33*-5QY^uch?##@@H> zu8MahGKo9gyH{ggKAwg2ixclLI6$tZ;aARN#Xek~pAVzCS-R>@HkO8hT4km4YP>cD z6W3STrC(wYk!}u(kKab+`=zFqf}f4tK<;~#SXbL(*On37gCK*f)JkI4_oX1KX8VnR z)%9r%KML`SB4_pEbD+T~ffZe4Xt>$Ua%QIy5X%jW5isa@wmngot^Lp$*&(^n7r(c9 za3fJS`c`PR%K8m1^xgV4hTmFrO^!N@4bv!laPERYMsN1-ISNTJVp5ZB3-AhQ}o8oK|Svzvf61HPDnhjzK#_xZ4`XImLYMiTy?{8Ho5jWl#wM?`wa;6LWmq z+$U7gx<_V)VxNT0KsYFJXux8Ol6zY2xTE3Hq^BcE`=U0{4v|7do8jyjMY z8qCs5MB^pn7Jy7lwmG1Sj-nxGF~MVa_2MF=N5qR)%jvqo`7k!v7x$Ojq=|fvaK2JS z;FqsorSrBsnl5~`z5gFu*Bwap`~49jQ4u0S_AWE~R=Xcof!cO3*NoC zN-(#MeJhXs-YscKYQPSK8tV$Ee?PY$5|Ml!+MK#e;!^ zfoaFBDN0PlbcOO=+^UsjpRy0D{3buuHGy(87Z_INfQn5_V)G{jubMLu+O#^=O$TI(01JPMHcYZ2@SVY%zA+r zYSXqCky&k96>h&f9PgT>*U!Y33UtUyn%m_?x?{`ciUY#Nn}anJdI-)d5|fyv(}@Nf z)1LN;VQAY{&DE5(#fpoIOLVd^)Gs^YqlD>#Nmee)oOohHm)?kJxK8&Nxac2FjQ*JQ zZjDN+rDqpDEZpu8V#paV3l*%i{-#~u5eZ@kdc=j6UE|%g(#+c~GsA0J7S=S%9ZMX% zJGLwUl3H8)(L9}5kl7NdJu6XJM5{ORQrmys7q=P(iOVA$;;YmRSHXYavIP#Zc{pQ?IkCHF4>WUzr+@IE2L>@>FN!?OXU8HpqI~v)6D*+AR&S<1Q$LjUT9( zAO7}RziX%|XDQ*6Q1)l#nc`vKFm1zl=et7_Hqwr&NMd^(f9%)GB9`}-^>%;lOdlTD zH`#1q3)sQKEQt2l`f?)6!?wup(Jb~`d~5`pi;w$z-UpK)D$Z%X&`#ZcDS}*J%eRht z-S@FY%SVFD^q&cR)7{fo9e++s+U@kp*WqirP3~~hn)FjT?#1xhD0S7NeZb^j8~+No z+G*n>4}VqQ%5V{nJVL-kIfA-bu4OKYZfG7EKA8dUmWl0Sk^*%VdhhOz6t5)9N@v-R zgq!pY$ztnMlOGA6hx+Z`%^z!&H7y)OD@qpbJtaCUec*S!gLJ651$|`Sbfp+~(L+%vwv_lO+>{ni{grpz+Jj zOON8ju6Xp)Kve(UZ|H(wpu#-qbTZuAzGB<#ePMK5-zVvKOdugZRZzd>8!zNM#%;C$ z&aMnr0r;Qk37r@&+rW3O8yB`=8YEX)$;libYKEk-{M zcTi}rEK<0I(H`d3-JW0C#wlY_rm2wq4o$(_MI`w1JH}x1cidU*|H(dcXi5>#|C4jJ zqg9>2U?CY!v{$53cgVPKTY}z2Zz0RpX~k21dZ`kJU-H4jhx#oD_Nhk%hAtLsnN|nC z9trIGTO81f9I!9ob;r;_t6gWc1T+_gZNElwEN@(ayQuNR={O<7UMUCPJDZF2cg;Cz zUmfz8jAA(6* z7QvB>uJhMckDJ62AE8qL#36jwEHC}`Me0TH7)HmNotB3=(wX*{3qdD$M=Xa5l)iNv za05)Ndz+mke#xaK2U=5>&UFTaXdY%$mLIC!cN1Dz32C-tdJCQ$u|hq$0zxifUk#R_ zGF0+}C|+tCrl1V7$LxL%xtWZcmcvu{E8IfymeiB>B8Jl|6;em%Wx>VWY4}$!F)I2~ zDBiz!UF3y11ab+J#^N!qD(kTWc%s=z^-fDh$$5ahld(b|YeT|mw~uuG=$~L7bOzef z2ZF`Gl{q5>_tD*>V@)v=B^~+LsZ4^ODultJ2WB5F_fkO^&;9)vNNHHd$a?RM#9F@N zN|*lgvBymU9k%0`{Pz%aeXjA9rp-jL6+@Zf2YkV6D*+Y|g^e+hj+X*ME zAVh_Q=TVuQ-@g+jrE1^@)O$Dp6ys~_XVG)icYi>cK7&;i%fa0r6wojnVRaC-YsvBF zIELi7FA~yWpyNnaLTy3r-?61UWgQYf5R!&M(_Wg%8+-oqF--(Y>+v0)QWc3o;4tHy zt-<^$bo(4*ew!kNsMa+bP(h;vb@|UEKmc!}m2*+X+~(G!{h!Y)r7=|a09_Mo5JcYp3$4F2pIlhgys4XK1K<@q+oUKtAH906CjT@k$6T%;?U4@!g0J-=q!%H z!yo|N&0mWPkvbdoh#VMwL+G&!Bq0S>BAE>_R)x z3R!zn+g?<&DyI6%bJPNtNAlylpT1fz)nF13jyzMwmLuZ6f64T?fw~<0?xW7PV)QKBf0&5mBoa)O@dT)I&xBa&w{rpl@Xb6z9r*EI zMk>HA+$`6XI~|%*A4BDApTRh{D1p#l;JN(_i0>4k4l-W)mu}=w!8T~w+>|jk+u0&Yw(vtdT|6ps zpiTTX+PnsI;aF@eN5K#UaqmdZ)Jh$opcbOZE`#{p758Pg9r_12%=&q|LH=2`#(SF@ z|FwhEkH89T;HH9L_xTL?^Jd6vUh0du|bI!3!5cd9}D_PT~efiXqkT(^6tt zV^Hts%Au(2o&EibK@!>cZ`&1?+`;uYEYK_#wEnGQ47nEakzkT!eSJ0Z-GQ6UjqjL- z7fL+hOfQg0Z?mn2^8|nDWKZPuic6ZBT{H{gbXtD3E?sSfz2H9GI~DT)!r`c)%5=tV z0K8k?Bl|UdmCt%J{@!7=<A$fLob`m#t`R&Pq2&BX^dkq@Ikp%=`cJQMHDD7$-^->-Y$lni?Jj(^sNYxWi|hx>!|xow#T zj-gBmP(4M-BC=yi!DoetpG&0#U=KOyR8aYPNNPeEI{+KZ5k408#-mv?&*P zOhfp&EyCjYielN~MwGJOnbcKWc>mmBkm|V=Bdh(VBQje1!-dRNm`z2>C;%bBK0KCu zTrA7;4?^Dwbn+oUpzX#>DknCAmx@IJ_`xzfYDDo1=nP1A9Vzi?CLSPh`x|r&-qhSI z?jo!F(M1w8X$<0Kj1~#fyWB?kiHfSoc2~=&KPw*&U{|(EEBz>tL_O?+I1wpITr7nz zPmyqXpT+GxI#ZLd-T4=-|_;$7H$n0AQ=OeAWLc_(_)gk6|UQ=lw()EVzv zbQFD6mD#XeYX&Gn9ctT_e2gCH0Tstl;5*?JGZ^}#;?$YZR!l;nt&;P&-ze6aV<8to zti7+`b(umJEu1PP$a;&6OHnv>b(&Dxc_YDSZDyApr6r6cp^gNBvh;Vf4XvXrZu|rw7&XVRPhDP?lmT;Gtjr?EuWct$N?= z^ID9J6MW&+ib^noFEG-!I~{YZLJ`XNcYN>@a8ZG)u*}0pNc8Dw;D7!R)P|B~D1i~O zZCe#it?4lpl&qH&j%*7mIzBjHV^FTbP6vjlT}^&>`U`TO3Bd?P`qGbqMU)6Y3&%bs z?rc5zPbhIQX$p$H^pN=bsl7P*$%Bgl5^@dF@B}zTP79IgTIU8H@j+QXvz;&&uval2 zs)EPT<$I0t%9*Mm1tVAsdB8@o$_h;q=#c*N(V1=62NvteAm;mf=O&?qLSJGj;2cx+ z3v3}tJ4B7agN2=2VgJM zh?19pPxKs&A!O>QNPdnJp<>YVze`1W0)K4@FI@xjws9u?P^dQuwjqAw%531%L;l@1 zBzuoSW&B8P1j7I83d<8FpK$mK@F7Y3J+GY-(eMO4>*6`KWj_I~*}@20IB<&oaZgXT z&%@-BgD7MzmYe3wn_k0}D3Gj%%R1ymeRfyio?)DM+|(*BR=ARXOCbt-V= zD`gb}H9ONRK)(=vsidmb<1^U-rjk57C1wb;$anj3phELCJs{;D$cf9@XLNst{JiHv$PnSg(c0MgvJqxIfEMOOs*5wNG#MhrL34GU<; zKcfj7X$B;GPOkon^(eIG97tdby5Avpv^bl=aDgCM^qgQE^8*N+FjIM zBvk{)Ii^U^0XkvmtuMfidEC6FbDrFzuL)`bsqo3P&^<|HIWZ7N9Jzf1%*6!O5hKNe z-gM3yHgIBO5uVa?RUOEt>6Nd?v;Mw}YV2`-5UeysB&%#16>|S`%pVVk#EgEnBk>15 zk(o1e=6Em(64ZW>F9S{hD&&^xJygU!WjCcGy2{^M01BEU(#bRuQbqbC!CYu87=P99 z_@^s`fK_GBN&bIMgZya|upeV;?)~Rb0f^JJ_j$N?-3esc791{ z0urALY$kIh-i8RB+k$h;#6>QXipSs_*oAz);u%^lK=ptvF+*C4;o+da=|PON4JTgb z(Kr(LfSHn?zdZmGcue8zR31@R{-6OQR$n4+p$G3)!}o4{#q4(kf*r3s2oO#(o}vHO zU4v8+Vr{qe^PFJ;zoW7Vnp^xkg&8dBxE^3`8nJ89L1N7BXWc_GABNV83!n#IpqlER zJEZ<9zhVv~lSh`8=AMev6-P0U!ly{uS3vs~XGS`&Nj7Xc<@*Wmg6oj7z5FJRWxMk*r~_c9STh< zD9~%0UE)r@(eHJhS`Y&#Fj?+CKgWA_rk)P3!7S3vx|Oj(yY`C5gV3oRSy@@ zZx@x;I0+N1z~ne&-tqNFdA%)`I2B=_;{?B_Nax=+SK>sasYR^i8xt{x%lGGwE|CTa zTqSyr7CMtc`$p~#Rx@-T%7cW5zhj!-yhn^rB{{0uJCU&K<82dfVS&d?uxx$V*kb znau4|rG`SB2>yIRr;$(Or%sY}eneNhpUd)Rn|-kU(9%gaxehr8KS6R8P_|H{+$+YN zb3oFZJfx#Mr(v_>q0M4-bPv6PB4VuT!+gGdd#b(6r3$%p>X6JMfyt_orKd-r)rpTK zp#nLqX#dpM3{qS;v0u{tb(b4#k@~~N#zAe_h}L#?jGfs&q*p@#+&7ME5_;!ysE{fq z(T)}lWEiLzEAj-AR`)MDv+UWD(4w<3fFZa57G!}EfP@e|)EOJ9j$~Fb3JP+t|7>p$ zJ=osak@7-0R7v1$H?UmfBfG82@iH`5mBYXBx!6dz%^e1YR7o6sc6PbJzQ!k2mr2lQ zz>DU8O-{1DI>fnSMh_z0tDAM3JX@S_7UwdSvV?u-2~>BA@2WKY($nbn*m^J-}F8L=1|n({TDQKo`WUl`hH`W|xa z%2?qHN)o~eJ4Npt?V@FGZqGrs9lBMNd0j!m_`&v1p7x(U*l732Kmk*Es>`3{b&h%X zOZ{K!sWMlY=pft`+VnANV}>z$8TEoQKbhI+;Ar#djHshK?V!LVh(0i4OPhkRIHQKe z!~A~Z`~B)~Ha&L24ll!HTn{l|FJ6mxT;eC=x9^R{OvsRR`NTFEcIB&NDHCA|z6`Qg z7>9L>G{?`4O#N6@`R#7nwxtkRo=h?+Fz_(`^WjEyG><+pf18p5h#p0T4k zJ@^VpPcaN6nAC@?Y}&z>m+r($? z^(C9z3MAY@wB?;EYZq42vh3)6$)CRi$bgd_7uS&$>;^MR*}kcBb9ou%cMMC(*nZN zTl;@NOX-Z&yON=JCYhha%$^8c3a zl#;Et0DbBh$+i+Lmw!8S*}c&Q8w1Smyg` zOZ9$ll^X@&E>h33Yj7FK!wQ?5TQ7K4O92edj8q7uA2uvHy`&U5F=!4xD1AVkrHpY_PzBx>pbl$s<5wu7r%U)>DgPK zIlP^>Vy4%#oMl~&%-9AmqU_C(k2!b?%08Pjx}>YoW6BRoFw0F|Sj^`7iEEeuQF>#HPraP^Re696!{q&F3C-}E+Q60sP4<*FCxtSu&z}4sSW{iVheWm?5FR;1rOR5=~xv5Z;3F1m&%kB z_!M$>Kb+vlNU597x27wihBgG?+@U|+6t4#Mde#seVs_0^jv&Dw(si1@tFhS<${aJ? z*3wzjYDVys^Hs#kRz^&`OUsQn#9$$;{;uyh)FWkzOgd2$OshHRCc?R;4FqJ^;g`UB zE#;IB%VA0lot?^FcuPzx?!RNmui)W@tk1HyJbdWff&?vPS3XcbdD&KOtFh-yK2E z_7+(UU97+lXH1=Y^=REteMr|$f?2DkP>%=BaQn7mN4!v`WLeLqN}bFrzyLSY3@zZx zPxYBeoICp_ywE{A1SZfuHzdhT6Mtn-YfNi1|N6@pAcf%`+3lI6c`|$>wjG)sky^^$ zlDWsi-4)1lu7+#o+Tcf%q`g`f+1w$Py?qMRgBAA_N43#WN~PFciL_yfr@NLupg`>I zC+2dRE|;;fvE{Jt#=_>_$Gq92KIYurW()d(6#VlbO_j%TG;bWMawUW~y#)Jd{|6<> zuow0jMuwCyzOF$78;z)Gx0+YC{J8?y5-vq=rkH(EbvC1LBF=Aoj{l2CBFtC!-TjI2 z|8<>xuIVu3H~4t(`U(EKSnI`3p`+##hDuokQd=mL7=%K{`mf?nd=N=_C}9^*F0ah+umE$ zo=T7`PJ7juW$!oB5~JybJHPy)e5^g3s$rZv5oF=_Hu z`W)ULubhnJ8-xSl)4?qFAFpiokm_ARM70mwEhf8cz#phqx#W@ePfX3Q3ZxrswD8>9 zpO{$QTvd}5wBCFjepe)TuF!6<$lznp&_dxYf5b8`d4K*6{Bd&%mNiTFEB;R6-)n23 z$m}~34)h-_3!_6IQ?j^mh*|Mo{nAjac1P?AU3**;{E9h-!SASmj8P+Deqa!RaBTR6 zAhYBpSkc0d?!%PNE={$>$bux6muR8>x)B8$^S?qx{zjI7h8a!}}+@ zGi&Pw?t>-z%>7cML(kGcb?&c~S^81_G3x5ATO~$_?tG~Ethd)@u<0O2D?#d*B?fh= zSO95n=*LdHuy-V!b6Fq{w+%h3^T*GI29#`-_u=IIW9ibLy0gu!Z`v4-Rpc8ssErff z0QEn^FxLzGCOp}S+3yU0uXRuB2*|O+vFA3`>tA`lBj13Ijy;mqVbC5Mv0C~esUCCD z+a{u8=Er+#WQ>U;-M=NJsnv9hBi&Ex;~WK#+o5$Omzj&%9JbN)Vm z+}y`3iTlJim`=CYC*W}X*8}PBE2L_w`>#^_gmB@^q92sAbtqP} zH&^(vajrxK#M!r4%j5Q$mbuPvjrtPYb*F5m`~HwJ1Wn7gO}xwyRrt)WM!#xF{N28$ zGDt+=h0k7*Wj1s2l#KJw2p9(_mt2}FBX`!HUXWeudUJPeP`seUmmqUG^R2Y$(1LB; zbZ2_b^ae3SST>!|J({N(kLu;W8ze{3VK9X=)0UbjMeaFTx4i7pis0+fF4u%&Uv)j$ z=tS1Z14rz}_LS%9|3$a6X!v z8MGb9k;dt4mSD4gt1bu#caJQH*oZ6h9~;ebf2=T4pZi*Cv2>v)LH1|Nw-1QgZYu!SHrgO-HLnFtx=w|EG!yYPP%>w zlC^h&dtX{P{=Dxh>X-^DIB+Uof`T*uRe@}X#}z?(_q~k*=D35qX`dRue#2px4ZVLQ zFa)FiOP%yUM2H6$wdL|2M$fIU-)#>|;#&c!-TAR#1x2cDV&BZS3=W1L+#0jUk` zm+qW-rumT9s4x&lesxcf7#e#5<3pvPsOY)S#NBzmWb#ViM7D=fwgmndX39mA zJnQC!7p)LNa9P6Z?OXqH>QAQFdENk8t@wa~easL5rweWMSU1G6Mj%HhZhd~H8V#K2 zg7rjB%bBxK>^`nf61pI1^N$mKH0%3~;J*+U$OuD>$akVzlvPw+*6eR*^G1Tnz>n5B zPKmfq@ZDNO=%S3_TWm2>PtS*6nwzD`@DhjRu)}EKjoJH4p*&Fb? zGogu#^L$B(yR;EdHtDGBR=E%QrwIngE$*dFo^^8S$eeZaWmHW;3$029B^mzMc7F8* z<$>K)0Oi3cJjrh}kAWsm&Q~TTuG895*~C#O#1#iT{!WOMRtQb(nXgPhlBE?|AnfD;i1N;t z9v~5lxpJzCK#ZA^XKkF!Gm)g(3-wjxL(|y)fQ;>UVOOytbp5L!Pc>1K`2JB`F}Mwl zUXdNETOBUpu}8I}KJz?HN0h1TD()#;OGt(@dCXYHHa|SR*DfM@K~v{b>mb32=7g7d@&w~E%cCLl^d+)4if$bb8ma(& zL=~KmfgqFXoJPdUb;8ZjC%wSNqy&Fg;Il$hh`xC%S(bb`^wYRFea{*5vC>|k3cc*H z<`wepvE~!Xl%D^!z;8RMhRNz5$(Y{Su+yD_qJIK?1H>tL*3@Y^W7g7%Epyg}S1s`- zp2Wf>>f5)c5HZD|P>S)DsF;KuC>z7dCi*@wtX|{Lna0s|JXx?E27_ZP(Rb^+Li9b> zoI>I~*4$?^SE@g|uUQ^>zcod_yU2*7G*``lbvKXc=Hgj$Vs|w((SM#VCNXZFFE%k} zo-Z!3dY&&nv32xa$UuABQampgmyBGJFf+K@1+UL`_`dO$#uQEWd{LU}9` z%%J;Tlb9fn1&LSEJi7V6w9Gn=vy(5^4 zCW8nKnNApDM5eO~p&{Ejh!|mI?4Ssvp`2&&5ff+E#*5^9*$G=E$Xucx1t%vV|7Gwf zDk$GPo;=Ov=67);M7&IAJAzEMvoC(uEF1o$HABC*$T+5SsU#-cf-_n-QEmujAWI{& z=O_yaedr$!&`xjbhGNo$73r{?i3y16qG#cpC_2)3JO&m%m23_f=t*RR~ea|i&8$x8>2qpu^;f2ROPhe-0HY%%<#9HrqMUk@_5t;))#0LmlKpto2tE zpPez3wU-nu&ZhCOK@yR`i}C>wG5-kabpAj?9-qO;WC*%hJDtJ5cJkz>lPXt`hzOms zeHB7-s&PpEDEgyJAMpv000Kl14l{FJsbhc`cp5sj{|we3i$s7Kg$=|g)6rG_O-(@_ z$zYv3Szw)rj|tglS)Y8l4x>%W5fSK7W~<_}_WWn_L73RtBT|HcJF3sq}?qPT-< z*=lh4khY5MTpBEr>^D`NFM19UKw|m-NbAY3(Ajd-vb)1SIESEL5nN@NcZCBsjN0UN z-6aX*Yv!S8jX#4~VmxW&mo>m!DQSowex9DH^tW0Vq2;qZ2#wDGDtH_xI5)OL2Twzh zk@lHpNObcg-xUgg8@-d)btPTz`H}P$g&eP0wwcy~iE@5rh2C*}3g1FooA1?TN_?fI zD)8JasuWmG+#2-%3EFlNe^*KRy2rJdc^49ZVJ!38z}T%36$TJ->Hb{RCC%Vj_O~N4 zB+($mh0hN8H?IOY=x-N~r>YMFP!~qSNC|p>Oau_E0h>BltbdI{Tico0P*ij(AU|&V zfAs1R+V!I-A0-uP0LB6{DK}5n-Oq2!=5BRHil*!;WsunrPu51?3+aJZ=(x z|B?I^%XNQGRM*Rj{E(KPa(;4>8o>D9wxyk3rsc2k_dig|L^*`$Bu2V# zNjZCeJb%HQFbCBGrEF7*647tgJHCAx3O%kFJegMUDfCoiAyPPoxeB^oqyjQaRRn~f zZFFWqPyxEP7rkBOX_}n}0)!#|t`pb@O&ridU)P)Tq0D7b;7#1WPcp42ku_%)?W@Y* z2tMD&P(k{+$LfOA=o5`n%XFLCgRdX_^^+yUP>D_hLBM(_VVEv`|8mug<*tVm z&5hv8zqi5^#>it(sF5i=5K^M#x&M+z;{pA8l{+%}dkX$#iJk3jDV2Nn;&kYawewT| zP0~0Hr(>njqXtkJv)p~pPk$r0cbJ9xHZ_47-uyoULyWH_pC$xY;-GVRWqtheMkzrD z(*4sC>*u#OHzZF7zjpb&IDqJt;pMKMmNXK$R_}NlS~RMOmcwa zv-2YWT?kczrXNWv4YJ2V8n3?ZU)p%-8%F;Ab_*oxb$jx%Jwq}yUmkkyqvJ2pAd_dN zcdsMnQ$)@VN#qdNo7YSpQ+xo{*?Fl99!oNNkQg$akpfQpL?;ZF{t~Ik4 z>y5RQ=QDg)E*>v7dP4uM`Zg=K-0zKz5SMMM;e4Cy7m`|E2EGbW5sW!$f!38#Q>}6f z7fAY57v5+P9ic_u;Qrb`neMa??14OO^jjc1uW?4=U@BX$Zja>lcj0z^O*V3nX{x%P3Mz4BYHA6t=~&-OlgWNjWYeXgT540r zRlSI}y19n0Z9!3ur#Xp>oU$)v!Xo!z~xD)>o&Y)vWZ7Pp!M9cJ≪Fi zX;qY}@{{5{+txbnX$_Wf_f?On_Jp1cqXN^IUp$BL@?#vJPqCxh)CDV@;TPNc%`u9F z5UuuK3&@&Mi~6;ayhS=)&aWyL|Lj(FYjcdL31Z$YJH-ON5+^yUND;B$9(NU1wX@J* zm`rpRMc_SVf70CfvW~gQpU|p3b&;GuImaaa%s5UMpd(HFiL5oOqo<^3Wfy8q%jm!- zI_jqYkRu)P8H2{XPH=7Fd!p43nbr`Y9M;v}yUwj;8}$<9a$8c%BYbH#*(n_6ZK*|$ z?Loix*qzIAi!53x0L1%@QRPljjo{{8RYcF_%1l+E;b%T>N&0yU5&A2UkGgYL$wT?9 z=@@lh+b795KH9X{ZN%J+Z}nd(HYoKgvVwvBk`Ig+HB+)&xb0uJ-C6hY@|Ju=@VnO* zm;44nTB`w?2jDiSnr>3l^TmH_Rtbus6Y_2QT;Af$7Ei_v?9r^ZuW@5V^H~RTs ztkm=R?R=e-$3U_4@Xt^C)d08;2Bnlz)1-NN-QG!q^Oe@q!9?u~{-8K}Y2xetx^w zI_7$D8oUS#U}GaQMm-;Vdq2F2bKzeR5J+UVV1pT>`S7RM-&n}B1MYg)m218`Dn&M> zBco2qpcUH>?3@Afl1Y<;VHN-@o1hg}!VytgyAY}IevN*MAI6Kko0UI7(fX|AJsN8T zF-HU}N^d=q+^z^LMy=1HxUo^jQkgY~LWH(_z=$4MZJDsVI7vM!P+rp5p=CgjtXb~n zuc>5cn3dq-9I=M1YiL*7j1qFbGGsJl()T<|aHm(AU*ar-^^CL1SV#m((G0@DBFI#j zY5%kKj&byxH*X|7LO0S4E5*7S(Q#~hxNEAT!gG6NjK}*P*)0#Ur)s$<(+}rw)CR~v z*uw6O8~qvk64Pz}EsSI8{p{2}tl0F{%L359!}PaebM=<1%;yJ|wlg4oH6-O{>sm!>2YzPE()Q-QpJT`;sKx4N`-Pv29`k_YF zeiO99WpLjw?nZwo%hf9*=(zBX{0g_e(_&v{`}L2XT1b-ISV8p#$FF&1cnR=L=Z(=K z?P#h75Mu{xKbzOnoN5C>BQsioQqVSav)K8tIZtzXC{KSeaCEKf9s80Ed}EH3aIfz< z+-6jE8Ez7_;e&0tzy(^k`f^R8bfPzimwcCDjwwHHn-qHL*%N(aE%MM5LnH zx@do-mY431N`ag$=tq)drVN@>cQsc^M>4!v*UZaw*U2zMjD~XxyEF+$<~d%W;LE2d zn#dGc%B&0O-u|&sMqUjkPv-eaT5GCWT%mZVS^5cY(ds~}Y$)_M4B_o1|YX2lvho^gI$39VP#;KGpIHUM3$R5Y)8aHuh9YN zuR)Ga-414`WA>JBz=Ek7GWLyLzx0+VWT&B7=$b3nB4Kpj6O^85b7;*FaNdc2Rn2zA zXH0X>^4=6o{35JaevF9+Uo>8`^DJl=K1uVCF-6AOv17xnvMvvgj3Ou;a7TJezo(V4Rm zNG|X^3=z~!HnN%-&aA1yzvU!>=#T_Z_NsR#z?2@ygX$(|=H`$?dutn&o7SX&e2gfBl?6rb$4N{+l`Z8+4?_l`tPF zdN^cLryXyPx3jb3hq?aLdMk0iyJiDru#$P#S8rY?_7;iYFj~je@IuusJ#eVQBUb?jnRs`|3T#Y#>C>*9oJQk7(x9qL85(c zRj#BulgA>vxUOn$JL5^4>`GD0>JOMA*=c`*o>r^x*r#h@Nyd@)G}BD>W)JsiK}?Fk z!oos8_Rz|9*SvA+3yjlF!+Av}KAWOU0Q8eztPQn$AXs~mf}FWoKE8<2Daues2a1fQ zEIT{AOGxbMRn{{(1@hxz39=dOaSZQlhfLBI)bNb2y+;kG{`hS|Ffj)Uez{yV| z(mG6*fl0~A{{C1mD73@sxKPQP=2m=5@uL{C>c<6<_7~NlQKUqHhrC4p9(UR+f8p-j zXYR20@>m4fozENMx?{1wef8K?o*45k-QJe&4b!4yeX{%^W%T>cPX8oqhmN@Iz!I~* zr#xn3&(UJ8w5(5#g^AgZJ|B25G)obWtlhI&S`D6JRZ`}+qoCV}wTUvtF#cB9tbH9< z##n1>84+V`CsdV>C*&nL-_kx+7MnMDba`i=I+p`_ehH^%0}XHZ?9A&hU!O z8TZ{B=A=M@>gwfrIFrf6$+`HB)BKh*4+858bAu>)cOJy=B27ZK^nohtid;O^ihLq% zPKu{l2iKN-?L(`<`bE+4m^tB5SBL$tdHLeOqnX}sypT0(EzpTyH z8St(u=gEqc=`gMFkC<7tq>zvhE53Vw{8(=Cm=Unf!g}u7Ki3Ljyhc!FI?M>!=SLUPVRbK*j*mi!F@7@3xjDdfzXaNRk!Az0 z(QQkJI^EpFoZ@l6cu{LyvQGVbYFfeDg0WQF!id`N9Tmp|{Eey2D)^KN!kH?}B7lVi z6d!w?-E;rmLNL13aD{{^WBPfprvGyvmLNd8j(>Df#YConQ<4?`1&c;5q4uBFas!{c zADPFz!76bS1Pi~aL;erh05hQp0k>|N*D`@kCmH&u6dVIx0PY2ewIHE>tnA*#KV%Rl zpLk0i>LBTVjp$Rc+VP{^nJA^_)8df@$`Fp)a+HN6Vf3v3fSkt=C}}o;K)H(EGtm8e zKY%P;Pkwqt@c^i6=D4Pn?ng`{qCGA;Rbva@%hN$a>ql5)jCyIRm32syyAhYp!Pz-x zgD4c_QjAUDhnn+vKTJy-R3f=>At^lYyy^ZKyEs~X z6{t4%1P$(*;v%mTkmGsJxy9YR^H7n6O_se8R26@`jgc!JoBz%B`WR@hryG)#q+Mg} z=`;i6ic%8u7>XYQkQ;4V|toxBTQ1DxkcX#JAOP|IR%atOSv%C;+2K!OfNxyl?yHv%M&~6-O zZ)zqKo7z?c2eCd}oi_v|x@fQD!eX^O{g*%afr?t`>c{{%KcF3#K_9P0ueo#}MOA2i zB)<}mPBoJUg}b{l^K-KYxef8K_6FB-VW5!o4lDZRzg# z!a-zMNI16u*>m2i3L{8uv}=RJt$oi?8zAYwaaBK1?_L^x@A4 zXC)-uvoPOmH3&!8?3+`3?HvY_s-EV?B&+SY;4RD>se!Q7bo#!M zl(*~Zy6lrtPh0i)&Z(M%A5>XN>B}(sbC+M(Z1>vUhG;?Q#Z_CPp^q}l+P}26hLHKo z$LN18j{J&iES;cs_kM#&WNvtnK}R`XZCg`Nf8~S0U*ItA5!qFubLCRL7?#_5b%H&{ zlcNQps@;m{tP2T0V7Ew&PuXa@dNnvDM@d`rC$GpJ-B6xd^1#TV0=!%=q1R5 zsXKTcVo+^t|Erf|`^OLXkJ6P7j@?gQgUSXTRz}_eqhb zd0OSFAjdHRG$QLt&HDWL^YrKy;jpdDHn}n<%?bw?8Nm!pd7M?I%r-3o!$e_uam76> zNMxTs?>gULtK;wAcR^d5*|hlanM59Wwu=|FIVsKohQ8M+Z=%hMQXb^& zsjFHA6N8Aph&u*?k|m>Uh|*3Ie4tI>ce`O3`s1aOa6h~>&rmb&JStpzo1duIJGbuoDFKuZJ@+rCpt zsr}1&O$YwYK7QPm(wb@y1a~WwRvPqawu;hn76~B(%j4YO-W&q!t+4W_sJR3`)nd_6 zvw8S5EqMD*h?b^WOKtmtWd5AEi>>A}=BS7Tpj3n1O^+7)_VNY3uvf)9&1&}N-dALB z*Qyq*?UAHN%6VEBK9^u`oYu;qU6v-RT+U=5MA%j#Hz2o|eqWY5OFJ{R;-a6ct7~p% z|9rlHiC_QY*;gyCIrPIb?EA{XjQg`_0v7s88u?{c#!UMQ14~S`I_1_^S0jzfBg)0# zYCIhG+hpvjHkqQ~nxL6gx3^E?CkT-;{w zi?XXhjy0yO@zY4R(cxIQwpu)@sBDi}-mqv-N!t*Sjnhc}D^InQjE;Fy7L(^giT$#f zk7BrQ$+;4 zUu+@TZVFCV?7v`VZZzau<_#JUN;Fx~ri=Am;x~D%8JR4TvdM4pwup9#T)ra)fhWqn zG+vwlcH=AhAz25_Znjzapr&n_e0jf>I%rK_qTptvx5*i2=v3v}tGGpy=UJSvU{zWR zP|t_4yj2gY>?($*>Ne}YeZv~8v5hk27^QG82xsK1D}moHS5L+kTwJ>})2_QRk*k=U zJJpu>%$(P&BOTmpiVul6_04pL>vlE2#AZ6!2rPEzS5CErXwp|!p*dO&PTQIH$N~fa)zQ6}xC+?c!s@laqnX*HhaTcX}^}YF8s=YY?#rxXRnqkEL&Sc;H6#WO$O zak46ZGQoV8#~`6fWPqz#X_PG%Aot%AY(ISEU!J>L_BF|M<4pVa z3kmgA2|1%mRFOgbInPH)Q{d7;;~hN7>rL>YSY2h%_cc9ERh-Y-#ai3DKRbIciMyhp zc?o4b)!W*QdqCW*PYKtLnALDt>_RbXZu^RV zmS=1GXKb{scB(TyW1y`XkArRgQ7(J>cvrKxhb+n_HfoUAc6@%gwiLW5LB?vvr?R+E zIb}w!d7k+X1@ek-HG*;`ekMoj z>2S80YK0(geJMgtIHPgZ7^-{uU>JxiEYFX10H;~zBFwxOFv@h2_gsB$#BXdCT*-%knMuIoZnw{4|5 zs}F4Oa?M_^3hK==-gGTy(WC9Kk{gY!C3q(*M^ni~2=~j} zsQM0GwLdYtC#Zf1+h!!H&&EyaWs|VzuhhwA5T~ ziftci9_Sibs1+W`9m&5mJxzOI*Cx1yP+HPZCA#;{{jdX}A;VD3y8djt_Lda;Z%;WK zZ{6r^mn$bpamMp*-0(@vr*0S@$N(=_8*++ddM0Y)46?W8_0R-?=W%^BvC7pu)=7Z9 z1K=sQOJyh0pZR;Gs3)7}eJD%JlL_Vpsj44hl9wjj9uN2AQ&!F1v9;8~!bJb$YhhEU z6xNqlgq@4w{U$oOHz5TsnJEB9e*T(|Th_Jlb7NYInLfPfL*sjsezLoopv)lcEjENd zGo0H43hDx3A-eSOVP8NdmK`Zdpa1vYzp?lncXDOfpQRK|wcb3eMm6)AOVrYN*>c^m znJE_PBfO&mdLZNDlfLjA)J*Yo08H{iEb47~zrZIYH9-5ZAv?Mt$vFA)Wu2J3=LOI9 z8Vs32(eheM2$xE7@!X&N3)K>Y1Ss`3t3}{#MvdryT1-OWT&|FUO+^^GbBP<^Mz9>+ zja0ZKXI8v<=ZCKxUmV8GH|cRr4R?F@9}YWP5HxgIiMWFp>;34*9^mf5_vF^a z{_R_^L{@*%*knuKD#-}2nXMZZq+_!Umg!wm`n*T^yA_24w+asb8wtH>e;87saSyK+bY;zU-EpwtTiZ+NYNtiDARC$*w;p%<*+L zAGT*qFY3KchzL-Q9xS*yruuLwX7Y8${J~B`L%w2nNx+@r@%hPaeVOmlJ--$|#M&uT zJ$)YcbPPQNu3I$Y?R&LYd&D5{BI#0>sC&=B;^^aN7tJy~m~h43#!bdyQriv_-xZ0R zb8sct8)ztO{8`xikktZQ2Sg8BU!ci5E%hp6873cS5wtCil*Q1g?p|#BSXhCLhI#R# z^rRPe`xPu!)7VX~XkGKlqNZu^02JhG-d z?jj5#0sAo&A#BNjK9De9s zE1UdEVq6YX0)%T5Sd-Nb>{X;2y{d++k0$q^?d{We&f8Sjz;1U}HEHrF5F3*L&)twx zRknn1mK$yW9b{jNgQm`~tK13dy(2 z_q~q)BkR56*>2na|0*4>7FVmSR!cf{ShZ`WT8dgNtv%XWwYOA=QEj!=O6?ic>@aG@ zh)|`r5V46UMTiIz)JXC>U7!2@-1qnM`_IEW58flmc^=2{dOcrDu{xrTXnin+N{{<3 z@4MKSmE=5O)Uk6DU!dhWHZh>ynbl~}T?i|1RVF-kDxkZJ_*;#IPQR*LL_V{znFt8Q z&wFD$glH#t&Qtt&Zn}IM24xQIraNS$jLW(2A?DVsSzy{`qpl3ORF4z0!9y0?Ld9RT z+duCST{Ds$@o`hXx52Xq9V==YfJfv5Q(HC(y2Tj=IFCb>O{zwtiGmdi#r$#HDUV){_|8id_rkXU^4h&rUxiyqWB(5uy zNdu=B8DKL#7Vw!q)01iGm+2K!U?SJro;Zv7{83R0&&=AQ&it%F?tWK{)Z`T*g+(cY zpU=ZEh{)lWg#VfZ?BPfXOLDz(oabrQFH}1i2M;y070^RI09QcJTxR)ecA=Rl4e=+X zi_KC1mQ`ss=H=?rfy@pkK%PbTnMqpxm3-(Ya8k5`v>2+}me~}57w4jDmI;rVgCJdr zz-XGpZ9I1ucV+}cuRW3fZc?&DU2Pz-V)&q&w-`s}fO>y;bDA>$GAOym4B-p%Nd|${ zDz-D}_Ee`Jtpa34Ws+3@_8H&rghnjNdM=$Qu7i7*Al*Y&s3p?=EvE8eS?VnikV5k> z-t61_`J5zNN`;Co;`G|V+0nH$_uBn?QS)2oB{=G!@sMigpAgw;T>zn0f#bm<^UWF3 zvvntAlj<2ri|EmSK-X8_5+C3)lqI16ttsF%RQf%>9k_)`qF6CN_1$K8h@E)!=U;F4 zpM(DSb7(i2B@nji&@tmz)?bZ%P8_6Zd2YWk zo*g~Jd@C|+CsVyv!K79%=7@TBeU=O%|c-RU)DJol!GRehNPnWfaJ~y7%Z@_y@ z&!X@S?E9>ppgeL;ZmpO&X8%PI!OL=iL49J1FxGRCOT(3(M6>lJs6B*@VvQ)20oRkY zy05#ZU@*K@%yZoo`c`6NXEUnUVL>PvUNG{4Z?3H{kQC*S)Z6Q2TDznmc&3laoS!?? zwa$DXd(Z^W<%vo$DI(ersnRH!sJNqR^A7KVQXq$9eLr#k07J%62}X2(C8dh|>1(IU z-pe!k-xlHw8;+>*J{=oyH>ol?o~C8%y}WJ99z{6!RRor;H*{w=>gu~adCsoQ40c|Z zQ;mKaM07`2<=4Ps95s^IgUu<~fosM{Q$U21zj~hyJe*<>1NJR7IjTAGVNEt?2&iKO zz@mI#m-tG_)5}Y{U+)cBOG6`dy4h*nj21sR7q@|4Pov-v1JKy&P#qL(|_9bs;jg%k<3BMCutlHSkZHcX@P={b&Jw z=8*4Uv1DiomqXVT!Sn&9mi25ZI@|qz#HT9LXzmZ5!xcmiD3LYFs;sLeA*8o0@sl>) zF2|`~5-9h)|J@6y^{;E?TDv`CZS1L^wmo2&&C#vr&6-S*utk3SHQV}voiaVR(J~w;{DK_~JE-8<{Bk)H>NumilN9k{#!3alOC&KP!qXz+&oSr$Ip1FvNLE zS!8ry^Sm>*;xHt1y)s>%gK?uhfRU{zObJnCmV4SKZW@TW?9J5#@yN~!qXfHm!Zhf+ zdf&Ro*oA=mFon;ewTAUh^`?H9V3+8tV4=>pE~@g|w4ht7^muHL*gHe4XW4<0DYQ|f z6Hb8Gg0p*`Gy;*m@?~xhu+}Ab3hYQ&z}(|b%TxcJgN5TttT%v)A?C-X!)is%E_!%H z3i@Qi<9R!Y`A4fi7=ku_J!F5yLvN<#hLw4Pni7%q|*hNqt4gQim6yB zRk)l7%fu6iT>m@=i?!vgY5LttgQ9`Gz03-?wuad*{L}^68`y(jG@9dBN{n@Ir8)&; z|8j`x{27%*RY@&&z+>|LZb%XF(*=t}QFvs^#C#6&w^G;3_r5kfSxS+T2M&y>M^29l zc*+~^#kz?+%SJx^nk2}0&AUamRy{KN!-^vN$@2hSdj7&E;CnrR^)h@^TKwDC30M27 z+_7B@IA;Imd9+&nOpb{N~HB#>QGd^d@zNN;dsehPm9%L{q-f0ou*q zWQ+=Z_JA?I8xq*x4S&!X-Hs!IW}Zd2tFqt6vQsq-jIs%h0P zj`H{Z;Bl()KNt*@rCp^@zYTE-b7hA&O?lnW`;N`FHkId3X6~ff%I@}p?L*_=RM?$w zHOx4{v!zI@dzaw`mA%evjZFZHhu{aqG}Ex~cKehjU>lrGB7V0Kl8wT4Qvb9Y%O?}c zJF(Igp-$WISGxvfMzt!>7s)Nxm#(YDst1~jr+@6b3ij+gtQ(Y*Ak=fN$2JyJCpD`y z;I)^Fs3QtpKrP6euPy8*bs4-btda=q*0e0y> z^PO)R9_V+SPPA5fis4Dij&}d4-`i`Todx^TCS`i6IB|`SKNpq?k#2kArvDZ`!td`%mo7F3#TH z?O(rp{aRMQN(y#VQI|}ac9)DrO&^Ilf>Zal`MXvmIs70Zu{(GHiV!RGM541hSq{HQ zt807>^S@mK0DnGilf+(FaO~PNIl*RnGE}0;J)igx8Zs{Aj{5z#x+@I}xe)Qgi08M@ z6yJ}$;Aj>BL5`C;ZCv7+UtyuRi)A9}nx-Uh|qQ?i$MfZA`1*!YUg znI_4drn-oP$eOBU$l1z!jGU_?nHT^^ts@Pt%cz}llihgqUZUyLSwEYBpI-_btJ@`h za7m{}_0}pyuF)4q=*86HwITNga>-1ppJo;9`w{-%pU)oeNW8B3{sW*t{_OCb;1a7O zID~9ZLjtycZX|marW`%_#^IN|Pcs*Kul~Je#Cn$qc}gv$@|Sk`L$-BI<3pOH-nY9H z1c0malMg9~%jR1zNY{0-K4p&13pD_NtbgyoymbODX8E-j{%n?o5@wfUVg?mVW=^C) zKN=DP-~uFuVFMbbfmM8*&+XXCeYm!!*&@RLHr6^$NH2kzG3I=dn(9hIPw<$zsPXjc zrz>fN?2v5b!-!Q{r}3?6a@|-0%!qes=>D#D2VSr`XUl%F09neoG+4hd!D2l7MLoX# zfiiP^Ol|%76XZ*bkU+PH|0Nfq?2P~nx5WX6jw_(BO@}&?x3Jjky=>emy|T^lb3YUX z>CQVl2!ve=@zwdRT0V&Cud?lqGtf#NlzgjEw|w^*NyqMObw=?mgywTr@<+mSfQ4`< z>Ce!|Mw`eB=tyDZA9`LrHR(4Z6U_#3DqGH9p3tQI&^mKc9%IGW)bvj{gtdiOp>k*-74`k>c!5w3@K{}M&e#FPxQDt}?VNP$GlGT-7<%XR!p zB2K;E$Sm<3U;p)uRIuV`@LPk3M44Ce;(kqjxaZxFHas?F$!L{wf(K_VE!x=Nwj!hU z^I-z&wAQImG3d@tL9iNIU}*89qpH%>v`~U=?3ZTz#L~iPZa18!ij|0YgXs+TAbjIPe;n3yR zWFhVe8>z42Hn~}3Yc&jUDYz!%VG56ey zwI{`ZvMs4eq}RCA${7#6<~8s0ttMH(n|+&OiRKt(SuAb|8-Z3&3n0R&UN@fXvyw~> zOHf-{aF?j{n<(^vy$?*O6HHEX+AEk=MN2d;=(#(am7Fx^%5sIpaPQ~#wMW7gtj1@^ zwG5ARLP3vZgk7tlZo*p!kQy9T%0Pz>gO%w}8M)`UwD*G5$_AaEXQ##5;7Q9`sV_4} zL-Vhhsu8gL%muo$-(aT={O)LOg}$izlU?yrx3g!}yq3RlrA(&oA9;dCTy0NLb*-aI zN;iHD4)h&q-C=k`H=-7+Wk88H7L9}`pHjH0)55l@wb)HDz{bL3g0#A?Wg|^Q9Dli3 z=RdeVHdr~4B=J%^;qKN+%x&2#pJ`g1Gmcp>1Er*|u#3)^UYLQ)?h^Ixn5ZDVng(q1 zqll&PnkcmR{Ay?7wUCF#M7}jtB5tO2BM{71@CIYfW@cu==2NKr?Goh3knHuju3ogm z1?Q2vaBi23Dkz+cinX{y-3boP3r;yI?;5wiGS4h#Z#Jya0JV}pCWc>%I*xW;Qgs&v zF~_Uj&q)qL8O*8c?Ct7(UPO9}kPX%&j@k48r@X*Gyj&XerrY*Z*IV2g3-}r&*cQT{ zC~WI1h1{3X_d)C-*M#V>`dpeDtE^43ja{~YmGV8fSvf=K-yi+EIH`K7`j?l zue3xlw0lgnrmr(e+~axP5uZO~@A7x&9UIgx1DFT@+7*k73(9w7K+=RMv}^sEBeb%B z<}+Xy&g%*zFH5B`@G3&b5Em_xC1AJpZ5sIYeT8~_>^ntf?g~^9N(Km%b@euP*k?0m z33hn*_Gc%jx9ifo(_E}Rk`sg?OBkb zmAPE?M+ZVf^+>UU-R`_sRQ$a3tshNIc|iq18U)6DEz_xo2c8_oB_HzONu1BgNs)1Q zr*r+bAXzzR>3O`Uv7Qpz`P#%psRF$$fjxrR1Xx-=`A z+%PB5l!3|w6?mwmvVKHo>|r+QhQ9?%TD@IKBzc=0fz{-Uz(viyFxg=CL|wY#d*{^f zS8RpnfCfATT@qe7`QdLo;xRk?LSKQFWtx1or`8)kOw)r8h8RL3g^Lp9~KKnHFu^bryFy-ehaPS&|G9skXZl$H??3fhHBliogb9 zB&f8+38F<5tMTd_MkLg-&dO!zlbEU!QM6Wa0du11N6y|17jHuXysv#xlB)N&ap(wzl%9djyW*ieOcFV|d;!P|o{Js3Vjw!6#hBO$pi0tA53j zxOyAFVAr6Gh@4ISx;^%Nf|?gM!e$_{M?KbsZ@3YNLB4Zrw~$BN-^Pp(45ZRLD$%9< z$pD=H=4utvo&uT1Lw_3&a@DLMpP;UD*z?pemPSW~Wf%#2YDs{p9bj80lmWuZvY~xX zgXW(^M{XIYgT!PdbjsP-{j4Pl=_mT<_5r`d>--;VFu%4qW4Cy`;WB5s0iK3-?eb~Tv1MTvKFW_4**H*L+#^k*QyA7F4>zthY;3{EAd(po| zfmI8$S>~mCA7!hvtm*m-!Etn@xr1)T6dG(20h1TyPk}i zR6h+1UKco|L24cSW}}Z^FYFwWFQm_x(cEne4N|GJ)Rk+TwYKi0njSO}=boku_0{W# zM2wej$hSsGQ0;J>vguC(a( zammnQ-M)*>SL3evtahE++}b8BP6suf`zU|##ztr63~k1BCO!q$9crn;4vVlkNI5L{ zqIZ={9}{`b3d7X>E4Q-kxmpqsRmvbl%efHi@!*DHftO-x20prA{W`lT{WgOqdJ z5v{JFk;~+ifzOTA&;{K0z(BCjTTG3_ZRI&M9yF{e{jIj3mJq34%egxD1De?G?R&q< zo6d_bL#e_h9L=4c^Mq=!y#-NO;XOY?OZwHv=Jydjqmo8`f$A?lM2U95+`S{Tum3jZ z-!8vHq_D0%A+E)Ampx{X(t$%;#*SlOX|0(d7jW(+fl~odEUR;)ch;g6okq?17YEkJ zm?M=0CGkI`OljwZGkDqV0)Zxw28CA#ocpR3Tc^miBkCyAh!!RFNV)35rl_RCcob%1 zK8tFQqBIZj?mZG}YdhkYxWS4nr$VqPD()Gd*fY(b0F-%{pnk@bh>mywtqBJx^p0S6hUwDT>+PYW8W%zYaK5G6zaPjpydNV;#|@D=%@ z>S{TNxW7BZexW#4o;nYC1Xxb+@}A5Xh<+MgS$qxI_jCeuAhtoLbhI5_*9T3s)6%0Y zZeEF+BtJ5EG1A^}y|L7`i|?91S7{#zYcxoa&mChy%QI0oye&?M+nlpdsO(#Px@~-@ zyU_hM-6crPZDF1MBX5<8jFQ#CrPzhD&YdC-G0dv%M+EUu``qwS-6Y8PPfMISg=j>V+*gyqDq-5gOV!=Q0YSuIv*P<(~%ie<#L<4i3|}=Ryhs$1$WXsTSsq ztj^K-b(Z?t7I85X-ZijiL7x9`8fd30KKqkw-NnoQy3g*VkQ8?M^82;S&hC3%(8@uC zyyM-j-n`C6umfw$r5O(TD;ft-RUfgdlj^JP4dt1HriA6@yIMQqHf>n4t;}+u$5!5+ zscO(1tI4Op#@;?W?RR~`U~Wjddu!)_xySRYR%GVUq9#Mfw5O*SL7D{rwE6NCrMk;o zI{Q@(#I_X{lF?0E#sVn;d7^l`l-b=`(U^V8h>22$obLx_eWt8`u{+cjvHf*7j?_a^ zA2}wzGYz2=83~)bL?EsLWsnmSmTEUMH2ch_cjtOF;mq8cM~HQm#e&a?)amD}lR1X! zPlE{SStQBS^+?5uqIdQDYp)ee$n~MKpie2&YEFy{z3y%_HYZHKozBA7_~*M*f)8~g z1u62)6*U0kGin!`afr;yC3Pw_)SLRJY~ltG`klGk1t?Oy&U^o45KU!Ny64{^*5SvF z?COgLwI#P$@&e_yDU<0oR`Z$-HYc4^1A@Bt^~@Gw{CWen+EQ(9JiRXUgR9ddWirl& zYpj_(5q?`sL-IqVaxmis_o{n*Ii5qZJDcMpaA&>UU*%^d8El#ob%ZCwc~V<4+pjhN zehwW%2-xhl(knKroS5MFOlcsmxAMzomJ<%?VVBLGN~HU#p-3tDnUP3g0qnDF0+)*H z_3TccM165TA^5o(82j;v{oX5Ergr{nSi+Nnv3=fVX4OTe&ls4dA9~APC~oN;JBh#6 z_Uh5JeHyl>W4Vu#Yh_h`aP5<;{t{aAFsTS-v&SXOi1K-}LQKgTqubb+&4Jf!xrr?? zff0F?N^sMQgF97w+4-`9C*m#<;j-8-h|t=i%cR{rqm1 z)Bdx@t~~oGf9g54y3p@k2!g`{n|^6eQza~u3Ysivx3!`vSyaEY8xe=5ytFfy+%-r& zrWIZs&JP+A$wDqm?#;MkM1p^{r0!Vc{v(RNuFi}v}qVfry&fXx+h_v|(FNPLt z$YeFf{nRy0J&U_Gf|gk*HINx$5g@l&TTTfhwnL*g8&D?=4ad@jn-V2#FKitUt5B;g zlKr~>?=^|LLW4{1xLa6rC%*S0oJTtR<@Xd)NDh0ij!nnko+B6}VLG49f7{Ppxd`sO zM5{$ILbtV&fHC)P79MjXC4V-~TX&M zadeH6wK9{#IfYwn{z6$TO7>A6p^|{`c*z~YEVX}093m2AiCATObUUd~z$eqdvp0YNsVbF%VT8l%?ntgH}wLyC{tm1?9 z3Ah@jWa?y5?dLcTQ=VeyS~S0Sxd)7&h9*IAXwZEWrpsmuI}IjJ*Yi?J_2zcB2c3ho zM?yo&&XEk7J*|3hNZZeapm#ZKx?R4M(;IFfta^tcy|(={VpCnWliGzI;CjXWvu$hs zczx{={q{>x<`v3VcYbbFaX+vWTBy5WJjZt@753G;jGhX)?`K8HpQz2GD84Zu7;vs) zayE2?hL|>U1e%=?EKz;z_272y<}Y0W%3)Bxu+_9%_rL5%H{k->D`B|>5fJ4NBVNqD z1Ql$3zoY{fCn_YLD?s6iU6`wei-Zfy?Qt9O-klKv9d&o;%z~L{_pcf{L2zR+W4TvLjC;t3-%K` zwu_h>1h2{~|IOF`Y<>1&A~MYWsAr;PXz|~OQ1hdbi9gSsg&*h#!|e|`knRLSC|e!W zXz}N-!3zc!B}3zN2(#^A2HF|x@YUEJTgVm8IGY7C`KO(6VSfH&{gy_U+Ehu2Wbe@7 z>V~Pas{NTt|8O~g(|zHeKm#vvz`z>nJNFO#_#dbDS?a-iM*k~I50~8qChNP2{Ix5T z%C+J64-4yG_J!|jAYY_TxTrraDfT~(hJXF^KUoecfa9F2!A<-FyH^vmjgcm16isI$X1hc z=jr%AM<;dm_WutDLKs}?O4jxNfV|I&AG#OtpG)lro_T3GPoa)o+*j@;j*kD0eE8Qn z-#ovIFxh)m&grAuf60yi=ji`HObmBD^E#Gswf}pW;DsQ?@tfGc{*U&4a2D-h9q0eW zUig3Q(BKKEn1h>j+0%OA6}Yn>{^OMXw+&y#_U!Tg4`KG#h7TV5KjgCS<@-MUPh8<{ zOv1@S|7CPg(G45@&ne~J0w?j=E9c%ta8`gSe-Hk@ci~^o>|)>|SY42J7Jb?Ge?cxF zpalf}ajDz{=Xtv7`6g2n_Jd*#MKgiwW`Xssgo3Ot>(@DR_0SNrMu8FFa zC)#6FDdOUsqDT-@=HYJHYXo>Q+CZ9e(eBZHp}gCZiGYl@LK9F=k-ocj?RYbZM9c3a zyKJ0t-cWeozV`_?I9f$AOHuL4b^H9pDEA)g;%D<+ljufll|WL5&R%$w0GQr2A2B?& zx2x2u2fD-bU?mIpOvMEo0WX<*5?~YP)VrNXhI5rU$hBr(tTyOKz0`&V5(OINv(?+V zwYw{FHg+%Vw0PpYdsL5k!bo8fL&-7@v5c&Qt9Dt>R%YU*Ckw%+D^_ne+bzXN(_tzt z$9{=1^n1E}+WuZ9@07sE-SL0-0z!@^0`sO`+I5rJ;GI-eUL;~&y{4cYmEEf9r2g0O zE+;~&d?7X69vXH0>Z1afGptIe!G!kS8`=3nA*06RL|^Rvv%0f-t6i_pNz>wn+%2Cm zf3{vSEI!j--Qp@)>+@1N-&*a$ZKKAGol4!iE=Jt(66OzDpzLq&g8d%VMJ-Nj+Qg|@ zDOI|k@Pzy5_4{I6svdrD=5HNkK3NE|%MB_xF*VKad!GIK0_X2HT3b3;r2)4ia3G%E z$bs1kIU_@RRQ)r@AC}}R7K;Rzntim=8C#-3h5=~~;Jq(o4PDvbTf7FGqdIEa>UP&}U&(SbwU@2|7Yi%Y=O(^f_ubnBcW`WeQ@)~#v67qPXM8>FT*49>SPbwO&r0;|(?>X7-%iSqQ9i?6ReIX|K1rDB z7AR^ehPY5JV|bLCy!l+%xWcy~y6H`pIl}QoBS$mLrfXgtmXU)j-I0m*MGE~fn5 zb4wU9sFe|1k-5ns5}}c7XMgWr)~@TtG33U?@%VHV|6(^%jjc)X$kU{Ib!O_@DVJdZ z#+kQ_KDu$&cs_FCKPD&{62g7p5(rV$-pgSNu}x9>wa?kCq!w^~hr=v4%YHiajpgLt ztXCh3Qwd4G56QcM6_%rrl_jy09jSO1XTGHMrviey`! zC$W|>VAIt8_5|+o#Dq-%sef9BaHn!+pcHu*Lk@Ji0~Z#}YMthrI0Yg#7S#TB?g7u< z>Fo#~!|ICSHCFZ}*wHCE;W8yneZ(>x@{fYAoyR3F8MRK$%k`yq6C^zi((;EiOOKwc z`4Tqt(AY!$7Ok=8b%133R=D*xD7zM`4T7d>6c#JG)Kj{W4DIN7mbZh)J)k@tC$9i! zSj^&V(uczwtiC~wg3fp`N8p=qkfK*TwJ`i*D()#+i*n|06h=W^XV21SX2f?bL(-R` zl;ZNTHg662Q(k1x3MdKbL1` zq+`1+19VE%4acTw6)Cn+ESz$`mAx{-^Ecgv{s2={2gLuIF70}Tx4=c7lZ<{_8)Q$3 z#`f7wY7btFxJ4aOG@P z^-A$q&(}D2(@G!3d0<0e8ld@$a|%clV0}rjQY^P^wX)am+-t2Qi+s2PTwQ69JA3W) zeWRbAEw}D$0b2AiRNSJ|2j{PV6^LX~0acy_QUIK4@i}WjS|ZYL0TDfmu3^Zd%G` z*0UxqXOoD?!xV9w*&Ehh%#Hxz?Bxxef_;K(D7Q`Rl(EzTyz_uFHM> z5`fG&Pmp=N(XPM(L~k-Nceb1DJag<(D)d*(V%LG}kD{tlocp@u9?|dk|0H#bNzr5G2D!;KU4X1yxi49o zDT=XW$j)J*%&n>sh=#BX^yWzO4OOB-1!28N2Pm}A#NZmcrFj1vXN*d%bTB>k^4|Np z3#?0j?9)5FLQubIQ=O%lp@ldw&joCWr!l>>lCD!S3l~jI`xfnLgH_phWjT6nQD?%{ z3=YI-UY?>BfmZh{HYNVn-UQJ^`PLuO+{HzyidFLyr7s^8-`jdy2gd9RgT*ecp!@j2v|?=OtZ5V<>9jZiH^*ESUh)iC3(*l&_*A4bt5F$-3T4RfwJkcAXwR*8 zckl3Z0J_4r`6e3e+*Wl?G=oMoSb~f{JYejrHL|<+5%rn_Yezc^kOeNBmZyDKUiQM< zEhxu1cECVOahJj(K$I_^f2pORt^M?qsxTY$$z?PzCe=JD<^|X-|-tQ zZEdalwIk8`RM>)Y$-4^op03!*24Hq-bJ}8#eXP-YHtl?Bd-3+0428y*apxZX^>oj- zM=X`nj=?G|OJ!l_Z7D1ZJKEl?keT3tT8fEWqIlqbwM1na_FKQ+kS}o>8}q$DOX8bA z7{L8quYx5pUJ=cR`3$JlV9?YF7}JRg<2Y`+M6q=(7okl0jG+&h82Y}GZT4(V#8;0uptz^nO zzLs{G3;XZ#$&mUtG8Kj_k!j=i*?ysEjP>&DA?Z(q#xFE26aa8`(F585ymeyNUl*XKqq>c<5wcK=a>$l$j&>OVr=|zQjON zro~X3zLCq3wlvX5cUt4eUuDb4ynu#Y`UKLcJBg~i@g}n&lP4aNG}Os8Qi*bzb2En9 ze3_2ef>oG{#*1FvvX9xX?4Oz9f{U9#Fi!>XGP_=@XE!3cZ*&D~`OCHX2@NVU3)0Xz zrH7cTR3-0sWD2LOK4vASTg@>7aC*=%jR z9j(&P_#-*9FGG4r8s`pyQGqH*la`8DjbZhnA6^Jtf72*E(43QA($kiBsXW+iv~_tK zo^1EnGevD)b43nnR4Z8k=xNff9i7=ypXyad+xoe4^5(5!)lN0D%OJfVm7dw^5T>}l z=g*|(kgHR%qVZoM@Vv=d50{5qu8r1fu5B}&A6MJjhV1*J_!n`|g*WF)6HkMjt1aEN zs1n4}0)dXqF^%U44TXUFzEWs7(J}DPXVdPaVL-iQ)>e%L<7|r0n9s~h(MApUtiWtP zZd|dv)0JkB#FK^|bEAdc(3;4&PT>7Vqi=+{IWey2b_BLK>Iy}vGFO~t(`lN@JVr_d zn9vjA>7?ikOf>&9<=cBzB_rFW8cf??-}fACg>cIJ(ZvkvA5D(>opz)xRxu969Df;= zEl>#PLK{=AkTwKg!&&73YMjfg>3FdaEs-uD(Qe$+AZ}-q;YQ1)60nuk&{qpIiXHk_ z!}_)le#&8W^25^XbLpHe=BiWR)8@{y<&Dsbwi9Q3Km-rowPsx_{~bqw`30>8J4xY= z_-kUogI=?)mz(b#4$J`~zBS>9Y>D*>X&^4xmFIeJ9spmys`Q)=qW(fhG;y1%IN9lG zAQOHZ8f3JTcYwd!zBV|}r$a`v?lLKPb8V{3WbN)M>p-~!LpM(EOLAC{1R=3RWo=+n zwkh;9dLi;F20)!v)o|(CSdG?TE9WK7jTA_`?Xy|+z-}&^yQzDo`(`L^dw#0+47`ev zcJ!(YQl*I*mwo%xIIt?CTFE`WwYtNg96ha3VpI7XyDY@s@d}!ooR^u?P{x){;|pwM zTAA7)S-P2P)HZHyHZ}iY6ehP3HQ+$2zrFfmQzE3(Y7EQ(I_VP_Zdk#a~gY-l%(JOB+~4QZ=d!;hV4RX;natQz+B$8))&YBF40uj@UHJnyEs zgrpx-m^{yLd$_u*eWlzDmNap572UlmjYS4*o>y7nh zb9qlxY6A5#HeTS*_?gpK{XHw~TKh+*ZN`t1Yt-}Kh0oT_^5LoMjk3zvk#r<DDt>_=0c)pVYgb`4DS$OfwRQwPjYm-D?z+@_F5)l7E80?;gLK^4{C# zx%9rf8RRJU?r*^}@Da2fve2^OAr6IvyR~%wx>SHmPq5+4j6GG}kYF@Bga*RZbu<%X zh|jwfdvva=&K~k9p@N;HT>&L39x1GRRUjvnss>+t!Q4j_>D%?%7}bQ)pG^L4|74fS zdaD4dWvRy{O8`ECsW=1m+)!oQFhfYn?2f8pvw>|XJc6m`W=L~+24b?Bu>?u`jZJ7K zyL@pZ=pM_WcGMtX0_CJdzEl&o|EV8HXZW?{Jy2|wGBulf_xA>roq!VAlM8J@I2k-S z!y9q%)s=tu0<;Pzvon5&s!&|F5rH7J;WKb*ztGZn-Nf~Jsct$TsIN%0W@Bmp*14_t zh@hZne^@qv?N;U6qmB{dd7GgciZ>XvO{9heAs-`;3}e{e6rswOVH&o2JPU-zbg`iJ z5|L!8P5TqPVUyMMIe$2>+4(Kvd)DlaUgk=UVK<-H7;Gs4q8Bu<$tR>FqMKNh2#h~% z?S#4J&I)~$a1~u%{Uj=2s@a+OH$ux}+%C7cQ>29;`a!aayM8Wq?`f39xf=)O_VWAlw5lji$C`6geu#zC@wyEeZ){?O>V*&kK8 z+;6K~N7^RZ9|EcZLF9r1kit0X#E5(nkUMkc%=zP;cZiohbLOFkx?D`YiA!F_3LlMyKjK#uwJ?x zs&|j&>D!%8g)f;xDNn4dr(gLmE(`LD_0iq}p5{n_{N1E$=hg7r+1-!eI9k!<z5=8$jHu6MY;{+yWDgQY@WjKm#AdNWg5CsG;h>qN3Bjs~5 zy}#_xLatlCa$0BN6(##%4_z<>$o^~bl2`p*9>yFOWfR1Vh&z8=59rb2H&-6DEP5`g z1}^82DCEPf zeLlLtfNfVK4TAQJzMmmUq@HCI){xr40C5Hc_*?|0)Q*I!j<#_Ra_@!szB>`~J5l_QfBV)G=75;2*pD?eemSgL$R@#% zDq+hz$TzkR^g(EKN=6eQo7{nqyiB*X(M|cyOEr+{^Ib0!@|Di4%=8~q;_8dsQ>iz$ zS1UT>+wm(8u4FO+f9<)R{N?+=X!OHM<25zCRauJd>}5o&_+%gQSS@HNdsmsFm$_D{ zbA*1|AeVXgaMvTScmv1eTo$uG?gHXs7nbgg>3w~@!R$cfi{ZL+vr*;;&P?_4)!HBR zf;)9hhW-@X%enVrn^oJw&&&)J*H;&2UsymZ$LjnGLbOBppOtdv!|=hauU=~IQ>g3) zA&9R5zJVDsNSi;=O-ZwQ63QI_%CKy+bKahOfc#Q9kDGAjzwW~u8F@$Cv@00r91PF6 znXAYf%xE{wR*&DC?>--Q-AV7B0jJnfrMT7g5}_iqkGbx)?%jb|1_uuzudBBt8;{7$ z;pXjwB8zy*LXlgat)FV^-MrmO#N^;xwdWw8@!qO;4AK-y^6KudjyPH`(X06=af%VI zR3*2T+TQ>NVE9gr!;Wv0dtx=UUhlJO#WtC#bK{tWGW?A|Z&bG_DMJJisd53XwLH~e z^0pzflmYF>Pu=WoNA`g`It|-*+5_o!rn~-E zK)}#jCR`qb#fCtSzX`e4T$9{O#*i85nzUki-*F}WO})y!7`h4bt9gRlWZA}C5KkmMoXZZJFQ<2V7zUc&0 zRsxdsQ7h^rO?tp^ZnKJ);(f7s=WK2h2M22xDkAHcR}zeOPKt=Q)Ep@BNTk2`i2sIb zar%gr`=@HfOkRb#i(B>n>^%*l(RP4JaNW0MhX9H7TZ}bFhX8T}Q~R6l=fhG?-d8JI zPFwXXVGk^(*ER+ne)c1sp?VH*4P!qxwYGs~mfL70oTyZ}ixO%Bq!tOpD>c!G?l*S7 zD;=xvq-JhDZ@43^1EvHwzZ~_-NG3XjND*59Vnli3GdmsViJ#8Komlr_B<1h|1Ke$; zjXpc zEmZ4jFT9%cCuE>-=TWpOoAMaPxqUcrFrBmh0~sbay3Z=FDyT<2EjzSX z=Kw?Er0N~v8z+x{s(Bi+7_eVNmg$1Bd0H41vZ!phv4wh}fcI+lpDxUk{_Xo!w0D2$ zjdE3GupZ@cU}c7dMKk^ajUTI73n;+SD@9JfIel+>hx~Hh z8USUsZ8h-3U9J4`FcyS$9&a6xwsokT@c61j?UtnDo>oTxLm~Za{-{whv;%x%^6oXN ztP4499dvI+>(8vMTyQHb=-wLTip!>Hmm3#PCSq1Lf7RwOgIuPw0!%okpv>wo9)(+&^ zBGIcciYr4f-32{(RccU>wqBUlTn^ohgNFs+Nt!-09gd=cBxKj9NcXRwtY7a%V8nCc zsRMC_YR~TH*F!CWyW6p-l~?flGTmwHa`H>{yuU6{Azkb zYySrIsVQvnS-gvac|gaJ?b;p<+A=#*JqTA z;+Iiv4Iq@{mwx+wIjr;jyz$EQ03P|PBOmIqG>W)c=GcBQjudFHo6ZK}-F)&W-wS*j z_(8jSi%8fN!otiK>UUl#US0wETxa-TuxEEVvxqq%xUqA)uhiezau7`LF7D=kBe;B; zW~sHq&y)fDF|A>S$n(3N1EmRZ8d~)v*0;{WNDBTpRL*Ry+cUG{hqE*fW zfnghHgp!nD00Jy(9ul_09I$<_t@5+dF@MJ7p2wiHoM^Rp0vT8NAuSw#QG+6hC)P*P+=D&;@wBC_!#% zsr00w7B!c#Np{F~YX*d@h@3?%;KTc7ColjizEzv}krT*_6LpiG=K)u5cIy(RYi&?e zQsv3my_DGduo0!PH*{kpLCMo%{t0BwSpRbl-`{wMg%~1oIw%eb(y0%g8ZN*dUL+bY z_@wJrXSK2~m#>fAmuWd%&GUv#{}YMK_#ilNt%Dg+n>?eDio^YlnM31-xzBI?M2zm7 zH>-E^c6R4=^Cq0yb>uRDYHAG%{s)j8%yQKQSs<>lrzpo_Umbv|Gh{@Nxcf z4=2|XwBG}V{8V(#ZHj`M!sUr1|K0$cU0*6&zB7{0@HgUk^JG0$lk+lMP2Od3d-g!= zb%KX`Kvjk>56;ltSgOh66g6vtczr#A&z{6;G6EE%a@&d%JQA;7ylCG6NVE5r*|fva z1YuNbM zt6HQzvOjRu_4!NqMk$Mf&uJE<4ZgMc6yQv<0=peaP;e)J`Z*2^u(GKe!gTA9AHh`= zh>aI6Cv&OWhW34DJyH`)QCx=zwVJm_YA;+{=0An_9Wwh82FhB~*J^i)NgTk1{W(g? z`azS8M@aP!tKergjoc9W$Z6i8-ie_^dkFyFt10hFSwNa>o4qN-D>*;@vF53{Zo(BY z@ZBDhMEcWa(~og{ZSm6e0VhXlLzN;859#2AD4v_PMFDTE2GO6|aT4|&+lP0qB|j6< zDp-w4s?RsElIk-WRdV;b1zqOL=p{*t%OM^+wh!NF(Zhv(m2ioChA0ME!)2qO!k6d9@& zY3QX`&~;DV0}I-|o8(WaYG)Yc;Et#zuh%S21hrMNWls)(yhvqV((?7ma2WXiZ=BUINHU>0>)bR zX*I}7C3a-$n(Az@3_Id-CK9)WgfT4`*xsX!j(ihS`} z4ax3(on*iKO{$MwKWD!v8kgrvYMlvhZE-dksSYeav76>I$GoXLp{vK0^=;b9&x+p8To^Z(KH7GP0r-~X^6DoBGM-616{T>`?;A>A>QbR!Ka zEilB;h?I0m#{kl)bPwI#&3m}_dgJ^5z05oh`|LAkpB-z*TI;jcDzZF;)Fx~R&!Zgi zwV%ksihVBf@3PV-6(T=-$&<0`!LHJ{VaHN>&Q^6*q_dy-x3`Z$NH?}SZtxewujZoD z0@EPutcvAsI{Zg%XL--NFGKx!;9)4b!sq8A2sARKxdx#qlev$2+i(usaiY4Se(@*mlk0!<9usr+r&rO9+4ZU8@c@+Dz}Ku;N>k?;MvLS?S!E zdyJIX7NRU})l_T^tvPVsIUQcslTVB6o0`|jC>p%WIBL8d=qx7rk?!6dKkYHMA!GQ! z{~hwW+j-Dxy>4W0W~U+q3cv2EnqBKuqD|AT6_>ldT?!xfF z`U>EH+Bfj=M_;XMzq-162|w&E(0QckrblhfJjY&&^hSZ6%fTjDdC45K^om?|9McC9 zKgHv9YeTMTS29~;+3@|HG9Wyvp%pnM$>L~*tx7$gx8JE7CLRAFr5N*uG0^K8Y-f3M zsu92rJqmK0CmIpWGr6w5RQh7rZa&IVnSQZY-?Z4ILEGVC2fIo|@j9Mhh@oGGze4Bs zs*d!5un3$Ih(ro}-?+|3iB#;c@r~f}y0V2&R0{?k! zcq4sF#vw*uwwpXNsMF9RxHgFzpX_N(__N_A_i!#(=vH#LU5NBIk@Fd=zxlZ=pq;x_?S6+sxp7T}a(?wBVa?Ax2t{7Nc7*Rc zdufn0sweaE`R3Jaj_?AXNj>5BJRkon31hXLAcN!BQ$1MR>1C*)x zg;c&={=$;%=^&9_0rESl{_md^euYrh2n~3huh`M0N7>EqjTKF=HYIP?*puta)x4D! zMj6P+?0kgY5p(@<*+)xYa*K!DIhgXo1CpAKv97d`wj8-jBZ^3aHHU2I`1onFo+oA< zQn`8mlD~MTi5Af-Q{P>nvQZdmsK-vg4vpDg5`Pn?)!@U3d8? zh^x1s)>*M>nz27{tv)t9j>3ifV+zNh8u7Lksvq2<`oea2K=iP_0|P zph1+758Tm8F^ApRLfyGM6+tYvP44oro$0hnHxCza-JEPC@|0#?;lY)$Nvqd*lXyOo zGb7dPtax3A`4|r(-Zo_|yun2lh{uixi7ER*KlZI;m2TgDx0Tj=P~q|1!Eqh7sCVm_ zbhRxUqih{otQ!YxoS)XsbY8QEfFScOy{?hp){yg}!oQU!+Tl?6m(n#KOZy@H{cR0x zHiw^b7yNYH7Jf0oXf|>_8NfRInZ)xuB7Nauhx4C5Y<=$kyd(mLlnrtJ>CMN0JjBg?;(qM7UOSNxsO$3kGFF~E|3bHstnrttgWze@oU{#}ZGv|h8w zN|YCMLJslJf13JdA)swFzAp>golf?5rS{+B80LZg)jQJjF7%=wm$&pEYWm=~ z3n7=r{$2flCl)?5%f6NV6obP-{O|0))2z`CGU@K?mP-wkHhl0E7>R$?8JQd@0rj7p z;4eUbg@3{PJbC{N0**U3QUW7N)Pw(in~$)OIX`AWbAx_HwQ!ChjFC2p6h8d_`$WSq z_6cE&=GR_U9Y2FGI6sQ55L$3VYtnUhAbU7~V+Twd9#{`d(^ z^U1$Ih+7Z9)IFd6_r(i^7$2gARa)gJ3DqAN|49eDh|bjiYWCk<;9iN;`kL~>O15j{ zKb`-3AX))Rx&N`h684wZ9B&iWL;f?UtxO23VgKY%;RC};J$7Vl`)6UTs0gcp|J?qI zl;#ufDMW69U|kK{u5 zKDiJ7yoXHE@ATiL(d@qmYJYh0evX+1CCX3U&-|Ye|GOtB{54)v$l*asvOdVg{!`Z9 z7Id zggS?S9a$3}{yF#{E1_2d)LgEA?P;u~_<6WIZF^14(P#>Imp_twXe?4|(QC(+V@eJ) z%@TW*#H!{dEV-5ni)+qqWBxQO>raa#!K!86Z$ zl{e{@x!~J3+Q1}`5JZdzgJ5X}U6q`~D~H`DLs>rDWrdxnYazIFQMgy}iEQ<+1|ZCd zLys8TlSaD1gE@R1H~RyU5JAgDirtz#jf$@%GiA+Ld`6~rK(MX=Zh2mcC_ziA3%gsG zyU0mItXan$SXI3lcl3>-iQ^Q*(8q} zHDbOF<|#=M)y&#kSz*X@>2teS=06fvD|tH=w@s|YnQ-RN__-t?`7k@Q?)nidn2@@z zP?u|}!P6a}I(8$lj@>rGC$jW!b`7=MG-yS{T^R=$ro)pxvQfJx92YhF#jwiqRwi#->JK`&+ zy}S40Ik}5-?NI;{6_LMvUKFzmZ{#z^1X;#d7W`fJj&L&hsmeT7EzVr1X$bD4Zt`c4 zM9%EE3HHn^$2)utv6=M@;~G=CEFV7DCJuZ>`G*Vyi)Ac&9|OH_fl;xYEUVw8{zv_SvpdF_<*DTKUAS(^?mE zvbtjYGX_XcD{TR--L7@H9)0CdHpr+!Gd|mU0mt%JR*v;dx$H~*2IfZY>t>%j&fStp z|90D5_9vjnd!Iv|33 zyM7JRWLEimz8Z{v(Ingv3D$Cy1B-r#C4ZmG6uRCSA2e0E+Y1-Kq}i{-5+zs3evNYB z*2h)ra8z|I@IA=cYP{u|4c2s=IKhs4ZlPSfmB>V5$9=Fu3gVCB_E?pHlCC0C)nVOxT8HjF5C=3Ux%*S`qMICxHp0|vLI_Z={Uu?n9voh-Uy(e2dH1cCZD%BtjP)yPHQ*n$u zz08I-RJ~8XWdR3AW*IX$)Ni=DwZ0_1E&TQ7@OcS!z3UG2eI-*qEb590H|hzFdkYeO z^*igw9|Avx`&##bVrXIkC2x-a`7H%46L-w(>`lP?_7mAp&J21BkI~QduWxSp1b$^q zYp4`!Gi9b&>%#I6RM~tniq3w~LdUM2YVV;osA*K$z^Pr{sbMK%j(I)|36fT99v&9| zBFg;ruycAA7Po!D*n1l&m94w+HD{ZcXAkR`bk{6Y!L-$3?kgUzkf}CLiOFRVmzFHb zT%+!kNRfU8sOe^iVcJmA2XIRcK8LB&_?GbxmEX`Gn&myl5pYGbocdCy{0!WuMJZi( zlH|40Wl#doQVqM)hqU4jUP)@hjWU(gQj$G&JOZOK-1i{;0P=Fgy~rjY3a^6C8y^QbIGX zDE+K;Y9ge8v{A3UPq~&%1VkWlz12pu=}j;$Y?Hz{hf%8)q#p&zcmG zI&Hw)r0EQqi^XJC6|;Vw_PxP&cMQ{XX2Zvx+ucS_$G1-oQQjh;tS})K=?khK)he0E zlUej!Bx39uVFnHom88slvGq-Eu+JnLif#GcijrE^Sp?<0;w*Usbud^ zE%kCw9y*8JrC|#__Avc*$cuJ@k!SK9t|~R3lieMULb~ie(5eN+U)IdS7D&bcbm5c4?t)op5&<8 z-bVeNcutb-Q6DL}t20!p06rKZoGtWVk}de0PGkpcJ~0u=u01EjJpGL@U%%^vvlpCL zk)%v@W$z{za4pAIP*IfH>6j_{7MR6gu3YMZA6WeT#XCS^YR}+xF4HtWR(hCzjEt(V8eq-|h7^L5YkkD-nol?Ql4JrZWszCvM8PwZ-^H4k<)_ z>nFheIlPm5yEAPIZ=yy(Kv+!!rpY9dO17NUGkjlkB5I9deNSsg&HiBih;EW;lS9*x z<@hnJt0tSZ!EDikx;M25^)d8=i&;AbxqNnW<4_^C&(_W3p|$+_ zK_$nEDI}E(HdE`Q@g@9&@wY)gYzQ-mDo?vC%X+liFCpZpCzxGTo}>j6?t4>ABIqpl zDIy7HCgd!G7OVSuw$W2#%JourXp?4(TTMnd&;8yxEsyiiSfg)?`(@uGD&^ykKJ!6# zb1{7Rm0c!y@IUv&4N3Fx%tYE#>P`hH z5T&i=SunvobCQsp>ki+yil=e3cG237fTZQ3zQpNc23Tv=Y5h#(7NlavNcr$dWQ@!T z9MbZGP9`+m4&;(5FP+9*X2|`xN61IQuGZtIYMDh_wtTtx1t1ZJUT}VDW{?O}gW7%j zYTJ=QTW-DbK7^P)K6do;ONqgZ(r|2{r1{~!z@W%v|0zE>EWR&s2&6^YkUglcetFCO zdd@dD?%dszm=3Q84{E#TO5%v4Gd3r^KJAIreQKCo`N|`c*nfnK4 zPK=>ZD|hOJoeXW>4>R9HP@RodTD9XiF0>I5o_D$hG$!KrlQ3#Jd{HR32vOKx?qVh* z8`y0xXSSSct9=i|N-n;fEV4NVoQW*xO}_}53_^D$SSFSw(;-kmM^_DWG6aL?-AS~KqAf?} zdnMq(cs}DjgNv;*a-y?&AdbgV*@b}G<*mId!bFA|fjt97MCeO{IG|D=>l&Lfd+fho zpgVJrmt&Y_9B-&5mV~^f3bsMrwGjFBy+8217KNSb(uqi1=4w9yb3M;syioF*bpA6? zgb*eT7B&j6RQSS+waRy~Q12_uf}sOHX5EG!=7bhu@-8aj4MzaUlX*Ms2zVr!tk{bh zv6+$AhTZU`M6}TVG@j+*0X2;iRzzJ3_Y{MESUEZm;_x-O z<)sMjr)y%!0s-W#8=iY}mS7Csau$%4@uP%l6w0*7>5KX~DSaB;AQ~wy^umy_QhWKD zFNxBIv1LuXV9Yk!IZstun_J}e651AwttUuARt_McK}p34)>GxO<)#X(QjuvK8!J7D zqbUv(K=6{n=~8(%*@ur_5L9mOhmZ(*$+2Yy;wkq}c_Wo{%>2^I7<_91vp*;YT(S*% zwdxnt9UTcQ`9A@q2=zYmo)B zWMgDFvK6V>O**379hQ)Vhf$q5=tq*nH$S&5jsk16;cZICpIe2M25T;EpJbnv15C7C z+F~fY2=@o!_W?6$sLV>Qr^;bb0OjK!O|@#r-kT#gI<;Q3>zlor>?TQ|r9!PClr8Vz zbBl{4tS$(9(ec*FM?=X8kstJeZ>_u*#=h9%UF3BdT$1T|cah?2URY?woAyO%H=S2i z(*OsW2DPRTL*-uIBdM0K+Kwt@0uv)VPJDkfA`bnjYFon&)sC3yg-*Mb< zS!)jv;@}$!(ARTBEqas75l6|ZK!76`ex@v=#?~}lnJt&Xp*VAuC0JWd2)K5ifHOBS zs@8tID0f(%H~53far~>Nxgq)@-isUtK0^#Nb|rlS2R$knltw^EMvnqb1Q3MR-`apj zT%CoCNlDZ=)5x7wr<_e8Akoq(BWQ_UvDcR=mFfMly6s~hvrq5P@*{{?_bWsv9?f7j zgUZ1Q=8MHDd0F9vx5?~6Ct{C~kP;AL4xVlQEY-(58nV;pmIT=Gm+J&O@)v;~-peep zJ_g?7Qa+n}O3b~G$NhC-LV=&rt(an2tPTSJP`Lu(MT^`*1Tjyx(UBXsw?Rv!LW)40 zHeL8OHP)x2vdIqZIWjIu37|Feu&03d|{G~ zW-hZ#R!HSPQ~G`$`!s&y$PB=wI-X3q5DFRP6vAuth&Cw@Y`0$7#yG0!+;h)Bcp(lV z7%f)9ysJp*O*=Q*2OwCCjBN7f^j`jmhpt=;fe+}q-drCQ^rh9y316*NOPOu9_VnXI9l{6*XZ;!`zPFqm zFJ2y8ysFY@UPLd1e6Nlr+($~e0q6)J-jcO~gT{^NpK+A;fJcwmC&WbyuJaz9Wz`+)aU$!H3a(g3%f(i z^C(TwVK-p1Ll%mPi9<@r&m~|t0YkdjnaEiXzaYy9CwQbtO*sxhAlx&#TEy+6 zq!w0Zqvx4vmmfKe{N^$w*IvG5ZSGO%IOi?7OG0+aUO@mhV(rco&%(EiXYt;>_h%Ct z3cqeF01l(6X7fDPBSw1_=XG7Jy?}1hwr;fj0p!S9KrDk0;47WN)NIZ_REveVRTeQa zaBsRnaw4n+SILF@O7v3!@HcQaPP0{S1Fkki0oG{ zCP}1q%K1KYkXrIcWJ0~^{ks-Zbw7n&`Q*w8wEg#Yhj|Llcn-!FT_VdA?WX-qt zP2R!0&ireuFhx#?EM6{kF3q()M#5x`dtjPX9l8qDOEx0}a0YrU4;-d12`O`Qnw*t4 zrk9)!P7c_2qY_tN>b|rJ3v6E~sd8WjV;;Hg-OYYJS)Dbdwju zp>Eo!;gcu1ot;*LGy2QG?TgKY1vb}%#m75NK4Wgq)1PO{mqpA8mO=oQ;xS@nCEzdq zCP?h{KvIRo3XN1$_jve*s_b;U3wUJ0(wG&g^dYyZN1bdZ9bLFhuDELpQj+%|Ckhh1 zQZ{ z-#V<*YqGd6t+E^(0pYJJ0?E_6q4f1~WW$Q3Jsm7bN_tdjC&bktVm$t`LA=-qau>UI zKGJkq$fZw+%oyu@_8Xb(4DNm^Dx?duru&ZkDtz&jVJ35%O|w9`Bopwtfb2tncw35y zEo=2ogOxP|w;lSbe=CXn3&V{%Wre}YN0WkV@UV6OIiY5wYuHTvTCU)AUnB~Nu;XiC z4_M=~u4HlIN}7Pjdo$5$j3l1HP$>OZD6fMtQTq=&5s?YhAzmZ_02Q4`yKM656w**z z0Rl{}Opi;`0t{5qSL3PyDI|76Ggysn4n|IaYKFmm#gZ)>qcYvbpF`Rq%=3as*ywWU zti+yS;;Yeh+azFik8tEcE$nYqVDmzlk`TLw+^|V^(sZV4PNl+1tECBYFVn`Cw@PB+ z-Bv@X%#VxNgL!M1eQUKks+v`yFu*Wb>z@XPYu-Nu ziH~bti|MYXrSjEYf^B1H<(UJYc-dWwjRd7ka?0Mpn_|7MA`@6t5fOK{mbH`#`LcqZ zF{DT>wtZ%}m%{;8(#V&F62@BbU-mpp-Z2fIC9=HzbYs?Ji0<3cOg$!-nN)5Qd?H?t?=E}{KRaBM3#^3fhzQ~btUUhPGIIbhhbPo*UH znczHZTg3kDo5*ydIqL5&fI^&o*Eq!4dR%&LZ%i9PBLjF7$cp~O=r%k4YvlkpH`T|T z#(&(exeknrWZ>j|0GornhGRC+6L{7Uhi#9>mH4fi*v!Mz*$pnwC zZXX6Mv?rXwEp3)}BAeDNcV{s*Cy;6z3KZ7GS9{Xe(09bOy9e~1%rqVV*q>+q_L)&_ zRtM?@*(U=Y4E5}gNL5Q;Xz^?QHCPVK(dgEzd$HbFbvRfWf+<*|7ssj!^lEZ3DMSZpbM7Q?4~Z#I zP)JDER?Pbw{SmoEK|oIJ=tE&;LHqr0*QdR?IYyh;b9bgYmo^;M^NQuZ(_a3F_Cq5c z<@Os^Elsb(oZpm04s;ptHod{`BcK9&6+$dIpIgdNp(nXPC-{HWO=<2$Oy zkW5?AqX+^cnDNeSpULR&nBpCprSH6IaXhvEq6fMuE(SRQH&4U9~B zdvWg@oxKXUEH(aR(W2j`$zXZKs#N$nMWuZ)g&}ZxVlMkIl93I{V!%rN*k`KI_~gxo zfMJ%`frHfWS&@OTaw$=c?r1y^8^l9^2sqYcV^36>lvqzTO@kFug_$N4-L^1t9kEfC zq|kI;x!j751NemHqY~d3KuUVAiU9W}S*lbyV{e?cU^a zPoyK)`i@`)wz8S4$c4iTDN<8-l}EWzI}IqBDkQdKOj`k%Hoa)L03RD{iZqn1U|F>< zS_ZHqlcQ^fDGvn*!Zkl7&wP$?=h9F1)KnyqIIN4TBcNm*JZAjHkOrb4sZv zePm8hV$m-yi*}et*<(DNGN^D)A(^RFA{r~ORyU~gB^}yYiv)*#u6$+f>J#$Go{I(x z2cdqJ2X;-cPU)Mrt&-QOqmpfmVp5?-KQ9%LYJ`tYI@c&Y#!Slk(_~%>d@xEQ2p)up z5sem51qLZPpU{?8_b=X!b#<{1FHf`95hlJnNLk4L*+5-M>{8XZFNcWzxtU2_T6o^& zqK7*q@kQiyfeh?P)j)wQxesf|Fg`mE;B=a6+-}p_HNTIRR`0Yt;0Aa8d`I!-UiRi- z*@u$)NIV6=L$IIm(nzamb|@wV%JR9ard*cLX*p@ANSb>Can{IvZo?c8(SgZX9J2Yt1uI6_9YNADb3gs3p>xAMUbZoO(XpW1#pro-I3xdg#34?bmo zr|sp5bMHo#0fdW$dp@_f0V`GeI_1WkOP?O2kunP!HmASzB<$8ItuqW<{g8)RA6u`P z2ccThfmBPgfKqE?-xiGj0w|)z1pDnG4HJb6ksJ5;DFWvu14kSd8co?G=OLHd!qX)| zJ#+k+99Fa9orh_X8;mU5I%$Ajn+n;Vd#gIBsze-GlI75QMr$(g)J&09;Htg%)hlB2 z=`*1H_}|59pQ??&I-qY+$EvKnK}JSq)1TMRQx^clPT#%uH&|85UvpN7B<)}*Y8|P+ zvcZ~?a64LYYg~{!dWzzFx8I4CE??ypr&lZU^~Ry-jh^mVD*PsW@~*gov4Cl-Vlbut zE1*K_Uh&!Kknhd|WQuli+B=m^c(M5|0tsrjRk?~yn|a>KJ`BoDA*Sw`DKsFBxBKEY zRRV)FSn)_r=Rjops7&1`8pBjTFAf=MK_3Bz*;=dQUqsU{&k0e>*ar4~-iJMe<(u|uNS3C%+50xo`v*sG&j0*H(SzhB z;(1Dn^FnxbC(XIhX%L}e@+ju|S1OV;g)uGxpKH>^Barpvygm-Hk|x8lZP3=ulgJ)} zyYRTiCuCk`h@R6a_4dE4nsVF}H|6|I{Q+ z0l}^DUTayzt^YP|HxuRiNijCjDLzS(ysGiaV>0x_%oKj>e{-M4Z9Jgp%MLPO5^p783!c_0N@x}{GS*Os1!xR3mOA`6heUQ`SofvP|EM^F|kOL4}f#JG7)(5QAw3 zIBW|qw@>$~HjFe=zW@FWP@8#qHPD8Z42qv-luOp_{NGNc zLPd!G4Yx^4_FJkoarej0Qb>F|w=p46_ubA^ZX6?m)^ZU^Oy&M)+p7p%W`UQ878e$x z0xle-&#IH9&lLKz-d%O5)VYgoj8yrDbe1#YO7#)^%JTSOMdzgvQLmgTgTuY~Lq>jy zZ-UY+eErZh`Esbp!~}QmN-s$^$2l+Hd^2m8-1Qv;TOwx`#$pHYKLh3Niiq|n9Oieo zMF;s&3*ynfIkqTwIz!mFJF~bby0hnVIt>^pQXYtF2ej-%N^!yW0^Fv}wUU;lmDALE zz}O0jUrJF?5sFYokOsojm%|2;aZfq&oI+v%5lM3f?((>>IQVExNTpGRK`hC2Vj(lPH(i{ri?lz+thH(t=vj)dIEGFC zCnQRr7oih1@_>(OrphUuS*uB(gbV{XK4?_!&CoaHe>gsNW_$)`PUu~~cq5-c%VReO z>D4I7Lv)M>cq4O_M%SAh5+NU%;%_%AOykR~?Y$lP%)i_lXkE7nfWsjXz{6=Q(E}d3 zqXA()p$G_2b#gI4K3X(1Hr%^5;_x0&sm(Pqk%(lhKP7wCTTbH=N<$}zF>X5(Tae@` zD*C|I@_%OHpO<7}eq#WUm$3TVXGh7!(_}dBjl7+zEM&4t8ij;{!;|;j7c0-UCt0JJePAA#kd4pF!>eU-l6WCk<9kSAL@g z6r<>f$NOb5jyD}Fmhkf8?^Y_}^71Y8BULKw!E1&*N>s(OrSd(o%rpQFDIPx}zjxYZ zQ#R%@F7SAisqD8b-~#R9w0U6(hu)Q0tVx@@hyg5%=hI~(+mDNO+Q_WM1)Hyvxmd)! z$b9s_YmtOAVxP!gnSxCBw2{KMCz2@WopH}c?k~N17$hI-5VO{k>Uk!2@#QU$`#n6B zOX_r6fVtk4JoXo^S+9@_ymBq7F9|kto?LH{%?t#Gy0%AeU6Qh%e^76fnbUn4A+Wsv zE2i9il0T8;T#I*x-2C2ot_fGstS4so)XB8zesDLP-Fe##M)fkx1(DNe(<`yU{$uT2 zDH!^Pa_x6V3bG=6qW^oBxW^?TEB-N#d8bW*QsC~;lX5`UTI+aHN5b#;+HgLNtI{Vdh1W%Wf0+9myaQ;_2<#?~4gmEij{RK|3Z`U^!_OZZ6>H$(s@ZbW3;rY7 zN$5*~v-r2U^ta@EJ~HunU!AUsZvyPQrX!N8brw!=PSj05RrwJO9|CkO}|U!I}%G zfE~AhYS8&FqLxBr{(pA3hSPhr?@mGU)&oEAKlt8%^;hsUl4fi0>wCEGVO9d*kM&>o zfHZO>QUd*7neH-(03@82?fpOf?Y}ST=kcFM?%NKOb-A2y4{+8H^RtBfy9vM3g7p5# zU!uGQa5#AXmjLgZ2&hh*nSlozUf$PVFO94BzncUl_7~EBp8LQKbk`38-k9HjR;(|S z|7`|^|9Bj*d6#U(_zLy*tNf=|e^=>`Dai{Uj7@ocMf=G9DgMtsmq0`8qx~!M!)MV1 z2AWK%1AHP5L(+eL+*@4yIPk%8ztOsX{r7+V_6be+EdapXAh$QL=<;*%bD=070XAos zL?HQpeJILD{n>`3WPlgh&DPQnS`wt8Y5fQ6`)}Ycl`rW$=PIWesk@{L1)!n%kuNG2 z^xxCKycRMW+CQ!OfbP$xLrM6m-H7ooO2leGW~={Ss@zYy2g-7L&NCOsJc#_?yMK`I zAJAOkduXn3I;xZ8{}MKK+=CA_!<3A)u^5#tj7Fv3G7W&kAdN&?r<`G z2~hU`;EDg)$>#HrC8R$ttHwp$vta+^|4viUAebcqyS5_^aN=|n)`$=N&yUjL9{l)c zWp0+eub6CmZ`WUCQG7>QkNnRAz#H^2feFw=Hkg?0^$cO^|DT9zfSGAp^PZ&dU)}u| z5s@J>qx>c!35r%m8ere1L$}hvQB)j^>N%Nl#p}V30ggC>q=PY3(s5KFKu{rF0QC#e zsJnk(vn;it)|hIwxW1x=uMi0tD+??<0ZDqk3~8O<4HC&``mmvl1Kc;0V2(dWswBZL zw{vO5q_=z-bn~ebcM7Q`|b44Hk~`^_?7%kzmMnM69!bAT(%dZ;O(~;=Og&LP`dQnb_*x zEE$%hPJJvxN0spW((b$KfqyXJBRc(6IYJr@MY|{f=_-4tZy*D}jLEeKddsuKw;nt0 zb}`bJ>fCC_@&kHnA~-sb_ZXb6#7PlC_8%^H!{a!d|i&kqO5;k{|{72?jAwkRzZJ5>>K zDF(h{d^Hudg8)u|Ie}fm)i&W?ddL8w#UJ^B+Bw#BkCuXaH^qHZ&aClV;6@R|_5W(O z%6)?4sBIr3vPVoe0Ctl4v~E87wX8!ZOB~XMV%n2%8^G^*a85)h^nF~sd?vjS7Tr5l z{MB0z0Dvv`$Mxflo*6Xa8Ygv8&pm-k>HkRuA9N7&ROn)8oum>@NlAY-H> z_q{FfIy-ux1v9EYxa)qhlQphLp1f5mOe32lO_9Ja*orq$+VO`u3l#wZrcFgvNjjw+ zeh+oJ&rR|%DcftJ(Go`Zmgt4DWZqX&ok(&KhzF{M^qSipJzz8#4-*A>f{~_e$#k_kBHY% zaR{g6Ax(={Fjt;|y!uNL7NrM|+=3Pzwx%p0!ZtD+lWxIinXE5*`*C?ejTPVzc@EUn7nQxs6i495I0)k|@ zCd6Gh&pinIe)~m!Lyu=*MkYWrL`QQfpW`s)PAjCmd%MEr{J-hjdl~c zRK~Y3I63g#S)P5#qU+<1EsWxo5xQiXn$#^K-q|aaeij8lt$G?P?*y|%zdPOOL)8zv zgLc4|g_4JJog&>W;+>{_`|tqf<|{XQOjQG)&5P>eZO*rB2=(T}Bmz&nT#`p)eb`NpMUbYrJW&yay$Bam=kakp=>gr@rix*$p?GIF#*XUzq zyprXCM71xJ9buoJM86OdY)Q|3nk4yjEgG}FI{F@6R^wpYG6lwIi!3;8&z23!QK>O} zd2n#*TD#ENIo-RQ;Zng55m9uN-< z*nB!QKd-!qyQ3`r9{KUP{Iu5#aVZ7JdwF=YLyTFn0_H7zV@4Oyq|vwcyo&c>RN}U~ zinbp1KTG$9IoJe7%TL#k!S1L@;TJ##h~3*?mtwRk(I)>(`6c=&-+=o_X36&Z-#Lu8qdBo6xl@7gK~TJn6%ek*-jaK zy*l*u4mby-P|v83T!!pB#7iG8%fA2|c^F*%z)^Lm5~8dfwyit*DS9Xmb4l`tV5XkK+JV ztmx}e9<<9PfkbxCkZVrs*GipOyBuISCbIlCuL3Vz!W?h8AX|gpw8o&UPE2w_IKae2 zH{{!$D_r4qi79d5v5rd^uS+=osEdn8x5AUnXWn6Anf9d$8P`M3;VVUq@{_X*L3iMIe%IdclP9MQPj097oL+DHxk+jU^U z?MC}lmUU4g441)Hn_BqT$(X{Y`WW;F2)a2P3?Hk-VKQ@mC~}W^+J&KT82>C8sued$ zDi%Jh_o{)%m6xT*%yvBR^6k51E#j&gyRIiCJpyyw}j&`@Oky)iM3({5-vAfI&Gq8R5p)L;)#l9u@i_$L5l#MvuT z5y^Dnfd8y4>3E6Hc@&@C8!>xO_&=9;fIDHSnu|4ZR8KkL0N@;aZ*MCxNIrK1#*we( zPU^-+|A5i{Qq|jK|0)&d>Glv#Xfw_nVZJ!x@$yW-Z5Mei52qcqW_t#3@Ldx#`EqI4 z&c~^GmHUswn-MDE1X##&)63tOK|e(^ zpdDndS#Hgu(4pJ-zCQ(amtW7Pm|7`qOgBLN1e2WeEaCgf;c?VfzB32D;0Ia6GLsIj zi+s(-BEBUy`!3#EkF6>iX{_hfVshV`S;FvHv`E`-_oHzkJZ`VltUbF}(1|%6^e;g4 z!ZwO!#_ep-s=`z{7sm_elgaGgvH-2w7+Mewpya2f4aJLsUt+|$<|lBG{pfDNRyaK9 zvIg{IQbT5})En>@Ogy%8u_9MqM^SJhbpGzwctmcybF7nxYVn}qHQCcb!fb`2rz&}G zbU2kl0p-BCrOCmGxt`oP01cx7`Y}eXmXSd9wK)^8S~8(;sk86$QrMtDhTctCkbIf= za5GH6N~*cyMIGjKIcf)CH}~jHC}>co!?<0#qWE=u<@ABU{1B}jyj3L>FeyOCIjt%gh_y7XhPms2?be>#yGQgb z4`y|d$gnUxOK$dE+%9&Z2vh9vr_Ku-U~FE;<1CL)kzu|Yp}DGPgLt)lU6 zOo%Mw*#MX+W#a;c`9U$0a1pY!c+rXTvxr5LKq#1qi|VtX%yR&|QPwWjvVDKPku zDefNBCWS!KroOIJ>aaXw$|C!|6!xN#LUr{HxdgDY6mlR-be*#`RRJ_2W`H!gC+pXZ z>EJ5g>K1QJ&~w~fJc#cd=UBVdQ0&DjrJ>(exmTOyXeg~S-ISuE>EMghsaIU9L)lC! zmBt|4#r^;f)AeAG<83Rg6HGGB4}eT5899Ec?aXjGd(txq$bzJYly1AwTc;jZEO>;C zL5%Jc;-^M?K;szwa)d)UAR|xqwP=%^WKIlUbh6CAi-g6UjeCcmcz~Dk7P_pHcWSZS z!_TGXs49{0@O=J|gU)NANJL_*2}q-pwz`5(DYW(U?#|JEl{2^W?A#mw^vKcDsD6GL zpoqGSw+xiZq|cA- zNy}0M;GxA7pJxCjGq5dWbtE&M6s%?$>0|U=z-V%o4|^{!<_s`F$6FTygu3)=kwvde z?rMi0cLOk1@$C{o(QhRDaCg=>p>VpUUr1X&h!c_D`6;`UCKe92Of1&y9OHOfc>4as zulGij!$Imb%R!!J*8v||P7%%*8l2WxW$FhEYGx=jXk-Wso;Io}s+fAos+co?XFlEB z#P9_{1F0KKmz6Jgopu9yQ8=fHlj(JzF-A?Jg~G1CS4=#9Dv60{9=+5S!k~c&6%%31 ze^A|63P5B;dZtZw*x5RB7r#k}-`*g)DsY0~L=M7p)saBq@}&Pvgt{7Ti+ zGQe;!aDJTV`2d0O%pYk8WAvP(t@?4 z6)w16&Uk1wIR*kgxE@vC!v;V(Jw1EAWWa%VhQm|U81TUMFaW)6>(vIGUA!%YWknpe zml@k~3Hd_ADmQH@0EW&}zOw;A8w*LX5T7s8Txb}| z?OfwfS#jU?M-xcemz}SwA_i=E?N>|$ckXy>CJBu@%5LzU1!s2j$M)tI+Wy+by>~r7 zqjB>Le{#~l)Y0)MpTR6z4^Y}#zjJC@b5lIs6`=vNq8w)$C3?{G_tz{FM87mY#007e zJXhd2YM9)&cdz++?zyjdWaIsnOS0`1JMLN>7ytf8dgii$be<+I=L%X$DCsZvU{ zVO}Kh2z41!m^l1{2^~1SoKs^NIXdrN)3#7FDz&5(3&Mu8Kh}0ch9;m^KlJH)T>b0o z<6`%=2%^5e+1jNUfg+Ekq@hM9)Qhq;IE32(e6d`hzlw`+LN{IlZw9#*rc;7t@L!Gd z8fNoPwF6-8`S?|CYNZl&Nv(QouP7B(o!Yq>DPVOS5CjyGI9)NE1?QLN(p3#=;ehF< z7uE;V2_2G3GU1WD$Sj1yc&d9r9>^wEpioZU%8ecuy)qE@!^D4Dxxu za;=ys7$Qe9tY|i%#jH1Xw;O$vILs5d;NQ%xhD1fK+IiC1olu?~irk?JXx_NbcAt1J zAG*QsK!N;m*?Mb({o`f0IepRB9`Gpn{K6_|w+Amtxz~b4ZLZ6U!DQ|T<^{6B5M1-{ znI+pN;H-3bU{T%7tp2DAjk0uTijb!=hq;W2Pcxl%?2B@vBa9HDQH^+30v5Y`#1hGv zq=3P+yP@Z#rSuAk^t5os7XZAD_2c1Mta6}dSG}HgR|Ylx{Iy&TI0o?jC#eVRP86z0 z5%!*I8+|=I81*U+?e2;`^8s7DZO|`qPf(>yHd4Bf&#ki5h26^I;pT?nyjsBgZfe`c z<7$|wDhRbF zT}8-Y(X?B5Ix&xJHyjXbL~Loq^brnqt!D_y*Ym!gL0cLSKc_1-xskirj2U2Mj=91h zVUMw{phM?!DNw0h~-3{Lyy~e%2|0g^=M>ywwXXc%my=U*W)@G^l?gAu=0)w|mk|&nM4kR-y zsgDaKo}_YnmnV#hN&&o`$;p`LGvrLDyBXWGR|PT|kIUCFre3|+OXk|?o5v>Xq`_=eOh1XOVof!crW`%UcQd&Iv5D@y_8l=Ov~X; zd2dkU`RU*9LG&_xlAsDT>x7omo? z95?nU0LayzTXR3J@YO#4k+QWq_Uh^?qS^V;Kqj7{D(HPd$})@&pC6B%TpI8UsNLc=1a8XVWBM&sZ*v{JdJ_g+WFIXQgUis zW@D<4afTlE;>L$Id+YPwtW-tMCGBIH4J6MW1FwD|v^R=Ckke2@#<}my4dE`wYPe4oJy4S9}T0N@_Gi$D3!N{nt(t2;1EEMm(I#P@!5?K%$<2Uvgdjt z?h_`*flv@Qai-PZNK+eGecg4<2xCq1Xe*KK&5iFqiU-8p;t|1&bI$R*p;0Ao4sS1+ z=VRaSDcjo;XD}@u^;-Rm{R>F6jl|*7ZOoA*tgi& zUzTF?9)F6<^8CGhPriD!hwaPPuV0I7AT&b>^>kUtEQ_MOP5Yv0swuxeE>pAHy0bIF zD*~VM@d~X62Cvq`u*POOk8NkyLB2;RMrO|{$4;_lxjUS)FOEueHILb5HE`G`5rje! zPO61~s)Ks5x~kAcQ|Lk{#Uj*8vGN$0iCrffAf?>;hKWx0ff)h&IwPihCD%K{M<=#f z%#dhkc>Fp9_eQ2&#~Ni9k2xgGuF*j$e_d!0Ui!F7CYO4UaoTm8{gkKac4N3372iDy zPq_1O7LPNWV&#)jRGGBqQi_p1Fu7S>20*Qpy1oOswHYi5pMEh_B`kNXcaPa^c)Sk9 zg=JDo%1!!p43?;vNy0Esph_qrQhB1k9>SlpxTPSDF~18;+tpBuY=Yyyy*x()(W*~K z$1f>99gRL?WDJ($j9I~*i#y^?XUCltzWJPxH{f|2pTF@!reVu8Upd?Zm_gHc6pruC zubdKj`jDjI?qM=N5=~__3%6hXxfLhhfg6+=!Ag8;iYMp4X~(AvQ-gBSvMGZ2?!#1t^_RY8CA;8eTBP|daG$E#87GP z0+s5%7RI6~#{sb6{DJ(>co`I;2$LkSUzdu@Nd*n!@~MG$=%i+)!?C1HOi4^a=_h%I zhowx)s$Y?4-KMJ7{UE&6bBS8%6DdcNU$KCVkT^C+nXe7rXz39ETSJq_Euy;v#|ORa zA8IFe=TfUxr@`O727+5$ra3i3?&3HSm^{264B%u@W9Bm&<-_(Wx$<@0=QC$*$7Z0+7tknBEi&sl(4q#l`FZkO)@I|$4r zYu)tp1b{@wV)F6S>>7^aZcJv02+>6X&BWf+Dr_;Q?lR{rK5r%oZrv7T=!ggy1373Q zmI$drFFYobqu*6(ZAX*Bp;P5!iN0FU2n?ug|95A-l{4c6S#E%Ok-Dx6@_gKcY6w$a zdD6mz^@*9!3?esn6wJ(L&@BZ$EY~*XiIAv-dp{vNg`?Gd^F%Eq-}Oh5+doYHPW(~( zVYFc2{+r32H>_1t+W%*A-CtS)r23_g zSacHp2b}k-eLs@wlb8s#NPVb`o3fw9>a)V7D=82Q)Y9OFADrmiO{7=RZm6-nDs9J& z`k#C&H8cbLkG{59%9G5{Q8YIRbUA1bA3bLnz#t|rL+?yt&=i^5oDqsH^XARyx%SXw#;_5JBIKJmwm_C(l%D*$o^R$ z`^cMHgyZAA)=)ZL!l)i>e$gf#j4u4^uf7}rf-r2)Cne6Shm}S7Tz4;UT`$X@U2|;|Z-79mHjv@I zQO33M6@Vnae}9X>P_>o*`Ar);CW0$cU*qXc-!{MNdy&&G0$0T7=p-P8_~4A+M{u3cj)Zly3X)am<(OKbyyt1T0U*dF?P6GF<-VJuGyU`>sJ^4TECu6M^1<#XrVqG zZuSc*qX>=i3(We@ItQesERi}jpsBd=R!>=z&l?CeC^2k)^*QUt?F=*?`G6>KoEpcY zA|R9g$sA-#$boDF>x-_l{ipcDX2bey>r#YQ>Kll)p0BvmLC(=GH@p`n<>#*TU#ts6 zbPt6c`o1FRd6S8lfh(=U$ra`P>1i@8J2pIL)x)$eF_1%?OiwsrXZpY4J3lT3R)54# zd@hKvqVGJ2OERs0>)~XemxNV9ZLE`o2$Jy#H^){w9;9eD?+lum&Q#LQ?2VB%nWsDz zGWi@8o0^*`q-r7nr3o*Vr@~ma@&ls$!l)T@?@(GTh3&i2rOnN}oh18c;+}ek5x1qk zu)qNOu@?95<%;_Qp9pc&C*KyZ8WdD==O^eC#Mw456Mg# zNLpw2^j!L?H@Wkjt8@>1lGjvg5)QR$hBI0zfPsleGJOFPVoABJI~EJrS%^2~@T)Vy z^pxRrH2d_^#7@?vXI@Ar z$a-*TbrXUF2YQfx!V1cOpTg5Uk;%Ki-2EJ&Ku0-tf^c zva)=`bQ;{wUjXp1w9RZYel9=bzl_3yGijT{ANW?=D1g!&n1_G<1rKcUfFAtIa{XMO z657CYNKQ)iM1%@L75;_m{Jjo-!rYkCAZY3|mgWDN6|kAIL0$Yzd%IafgaM}nEFS(P zz(+(S@)v>_k{q{$7)Vb0!XL7LWG26i)}K3qm>Hx0gfzI6knzLxf6cF-rr{q*sE8W` z|K;W@Hu!H7ECB;ONd9LB3W@`5&w+Vl`_CQFi4a+o{=OUJHTq|W-rwg;oI`vf1ePDC zvj1rq1UhvV2kB#68=LMQ`}?!^B>rZkZ7e^p8=C=c8GDUb<@M)2=y=da|HctMksuia z%au}09PXzEr0C`y_ZaxiJOAgGh+FYp3`yT7g)tHM#eaKq2NcV96_Y)E)d~F96QB}V$CNai`Zu21SbT?E(GX?e6NP?Z) zNaVl0Mu3MA%aik3LXz?I^|fCRKe1CLc?M{@_)u&ALA3oe>Ldr4Mu_54clac}VPvgb zh9mWX++q0ZOQGl96x%)Rge=n?o9Lexl;g?TCAs|HnF#{?TA!GQTge zpK!%o&tjIz8N}Yk1ce-)Y)@kdz8EIY&Hb4}{cX*Eyl8TRj`#@*B)P)ydxy>^{tZ<; zZ9tix;{IYH{e%QGG(|%h_rH-IkMnI>rmZ7)B<47SZoDiGG79!i_*k>J+S$HvL*tdal*ikm+#~g@drj(?^Gq z1fH@&jGPyLPmF(7DHKMS8%&BZfV-|!`3aKeU!%HkRUmm!0qaM~Zi0%Avn zAHsXDki_TA0Qdfe!u4=HoH&ipuGbK+eeDCn`e zvpwywyF>V2{Olhrty>G9h|fw7UmN(^8`-?$U2$xo9&v zH^xgP)mQ*BJwW9*qtE|c6<`M;QH3B%dyQBB&tsSfgvJ9wEpz%YWSzk=^b7A%Wjv4H!jz@*$i2enN9tOh^@OnLZgU--E)??5_oCoCcW z(h&nHsP122-anrLRH<-qEmC2HsXy+*Uke;^3M2(S6g>$*+0y%cr1;mcE*w*}jr?}! zn?6Dy-x*m#13w*Tpww$*{;&5#o&f=bM@Aq3^NiF1(`e)WJ18ti5G`Oq`iuX?z9SgI zr2KRv{1K$VITqpUQ;O-I)dOUZQ~3Th5CZ(*VP$0yivIw{Okx;W{5=5wyhcYvXi^T& zH`M_QDpHj^Nr?L^10-ZAWR`;NPqEy3`TVWTe7_o1_ zA~41yF;M@q^lkJ-cVoIp;zhXvBj9xIv89;?6L*2uC6F76<*$@lC@C`TkxM{&w2% z1~mldGe~_n{x777iHX(xnG71uAH=#|U40Ni6&{fZ{%Zu4gKTd*+CI-ln+d} zXsAd#n=Md#?*bVfQrlkggBJqiyJdX7@>ze@fPvP}&-~LMd=u_oyWxS=RIIqOE%?;E z@Fn_Q9JaHj)F*BjVPi1bJpViq;1rNliMj-$!0Vv0>1&%Io0D0uK)H)%cJQIe|85nS zpLqV#c&sK|y*4q6Bm<&_p5wVf@nR!D+z|pv;kKDie8aud>&+D8rFnz$mx}Zg#XFVw zCOo|kF&iY)!5UOC4@V@XKu0h%&DjaT&fcrwSMZZv;C2o}(kL1d$N-;;S~**le=&w6 zW2+(A9f_&;ZnlZub?01I?iI@KO&P2@bW{Y9q(FK2uc&Nu>?Rw<0VMbGgE?YP)L;vL zofjdN#NST*P9=c}U$0BdekPZ(QbEko(OwMD#YjzzJ-sT9@Y0<;Qi7~u=8!wa5MSRP zTj~xQ;j=o@Wzlg@dsg^JM0;|`l>-|N{tl&( ztrN<)=N#%Af9bN5p+i9$K(g6fYDI37@BiLF6EEV|-whC;B*j37ETChyNx)r^ok@;y zEP_n{qWcj8T73bM%;MjEvq(Ofk%>U!am3(hfZVgCu{s4jW*!?!YKDKLz9BAma; zO8}E~M+c3~@Ojlkx?l>jqeC?GluwfK5LNJKG7-;T^C%03WV2o5#8ZV=;&B+}+M}*M zw?DfF$y0PUZwEvjG`LE`=dy%!!6sugEe@BNhR>Jsi2v*jfMXv*xCk%kzAz4|-Hi-z;>lJe?7Jr6`1y0Z5p`oE?=kwS%e;5aM5G8D&W6nt~b&`(N zVej=-K^A53FAZ~u`SCz^!{a>&2eZqTSwj-8_xch-KKK$^0QRyl5ttMWNQ2xKVMq?o z&*2eF+SVfp>`9nDoHyCC3j7*5um_8rQQb*Jt0XpQqh37%R|AEA>({aO^4$!`T^}a? zS(s$pP`y4eP+7GJVR!8XZ3R$%%r!82Wg&OeD8Zd{p<}K|?h4V)fG%Zb5-s^=A@x!F z=kx!$m&C)LZ?SbphhOrTbHlmeByU^#Z4to9@W&O3?lK=A10dd-B zN2V>~j)`!u-;T?$LnIISL{*R}&fnu6CUxg$Pp*KQPyI*)#6SGasM`E~&!)d${NSZT z0K3%gx0m#%qyIc^5;k-r+@G%j#{}?)HshbP{PQ=6g(1T*`Zuv;0TE)Dph@^3rb>9i;c=AUAz08@=f}h5CCtk2?0IrO1m}XjpI>ji0~<7_xbtlpDYJa zyubGFrad%6@ir<%i*3EUQ7Tbqpapq6Us}p)205UyZWWRI`Zax|D<5|<5VIT5%5rh} z+FXPTKv(fZBgaQac8~a>;(xu#Q9 zPE6R={SpuG-}sGM-hNgnI&6wtDM}ewL896p_Q(8je=J4FBL&)(Mk|WYPfB7KG;(7e zFvT4D)27Ig+=;>+mpiddV+I01j%(jxiE;>Ls?BTKOa@-fVI2bG>B`uKBb)hmf#lhj z4bOoJvJ9H}!``$`tnG>UxS_A-Th`hcrb?-BpF$?05XuJeQUcICcg=GA`M z>#Sa)iwPyFt&Y^r4vI2R(17d?)U$t%N$K|y)PnQ%PxNYS2FA>hO#%4;2m?{p``)(L zzv3y0HW^Cz_;?!L76>#E(ia*Pasx(njdP46z#)C74q>fJY7=H=RthH)v{=ZH9-~n? z5_$!+(O>kQ$6XG+8rcrqjcf$FmVMQ=R3u(k>NAwrzRR?G0H7kFb~?#tOh~VZvVVMx z=yX3kccMiIZ70uSZULK!N->R^7fB;IYS$}JFIjY|;%WD%qo*>ZDWAJKI~N_Q80c9H zKZ7c5YRZL^uc$_jo%FoT;YH3DQJ+aq*R20VU++k28<5%Fsz+9Bc*c z)=K?64-Ht~O>XPtrbm`?8QuE!_;gj~n@oLLr8Q#~->w|GD~Wmy!Im3cTZ>b#w91K= zh&xMOeFS$vZc_IAbGn0>lc1DVJtu{ZLUWq0QY3Xqf2#e{9*o{Pm!_mu`RPjZDx7m> zx5qKj#@%5yCQ@|($U>z^5Cb1Ag8O;Vu6uQJ`5v_~0C=X**^4KHA*9`1y=?1s6r+Uj*ofEUGpkl%rUMSGd88D9|_nDbIXk~6l-JDH( z)+y7UKx4HNZOI2AV^xJ}@r??UKfX9Qn*oy6l?*y8#$o9yu4jT3wy7jTC0KGDI%^hq ztzNhO_1#bU>x4~!Qfgg-`)K-xtsei-%yJBms6hNUP_*V~rgV6E>#O?_D~@6h^MoQC zGsTN^KFj-)!AAmJ^ZszBC+j!iBT}=|y>}nV4IP)a`bx4v$^(=E!GRxaJ~Cea>Dt}i zSNEn(+T*^&DPcu}(;o*V(_@-i-q!iI)v+0hL}$Y@p=+P2jYmnIl%r9uFr>`bn#2Ja zoENc|>ITzQ?omdGg@n7d!oDR>w0i9mwIC%O=6%LBqIUFiT?Pt(G6X5YE7 z7>$6yX>t<;pl@%7G1uL*9)^inzG(r!mngLP3v+qKmIAl>Eqv z0UTOHSteE>9FgDiK*TY+GbaOiHNQWthhr8~Hxmq|G}{o`%s=2;$UZ_+iqM1+Y_iw! zrD$G<$-u!!RqeGV?DFFVi6^++m9=u`-*$J3vwp|h+j|vWqoE_$`H=6R zi#kJI7NY%;=Fea;K|`HL&vPmlAdL5Cp9*7uvCyPt++XTu?b^7VkaqNG5zmfz=qC2~ z<@(JVwp4(|Yj25|<*#O$0O*D!F2>DQmX~`rwMhY35hvbz><>bL?i6-uCJF*V47rpN z4zo6Cy?4Ue-=?adeEQ zR;^RIhb&=G>j<4+ZO$)HJbJl;SxPV%Pu*DG>bjemKcuM}m(NJI>$TT>v?Zv`WLQrJ zG~U(X3Vh3%P5Y3BC~OX(9YD<{%pH|b+m=mwAY&9uS3ODF`j@pS0r>8{)bcfZVchJux&RLAZHiT!}JbRlax#}<$61_zqLb7s)4xz6%1 zLiUhCS~!1n83jo$=X?Z<3S?ayA)O1=7cP%XWqS>99TMU(fQMpp29Y$YJ-70M#_r7%^~*=7h&ssXLS#J7iXJ7SzLvMOFrZ_axN6;kXxjB0-N z2>c`id7PXW7_~v9Wt1F*b$}gPrcm+nHU$*Q%I9k=5ad%uCzR)0seUU~^3v)mrIagk z?4zm>nJ8|y0o&`4zH$|{NI&QG9l>SzP*0!Jm3PyETLJi`AFis0jj(uZ`{QoM`!e`l zBeA6xTB5RiFB`AeodBKixKJMdHWcOCp^>n(*Td!YU1lF2A1V}7<)pdB(*+`z8rlTm z*Y!uIXfe2sNhwX&bG|8)H=qE@vzhJANksNYvV$f%bum@4Pm_^S#|c`=GPTS9TtXJc0#J9Lw?iz$967 z$YKEOp(}uHJ2o+TD{kkA221+s+Gi{#v(jV-!CbrSMFN9?G!l02>JGtoOE2EL?f0Q< z<{DNp;9_rpVDA&gvnPa(_uY8x53Cjv>tGb7pMDGLJJ__n4a=gF$dihuGx}U7%E{~M zl=$9Cci^Pk(T%$8>bDOqkkl3)|C-XUf-mm%bH=I*+w1P_wKuQO8fqVI4V_K44KKI> z9d0ce&k>xgN1kpTVT=RyN|K%IE4=CAY_-mnmjG3+?(g0oq?E-~kl^tqCgQd>`ilG3 z@USzSG&qou>P_^PF2lU{aa01mzB+_Q7mi8kH;Sn+@&WB?)r~w}q$Z>S8&)Fn`{f6o zZ-1ijuo!Mbc@9CL2e*ekrx?_V4DpE&MB?H3;m6?;AZkYql6XG7TT|X`H2Xs5CCL=S zXG!XBMnzl3bxd!8ILgyX@68LU%xw6uXi@9TH74g5`KoUnXfaI&&vfGQL^Nb>C3PHc zivpvk%kUa5Kg-y(7jBhzGShdzz$Hu$Af5@w9XJoKzAlckck6(9(QE6oMIAo+Ahx7` zDW2K0LG0*kk(|7!k-_aJE$_#{JT|DU&Rt525KGU6C*O=*>zqTah-vVR4-d;72W%{2Sscr$>2>)hPjNrBhR6RAVw zuQz;9k%llznTw360@f`9f@i_6=@hIRK}6(ikN z{jj(U8g<@)x8 z%lP_SeuM(i^yaADVdI1D*Y{5=&x=fV0pkbAV`tcNar>A8B(WE_bOd@gGor%9|Zk8F}3#&J12RE0K-=23zQLWw1baw5E*&ilB@L(szJY;8=Z*N(ugi(cp&%cj;WV1XF zF`q8URH?Z~z^15}$w%*dC8!Ii%*iHw7=1^GzWpU##)HiI#`8Ufp7+E!0+gO_c~i)> zXej%f@7%4>BF95hvY)K%1lyMDiYBc7Def!_r}%Zhqiesn60Vy!BRPkNP{*9n=u1Dj&dMNnEm!8x#%TwQ z>wxV8V-EyapZZP_sv7K(b2Oe3V3c}Y0aegSvjKv3gSfrMmDcJ?ja7xmWx7?k)R5{G z4Xyxl@NxiX^|H7S5uu(p&eyUnoR;Zhe(fgRW3g>_a9jnUVj#1tBBa&Dc*03shao3Lpq^|8DCzV&zVSp9a!?1XBM99Wmi0C&gK4sj-?SSeK*QT5 zXTRQ|!a*S)Li;{Q#8~AIl-05z&FdLr!S^rIu2w-eES4R5 zEW84U0)XtN9+y*6-nqp-2;k`^qe=Qy#9?>MGnHYf54dbbtY#Vpf!ujyt9WP>ext(| zUF4wpf`N%tC6kf~&*?hytu;pe$SnL?8@Y`aL#Dl9GHfcHoo!FvA%*U7J9k6R;Hfn| zHa=fW76$cTI4eOB4W07m1ztyw(2A$?)9hQ_4^INmYPZ>3co|7!g4B!xLW z8$htve20{K8#()zFoxL<)ClevtN?}05uM<&BtoJ4I0%J#J;N;L-!5)WyLV4AJck5>q>b;peVg32 zG>-mirLEuO^k~oWWuj$ud|?Semwb(WW}nTA?hH><>EzQJ>LHEiTNnQ9k-NUQMoZss zK1YZ3oBDj)PgZ|BE&3wCoQ4z6iA`2}djBBt4He2rN+FW|^v%RPJg(4yS3w zEV-{1Z}a67AC9~mz^=SqP?=8sD@Gnx4tdNu?=RPSZbVlId*x!%ri$c)I>d6qJaK4_ zj86Ns1S1KKGy0#^g?6CsxTI9k@h=greb}hldpa0RSJl2VaC^*kYUp!SQ>x9QyCzRa zW54_%34)%lzKwLD<@yb2O;le7uF2|2f4&DGc(za zc8N_#zO}^h8uL}Tay9}76F2}YakPbYrI(h#&D;a1w{z}WvI%nG9C|D+VFKqBx!r-=1GQaf#L$lDTr z-;#_21bE4hmW|i``4!0)0t@ATZ|Zg*a>x4*aN!Y~1fV1Sibnk&N(j8c?X4`(9kM*G z)6xF#6-fNyVcdQSAnnmX(y??!>`GIiMv z3ks7ejGhr+!5u zKJWz94vAh|I#ibsncqAZYmfbvwJdf&6ZD8MZmRKApWd0^EM1(Le-&rrhgj=y#^ znyt(#4spEu#`H$7b|(x}UfjIVVYbj>Bv-bXTD-e+IRG$=t2f@$R8P~J1q3OdmBM?8 zrELc7FUh)^z6@zc4K&-?+_at-I>eRj(yn+apymeuJu0M{VKcQu*gCrsg!%8~5Td$A zT_^;qcBr(Xops-Vyxik4o@CN5_#f(p#-? zDwzF>1zbFF&2}zH#9E=YTyuJ++VmvjMSy?)_G{DS@6i00LivXPh!1LOY?AT@)ARS@ z%|)kN1yv$BSV%`_i=~pO%9&2C$Y2DQ2q=FPQVhReB28BO`uXQP*(03KL^^ylzNDdo zBb|;skR-N&(@gzbF9m0{4Gb|RRRl4&I`7+@K>LWmXTz7w>Nn)M)HH|u)#iICrFssV zCxnOyZY@wb$#mLJH?^k}za1oQIt&jFw|EyiZcVaMyeP8`d-7S+Xp3-}^_cgApMR45 z(uX*={ipDAf?ffY_{fP5ne;BpjVC7sxDvs+Tl%VJML0_6WeRJqFjthBbHv+W$|>Ye z?gEwt1x02pKp>>xHr*$6&_3+~h z>Z)UQRer$-72G=3_{fOBevT*tSe~)M@X=^bv(x?3Om#|djuLUE_Ji}iPxep7QCSmc7jHV}W zFGh?LLb!{=BlL08?%AA8vY$k4zo z2cknea|Ci}=Zn^Krck9v1*jR}mrnQG(BXXA9`0Y1V2pp*ShLYds@&%sp6lEvdHmq5 zQrLvvrm?)YP*GzTvV9%7RCy`XGMBn&camh9 zx*oc|2snmsNZ@=_qP?r{*}LAMUm4Y!MhJ3tUH$=mlhA}F?TbCtVD~@4_NJUtITA$; ztrC{NkL1GWuDOdpP#LiW*ZA#oZi)-;32JpH!|GN~;EAnWs7wM$X1+{@Je86mh4@Vd z*=M(F=Gu^v}BP zjY#?|VxR87dBb1uA3^P$w3jgsuz4eEXWu%yN)hy8e*EYpFq5HR@U&Na#t~g{oyU_A^oyMHf#t7&{W?Jtx$Vw-3v1_r?{nDyH*$NXvBL}KSQrN zy^a^(z7I;_URdQ7Wfh6eEDm4{^x`iB92vnI>mV#!UP*;7H0Z8Lca@-kXV-ALDb2XH4z$9Cypl$!+m2{%ZE z-KeO7Rf z#WjpoF}{;3_M~W@2fpmlk6Z0KS6?*wQ)iMdNf|CWpfvdrnphvf`fH+Pw6#wtJd{c= zx?3YvN6vUTr`JO$JvCo&@WG=C$y#e9ZoS z|8d*uBE5ibN~peso-C_3LS}Pq@@tq|@s`g9XF)N-H_#kNS**-J`E@|AAZi zK$PCfs9si%zRe2M2@g3lm_qAtOu56IBPVnvHMc4X!jv|?uN5;!S?wP56|se-IS17{ z1tq8JCQ&MEYQ%m7$>yH%<9yH8g`jeks@6#yg*?e_)9eQZ+TE#~im@^sMdI#2_(TD7 zG56Pb4Yj3~S6$JV2ZP}ta%=r+Vw~eU^0SP+)MEPk%_7TtfO61HcG@iVhrGTr=`PU34aR`cTV1wk*NpWxOAD?8tL4|q`B);sdR@_$I?QSl6Cl z7JI&rZt9kuc2ypJFOX3^s9j>MqnM=PT!=P2y!QBw#RkcK_KuU4h8ABEhG9Gg-P!fD`39m*MCny$RQv^8QN?O3h~izj23VZD5K5r zCetKo@tGUU+o%~%Szbi7UDa5t=vjP6ap3kgMDL8@cR3{PImatP0nUyuZCWrb*Nz3~jW2 z^{r9)&8x$B>lJ*07sYoztIso^$rS7gb<~csW&3<7QXfemwAq1Eg3G$!uGIL@S>Btb z=os3kob7I@-95FpPZ7<(wl--fi^SsaV?b(K+dJ(0a%`a=Nhw#-$g(_NN9YLCsM&UhifMIVQ zpsj^YgvxJeEAoB=uv?~DP?%LFxoBCEzO(WT%QL?&8Idtu} zIWE7!aTCInO6C3PfycZwubb=xV#DpxjN{t)wfoLfm0L3}-8Syvxx*pXaeryB(6QMu zFyL!>o`2l>F5Bk;eY(_^uz=xfc>|>Z)%iNlH@?%&((N+=UQ?CopuCyYYZ9}uO&dC? zBL-uVw~w3KziLshw&M)Lt+r1h4r9x7 zsqk*;6Y?+B=lY1E9^3IOPy`>YbC}+(jl#pu3>U9oB;3d!#u-Mp79Wv7jPNC}&}5?Q zy=%fg*7~3?qyP9@iP1*xuEuI*SHxCi!;5_bFj*s-CRFBi-V-V-k`SSViklAOh47BF zdzub>J>j2M>>=o7pV=~dMXo1FYj~l+UL%TU8_eR;dHBdZ7IiK%gpX=7|F|%)g?<5x z-opu0KVWh3mg;dtcxh%(`CKTOcm@AFnsBn4G7MWPeL0u!f{5cm1U=(`&U)oD`Ar9L zew8WaE#4;TOHFc@O#%-4Y;oXdCq4yJ9OM-+T z$wbuU;_(fo26yyz(%b3Nh8f7#O8Mf!Cyy!ZDf&&<_kx2Z#_H4J4IH$vF54R8JNd^s zu@3JU23%Y+D!x}LRO!CEEWw(?7<)9}aPcOzb(8 z6?jwR8?R+h1k7z>5%ne<^8*?RZ5)k{fSHem*(aGS?t_3QG9I7wDDO%yK@uk2+(tWHGn@F{N_T(X!Z> zSpd7vdHhOK&E`4#-cGJyG&Xh3WGOoJoab4vH}Y!uQH)~ZLD!S(7*WF)tQgv)b97VB zrZdwo@(S1LT}RCHh827h^r%07BIX}`XW9P(>)P#=n*omFqlrHJCSM%nJfUf0z7JWT zKDAm|><)iFyrDG7t)1NKZN8DtYHQ^a2mfuVOdWmed1d%E#NkUBZzLy~j08ie?TIO_ zGE*wl?Yau-?W^;b%-d?d{29s4E;VD!^e5;`3h2V`Oul@aMWA>@{?uXp-c3yV!NL43j(NutX2wM|JtTcwtVW@h!%+}{ zC#*Ib(qF=ABUvJ;DRzBF>W&D5S!R5+@jDyGzGCp762^_yoNg z;=wri6h6acA}UbyD=C1^Bpa%Hc6Jywlrn7$ZK-1mhp+R!dECIleKnM5AZ8(R7^0yN zLosy*5#IGU!7sE&O9YW`+|?JR8xbKNanwJ)&kMMp=SH*L-_5k?>UtmU{Z$xj4DQ=D79piQFzJU;H9@!93k_&sZSFrCAE<@kM9w zELZk)7427<)UqLg`befzt_w7!k>PlhB}boJ*LHI-oyI9%tqjM{PAL z^O4lxbnFzdN&z|fU94y)HIg>xCq;B;y!6W}PFWy9N=_;$j9>-yiz(w@Gjo_nrEyF& z@$bL488mFt6=)WKWw(*WA!K`JTpR;7lkBR=7E9vvh+^%UqN{P=pd{O(7Ta2RLN?TP z)NOSms9M8g++3!|AZPzc2IlajcRF7NLz-Ap4Z7ql(WT zr1j^Vd1PC_u9GlipxVhM($K_1UpB73;gV6)kba^ewa4SI-b?oF@CJuNa+H|v5{)Ra zfe(@3!~FGS_q}Vh2ZFW_dri~j9uAbnFSG=(9Vx?ZezD|^Q2oFov{bHTTFpaeEGlE- zNn052xXHbp0JwZ#HC0uxg3ks~*tjlSo8`LQGHy+i~34svBxrQrj(Y?O;t@pJurJ|UDc^V=jZsC_BHCUkxc69 z)9%tCKl!2qq_3Vtw*%yMoK)8Roq4)fhCmU|N+{NPw5bpjgP0pbDo#ZJs?aH*M$G(B-vV z5T5JG>rujWX|87K3L{d`ZKRD}Fe`{ELnqEe{7yD^-!6o(Tk6;%Ig}d$`7vUJTCo^o zKeNrpoK)7uNAn0HbelPcSS*O?j?t7c_lB;Ns{JDHk=01QgRG?F&c;=nE zHBbHEPID;CuxUcnT}XC5->ZyykkJhCAyqxf82^k&a@^*g8e zNm!o+gNr|Zf8?ov`^X21aWWS5Wm9Q<@ZT= z&w%b|?CGFR(yIO)oWP?`E5rZDuo zizj-m_XAFgL^7@HP@hI+@G#=289PAjK6sD?Bt4ftap4m!1HA*w_vu0g2G7V$^D0H% zyA@x(ITmXa&tyIA(Da>8mJ9E!yC3T_VGtygJr1om)z*soq2} zHT|9-1M7j%Gz|&|X~4SQU(VFr^bl1ri?M8Sef-K&|7A`;zgt_Q8%256=HWVxw?RdO z&m9{UaFD|IM&Aav?aKazerw9Hx2dsHN{M~BO0^5Amv>tj{)%0Z`dosgMshxMZ-*-5 zmMt{-jl?AZVfsPBVa zaYFjCgtpyrh-zzn4>c?dKeHO3y=4#UL=9x03ZOcs9{|V`g@$I0*1O(_Gh&u|=~Rk( zM*a7AVUeqM;C9ufIM6?6Rmk6$i6MFChgL_(=b)O+?U!af1E8vl!riD+QT|9c_jFBE8fcO`M_)wGpfVyu z_)%aQ^=fU?%3r?6^cicxN_pG8p6dpMK5

27Ly{7R!i*9Hsz{&$^CTuR?U`4F=Xg zaK$5#ix6i)M&YJW`lLh3vSaU)&g*cLa6X9V)1#1S)+F}C;cOD|ZG`wr3SG{E%f%os z`IS45zGXlRb^Q#(_geUbsk_5UO5t;6D0x5jPUt+-2Z zw-$F=+@0bMElzQFDee?^7+ea2yHhCc?(S~C>~r3I&OYCFUHmhdFq3B`nIz9z>%MbX zkW(s@yQl!r%@rr7k&AdLtE)y+X+(z;9;;RA6OAoh)WvaNKgKm$K)v{tfXS%}*6DKa z4^V~x4ZS*#vliMEM)@B|h%9Jq%YB{|lQ06xX1Q5^s6*OkCZ*ui6Etf4;ROSDc>BPG zx-+2qtnW5Z!tNyGgvMrxrGT@$c?l>$m>lalkouLR!?3tQIYb%1JM#xYlX9b~`o7Eh_U zu4-;_$nZN zGd<_ky?z^XQ(fEGlZJbJd4Bnoaj3e+Ehv-wY`0FOYR;ne`QFq?4{V6m^!)&Y7HwCu z_7t$^qMkH-dgGRkL^$UR5$csH@~_HQ5h~hP1KLnwkK&~Hes6U?)2KE;8_b1uNgQ;w zaFu!aZcz{A{12X3jj73Et8>gnjD59EgK=Ava6X5hnYILJU|Wl5`VS!tM|7IKYS85i z3o}d^<2w2&PY>AO>6>u9h}j2M{(A@RSo94aMYHr@M6OZ{nf9Ul8WtJKeY-;HA>H0H z&A)f2YVbjB{w*#39MHp z`}Kubl^Oz@tNKSSv_7B$Ia|x`Jq;}Wt^3|>jD49Wrz7YMsGkqdc9ctdUvrRe(i=(R zgjbH8K4Pj4(oUwvPq$Rr_3Z=w4jGv_;vRibyV)w<>B!As(QALdDzr*dNhOUnd@1s9 z6wM$-A#V7vte;|bEMCZ|hhV zG&Ti3(D)$$|A<_Mx}zo7E(0laHxu3Mg*^s(iE!yu$-q9f1(R&HSHbHBzFQ`D-Fh8c zY2h&6v`(V%o*4*yV+N`D^IM-2YaGmDWf+CtUD~OG!^098vepI-N~Y`sCn1*O4>%>Y zQj@+l{@nY%F=@7plqLME+u}Yo%{mRsBT`!Dcw_FZ`4Z*th!mj*UVuA;ndGe;hsX{S zJAma@6>9TK&mw(77j{clvfg;iewP6A!!7{+(%vogWE_`8+vYOH1@27L-WUD!40&}a zh>sM8F>+?2-H$ zdOI$Y=V*w{Y^-_G)teRzq2Iq#_**>foxE?CKR4wg+1nDlm$hrGM+i1HIhtHl*1W?>6ZE|FW zKfhHZ6@vEs^y~v~aa@|5H+yTD(pu~b=lRf&-#h@yT~p2p#U#G4a~42YDqW>%Hmr$H zX>huNK7#$usU;PHh=TYm{>nBHT&;w#mBu{7u}*qpHg^ggKmRKa5R<`jJH0#hYYuc- zK#LKoJu}F!A8n*|jxa6K(|gC`(kI5T|31gQAs^C&PUqd5ZKjsE!QMgPBi>SA)<#h= z`I}|2p|DM~s)fDI(Gr^aGW^AL5YcodV5~^x1A2R?W)KVC)W!qc&Ma35i-%nePV4~T z&twUvTgUCoI`rKGDCZ9OmY;gBURZ?aJjk@Cu#U3a>6Zcyi}0I}mUW;^gh1~^nmUvA zOU^f0N>e@yn9NL%VN>beRSQfXqd)XpW<4JxpFZk4I=}_1Vqj56#4JF}Hvnxb4vHiCMplAIu`&T5i<&^0?>Oh@h~>ZY z0l#HGYF&h-Ti1GEr^T$61=4!SSz@oM8QSGeRX6vsA=;sAC}(6gm+qDNzK6I4`+1Z6 zUIQvDQBMzZeg8|^+DwQD)JUR=#dmV8CHMXXN&b)9nxt1LqDdw8r?WNT?q~=p8W#GP z)L|Ad65vcVSh6uTSM|ZZ1+FXxKXJ?2v#O>OgUzxVv|IdYa{W|m?a8CMJ1#NeWm_xS z(5tDRBeq1Ffq|YTrpVJ_oYICC!H`6!Mz$;MCWPPCI-Aj3Vo}V+jC`eOQS3w5oyXd0 zOQ;j$JUHGgrOS#GH2SLq? z`ls-jVGhO9R~#=pi=Z8r)0tyVtm-No36m>PAaoQv=Kuj9JG4R2APhSBdoCbN@(J<| z+BSj0XOTh#GWxmuM7m$_cLhjEx3XJgRag7>Q_Kub1BY-?1Fxj|xnm5vnf&2l0rlW8 zmaA{;U+5(NF!A*RMTW2~4S zLP4_chaj=Q2^~F9hQHUz4Lr{B4*lu>#?aPJb+JXW`K)&9u43R1S(Uj9o2hlv=hy3q zaBC6-g?I^#V#=Bbo&CJ{dcrxpn(OR+qM5<*QRCBs?JdEBYX_jkAw~<;CJ7&s1I{#` zm-=kvR`1?}_R)-%(T{PVGp2KuD;RVT#}K4^VIi9sI)~(MY+79*A8dpvGS8OVk7{q# zwW&pDRadT{)9^^oZ?MMLG?%yEcR=_8k6T*F)Ie5>nAb<;ShyG9l#}0u3!x}&oD#$R zU?3+gKMxv&mWC;dvai%_2j>v8-G4zet6>6@hrq+R2kFMk>;T+%Io@(_{!8QqY_IXs z+V^@OQyK`3oA*NabZaI95k|)j;Icrq3>zn)*Dw66sMjyju%&5aZ531?TDhFv#5|tI zBpu(2E}Gm7iuhuYg<-*$6!#9jGR`97qCF~$ZKPbHd#~`U;kydcjT~C*>=p>e39?Ogu+Cw9qdzclbVFw+7 zly{<<7TDhc;wm@#0RWo@Rri=E^6Cng(4rrf{)XRzBJ=i@#U42W(WbWg2?7j%V%@pd z@mmuPt&PSnIOT2~v1`#4hscY_{Q6HBayXk{7Ynl#*xRn(_w-jgAcYWX~=X zZal>1J1d@<>FKp?bA@ROP>8k-P1{`N;QXx*=ZaWb<2J%PCZi!Z&4#E`<)Z3Z-_ld0 za$@V3Y)nc+b!^Aw?E;@Yd59O2;cGa!@mP3p2EYyt4{ns7;!8PRXHRmRkLGJs`Cg+K z+;-2;Y|n2rE(^k8ml$(jO}ZH^qDP?$d9oO>ox{}?fcbx+KOTXXoxL$+b4>d+TaM%C z$CbKmx#Ns1EKv@=nrBNUIKMeG&btCv6gD1{53vDxO3}kFxpwv@2+t)yZ-$w|`Mc8Z z8OIy;VZ{wYmtn$m8te{0hF!wF1G%9T#EM_n<5_DO?yO6^B;cBe3WtTst`?pl;Qro? z9ZPLj4BfKv&)8w%L#XkR?tWJ#I*HDlQ(p?TQ*$T{*7UcqH_MJnUBt&8d$$;63xKF? z)4e>&wW_3IG!MWM2FwG1^vhC|L?_GpV-bfH4W!j3rva^)Yyr)4>53NK zDo)7#<&+XQvXPC@$P8m_Z)Z@-N29Dgr6z0TlhORNH26_VlO#RMtsV|rGMqolL`m2n zqk$2!MIO>UbSP?btYz(W!nzVA!l#lvyG~(~QOhpRR6 zao^Z6jztnz$HoO^*IfiRaekP)w2n~%Qfqoc*Q~ZjG;~bv%RQDb@~mPm3KKWFpQ<2x zacDT|hLf-^{|=?+VYW%KrA@bd}p$W9(GNb#-dRzdGA1G`xUGvZHx7|ApjHlG0VP~XQ z%t{syRm6v+HF=zNJbgCX+Ot9{(LB$l-U_t%p@9Dc{*r8_K?-hEw-0=iBoK(fIch)< zGI+A+HSL1P1uB#n?ETdu=*sMD) zw))u_1))yi2Pc5XtTJJ8ZgBd}H?DbXaj2o_p{|hXPm8L>Kr3E)A=7T+fZq9l`{vH&iicWn@x3Y0A)Y zz%KiuPs&R@wpbxfZD-{Q$_!?kwOmb1z=y$rgzr!-(b#K)lvZPN@xsu#$UBNNk+=j4 z?D8`fwz~zCAo5xpeiyj~-#_$tWl0a+regs(0I^eJ zDCxzYDa>PGuPQFUYfJ|$Q){lk=LFj;fF=?>=n5qE;VV|0Es$d9XnhFK2_6y-33tIZ&=jfy`gOjAZ4 ztc(Qi1+;edPTh8T!vu%J-wkozBMu!ecn)f{u-1kXGDkd!afslg+%KYKcMvK1!yBBa z#x{29Xm?nJu4C=JhjmKZvs!`|u3D)6ZEg0^@}8=TUjyU@xwSKYp$gSk=DA>xy3(23 zLmQQSZ;M6E{{1+fN*M~s8v8jA9JB3;6E}$dtoDuPb$yNRbttnqM;DZ;r&Bf^7<&Pu z<{|VC&&xjA0trzbLzSW2aY2bl_+fdv@e{fWrsbnt;CZ@~p@{SjEo0`(KQZ76vA4ZXN zY6;K&r=xE@z+kIP$6~+y{7~{Hg2|nX0TA7HSyt^v3 zBGdgb5fP2a4Jm`3bmqp_xx9wnX>9}qAq;D%>z&4)EtJo}En-kEY%!d#=C`vP*%^i! zmGkNDfZb6%svqXTIo@MEGK4US)#n-}N_$q$s?crX+3}AJ>>Tu(qHnNz6abV+07{5K zD)TzkdlGZEQNPP0vd649&8}{*Jf=C46;t@dsNHo~_L$qXa03L*9My*(G(3@xJk2;VE`VHSFNC>|Wml7IqQX?w_} z=rth|IK*+R^vV86Qu%nQ2-d&oo521h`aIRuGf-H~wThutU5t3!THUkxycU5PplE*{ z$UI+$eFJ;TrgXfS;5YmG>rZb14B^|QP6jC8AwajoedL3=^W&y<) z<#%qksnk8DkbK(KmpZ!@VgcGL9W8gGVmQ7eQZO-OvMULtB5LLKq}~SJ+ZEhbAUs)( ziY1Um96%XOn<2gUwI%GeU0~R`T^!&GR3NLx&RabT{N9M(g}s3ob`k)tVZXlWiZdiL z(ztQ6>NU}5Q6!N25R0~jPQE%^YN5LX~eJDP*neKh) z47n9zPY-m#rur3oH(wL+M#K~_p#Oy(Yi)Vn z0JEsNY`I~I(-c^HgUOaHWD!&=91&gj2K4P#YnQxe(6q@V z>Ma8VaRo{1;`^gfypL*kh+xXP$n!-e%vu&Kyl}H+X}P9~WQrcHlE31tVLluBouKhj zSKXS;@Afx135yQ}D1KS`RZbae6i+QpKEs%jI0Z8GPj6XUpL64aM<+>`q((w+Xn^EC zhDI!$nJ_61@bo8;u*063#_VrgV#T2qJAl^3ps4m4iZ7OibO5XoW) z*+l3pl*)azaIbBHC)=j3be?7h+Lnf4^bUNgJ6qzaVop-;|Lg^T*k{inCAb>5BuKt% zp_OXX6u7y=suS-{^8MttzzxscGVk^01MhXdd&d+ z5oD{uj%*wKDu|Y@=+Y+xV^M&=cFCU+M3KMb62l+(dAdZopm~?f1moGB>DErS=CNzo z^X^>hgjmo+%_bkf1|K~gY2l3{%5)}uYrwCv+ z4sL$FBrvnIA~`c$_6vt}_Beh+v^fGYm?pPdk%i-g9`MtOawz2Mhc`>3dt>|AY%U^g zw==GbFM43-qv!=&C_Z#+&dsfz;2cquEck0OH(^$Trmwg1bx*&Umu``=M13mefUNs{ z6l&zbRvG5ybDwO{>xTQlT#aV)^iW~cEQM=Zw5M3{SQfVqaSodCa*#`#P2FO^1^siH z`oUs(jx9`;f&IYV`MQ$y5#1r;V~oTN!;i+86k>k&{(Zdl5WcVmgwxRI43R&39QfIJ zG2imHJofpj1LpN0V+70NLbKTtaQC2+6RWzI)ZC+6uUTq@>4GBRf@li zhOLTpcJfJB=RRP`R2Ds%E1yM|{k`fu0-&#or8O(Ne8t_LSF&(8k^R^Dl)md$1?oIN z8SXW}ft}C(s{_Mvchp^T7=uG1I+J3^nX9UfAsO&3Tiv28d#+cHjC<$T+OG&*8ZVaj z1@sr2&rAZBJZ2nY(tD2ajYKk zqS|1R5$hy;hT$G=CM5dw;63(E^$3LBv#4Z%0PL+LHHfq^Vcw%3N9w?0%mbJ?yyp zBAxclDK>$R6V?ok(8Ehc37GhqwOP)+-LpQU$ePGCH0P^fc+X#7Nh?pJ_VMW==beb3 zwmDh!k^}|3*>+YN`XSkJpWpAh zD0pHBv^`1uQFBr9Ami2YPWuPfWTGy^o$_zsx+b0~hYSIyTy2|)Pa|-_bcZv4l&98B zvkt}t;bFSj4qYQHm0=l~BJ)YvI0d|KRm>c<6|6$*IWTo;6Sqdi%kXuFY^eJ{Z_bb)pn5t zF0+sjO3#iSm|jK(FAC}U)oGA-VzMDlu=C4o7 zg?g0IWzbazb;&=w_?}09c8sEg{82Q_Be|Hj>RL$+W%_guO>6UAB!tzTZ9T1u1Mk_m zv&nwwYgZ?lPFf12Ki6>}IEgFYQY*Gso14?L`u=pp6x z5Ccvw3cJRNs=z*?B4RGq8_aF8EjO;Ky`v7`B9VTDn`Hv786^NSzXBA81u$_SC{q)Iy7d; z_&5A5S7utu(B)gdEN%(v~NlZO*0I3)K&5c ziYt5FJfe~;&EiIS8}~)#ZuK;YJ}+^{nvkG{ey~&1JDg==^}HjQZROF+tl_P4&@I{- zi)Be_y>d|%<&RLeC*nH!z~pkwM~?3r5ZBW)iRdi+j$}`2o$QbX*Ri!?7(87XkJPX1 z!k34+|0 z?U5wrl7yuzM}5Aje$y`SkkfDD02Yu*iCR0@P{n+y(p$6#epU2c@C5=*D_euOZhFHp z=18sX*$5y{fPIP}s%bLq|XSH1gcX7&TYHe~V9oNW;PAB9+Y*+#u_;J*>XB*A|9%L3!pz zdIt?vVATVnDSt8o#Hdh(g#8Eeq%hg;<3Agh-9&|Qy_mJj{R|m7A{%S;Ai#jq#3shu zyRFc9>WfN?2QRPtWrq8$ePGQJrf!d1D#scf&jgK$SCnG&9+Q0$tTz-*HZA8K zXsW?fLL>LQ^ZFFvj{JbWCghY+M_;eJaD$Ma0#0#;h+x#y0tS)uBU5O&sE^t3<5SU- zw$tRgV%jE8f?n4J*AG_)Ph5iwim(N7Tjo-09TI5<7#U@(Y?swG#dRAOnnp%J48or1 z_jC7Mdn_f#VcS53T!6v%9#*pVufpmjeG#(YF1d_$C66023KQC~{)%;Yxmi#iEH@wo zW1U(FVkv(VZ+5Mp5c^EZ;}tJmrP;C3PqzF=i=2rN=DB&=?l1(;=_tC*99==RW(L~F zFCR$W!lwTUB*FIlYV#Tzlg8qULTFOyAhHz=jyWoaJY7NMTQ_(@`B(y z<8%DbByQol;q%gudhm2dXR?-m#@1ya9=3%032nce{P~UDzTB{i7QtXWqM9usQ4=UE zmv8b!$ESy%5k3wiV)7ker4IQ0_2rZUI_^?t66;@P?$blKyCAPg^3_XewlI zXVkOd4e{|r>}I3_$_<3`v+EAxXL3SP@Nb=Zm1+lpN-{G!w+bgIqJR@=^+ihW`6VMb zE5X{WF4lA-PM0NA3u|o%*J*?8Eg)UGeVfT|&jE)dIlUqDVvDy8u-Bjn?+?@gn^0iAgC`-~bJ9*U ztJ^=#7oqKETD9%p`#fsa;K40KttNuMgx2*=7F@yBiw#RF3)aDYcbyHrSE2+bo}>5v z`TX$Tk$eH9h$o_>gg20!DDr#fqkb~pwY6=$o+XJ9a2_r^pP!k^=NW$e$B`VWUXyO* zW20c)`7s-QH|OW{AjPcXaRwy=ytl?Nd(oD;vNg7@a>mU7$n-vj-VH+Y5F~KP2tp{} zK1Lvdk_(3Ej)vGC>|10BN!;uJdEu80Q@e84Hv;Wy*4rrcHT#ql@LlLW_h>f4aMio7 z&(xGUq77Ywk6vK@JEA~&r8=V#1)H@{M4COguf7wqPc3>+Xw3qdvyc{N!jedfu5u9- zT6NmwM__^dzO0u}wP?Z$P4rjx>Q6eWcU2P-yY*%$_w&_eDbiRkBw9>!?K}lsWY8|x zwEKJS<|dPlx!)E{L&j#T^UnS%9RYh9FI9$^oK3{~uO^RrF!tj#U2aq6wtWvD51fV)nYbmGd z_1iuS`^FnFbu79a>?wVt0sOAi+U*a{Z4$X%AXe{9iITd=_il)IB+ji3V_xx%Y##7$ zVD4gTHqZi_>m7cmqP00cHN^2#AYae0!mg^Qk!>l{(;kM! z@*rnFXLTUaXtO*&%(8N${c4Fb>Mh&7nWkqh^km>B_6_nS7PbaH|MSB*Pf}xOn3K{e zao^ocO}OB_q`)bh3{9dic%kL3`U1|~3MeuLn1nM&jX&@3=h!+9+>j%bb$di=u8W5U z4NZrO9SJXy8s9W*P`aU9+|U4R9cOmI<^x4lsPi)dH@@{mu)`kN7fCkKz7}}Tdb~kS z4VR7A=;$qb-@mF92suTx=NSvxDWHP!Rsn1kUntp|zhtj}_30VUA=faHt3Y;ce(_H( z)2wJuqdnU{xW-;^{E=-EimA*cCf?U8FI(9)#xVxK4Lyq<;3VhYBDt4eVt$UDWNm{d zVlBDrxF+3PgvO{9Sk=7$+?HDD$h>VREbI#@Tn%YQ#~au1@`|DamufA-gTFmx4poRc z9K|(31qw}jth}4FKZ=furAKGPc6BY*a`;jZtg;-@5`usj$88bcMn&NdBNq1TuT8L&X16z% z!Asd<7{vqo?UwA7s8p4@u2~wKFVkiIB|~FJX1Ls)60DdPC5@Q5^&`umbcmE~PU?Yw4Ez1;7as{m!t>2oRL>ExYG(KSOiFjH3C%Tf{- zlO+wdl363aAVcd^8Q!9rDb6hN8NkA3JYY;a5b(K-^nr)mN?2<}ArH%>>$4a&piWto zdK*2ST3A{THHQ^evwE|cqsS2OhUAdFo!FvXtRhB}S<|>t3=X_T1+YzOnUuDd6+tsZ zE`x zY)j#vL1(3{ot5)vP?Czgi`BuVSTQuaQ53ah0phfKhTDdjTM+x93$0{GyXfFVf* zXkGma#BBTlGLEx1h#SW0hZ+5HLh%@d1oMq&STqGs*eJC@2in+zQG&v6S^hs zN$WkjCEnsUt0m}RhFnxULSIsu@U6pB(sE6*ndso6m&Ef^Pz5SiysMT>`=>Ev)E?JQ zuyH`r#(K!}>f8%wGk-;`eQlT2SY(IXLH6g&4$<41W-?~<`(l`=Kaxo^IWA;RuQtGR z=RzJG;=xP!Gc_4f{4*>b7n4h#A>sk=Sp+SUa~Pe-ig+w7w29j6&w$>0@dN_ZCWZ}qhBdJ zh0m=gFHPX44VzOt&!Y2lHykfPiig-|DZ*9UftKZqxdLjWL z*&s&{k#uoc8BqZW%-(1G@a*;FUWMV5eWGnX9?COOpQEm(wwTO<0b`R1#;;~e?+%EH z71DI+VPfDT6nr?lDPni{Z3}$RN-JYMwhJefVWaXayIR4y*yEx-4<)xBO4bJ5$PTdM zrTrEA>(g1NIMxT5`zEiOkSx>|*{J@($pC_{*Rtm0 z?G^rEm;@&8rO6#J4M6}Z#TWxJt%jcS;ZrU~VD0bnzwj+0U`&j{WTxU9kC9JV_TJ=_ zKOA5R&KT?N@R--iEr7z{2FdZTj}~zfA7abqY{eQ7!}j+==B@yD18aOFEg{DJITN_4 zX=}|Xg+f-(v}rVS0KdsNHZDcgf_0-&TFAd*Vtn528<|`4Op>1- z{iymcW5tG|$Ri!M?F#Z+7=g(Ls`FRvW~Vbq)27D<**UR{m50}wP@(NU#yOySpX3oD zZj=`AEgc85@%|Io@~<>0Ga{Sn|1Dfs^aC_S7y|#LyNP_K>ioyczs7D#2{@%pJt^e? z;f~06fE)f_TsZLd-+w_Ow32Ll9J5EdZ50Ck8K{llpM ztoJ?uBKsfszKy@kjP_7Nf3s`XO)v~I{(l2m`Abl>vPyS_MGAut|KCmd&o;%1T+?HttT>%HG1fO ztZe>@;(m$%uKsVJIXV#h?a&Jo%l}&vh7aY1^#2{|CoEuu5(#)i)?UO=k^g|=|920N zKVli8Q6~RoOc_a6{13h9KihRq+4*llmzeYrky3^}Ff|I{O*I5j# zv|YS2UJNYm#fBQC{d?Vi#39y?2BQKe9%!EU?+*FL+n>HcjS>TKoPG~{-?WE;9H4po z=i9hYa{t(;t#|+eX$_eO`u{@SS^U0p?|=3V@V_F{G=F`TkFT8fr+4d4{~bMmZ2J2z zPZ)BSP3*5Dha&X>qrCq!h<^{L*BXjloPwZkjos;QDZxLFW%fV_CuyL3f%>nFqSnX40$%)sloyu* zq55x_icn958^vXDf+3L;1mBs&`8+cgL_ zG&D44$ZC7Ulo0$w#gm5$+2f|F5kE?;n2c|{4Je?5g8X*Gp+a5fzs!u7Z0A&JT>!7m4F2?I9+=#(Dhsv#d9g*b?t>%SFD zrM`FHRQ#Dyo;LMu^_P13tuz+q5ZP*>p7`$}>$?_L>i_}XI=HX!LK3(h-))OVq%-Dg zR>Rt@OW!Y60Hkg^Fsy|Uepe&!d+0Dk7L*eqQpnP^(M#;><#zRzn7ehQeruB&%cg!d zF@w7RU*%}OYtF~-c8GCyDWjx6R`t>;*U3YTU<~Y9ne#*5-Ll#Jh_QXAC_+;VQ<3t8 zyDSO-l!r)@{JP@Sj z7bci)73@!`iil>*X`Dr$qpb&7ckA*7@ofG5B&mrGS*)0~Ow#VWSxp0w* zjhFH-HL1zZ`q`dg0A=Rnw`h})JEV^c%Fzf<*%CL1&iEcw{`(5#0NiWVb)pd5S*MEQ!+Vezt5Zlog0*uS_mnWTVEs6BRk+t`mS2 z+#7v8@QrC}{xG#eogQ!ZJ^()Z-383?$#IR+jL~0rgmg3~$JR>agBGd$C!SPt7>cxj z7H#_78Rc3u(TwK6yA$*Yw!j9i2LDBY(tHMAbA&oSY18MheKrQPrY-tkm#lHw0(2x z!yMxMx471F-Je8xJf1&n^7TptJ5sM>&~i-{ z$mKq3VQA`EC}h)|d#c^dFeHgZm`axun+DOyGjy<_)tG#{y(EpNmCvh!W=T=ZrlFoJ zRIoy2!~=SZd$>Oii;p=fibG9=`;{%p)*o=7oHWODcD_s4*g{sX>gyRdd4wA_&x5(3 z=%`?gz)|WD?&$ItA$_)cZE+z&PO%PZjNw+HxqFK0Q+Cem;4D}jEPZh{TqgbgNthue~E-0jM118)Y1Q^>N>(Rt5Nb!vkeNCv>sg#SD$OQ;QyB>70x* zDNN6+JAh4AuH?xXmO8_twx2+b#jb{w{PYovOsURaejz@w7>@%a;@rF0>~dk zUls~W9z*DW=Ruc@9Vv-J@rT!%Ogd1}4kXug$BC_%W)}f;${&Qczv%b`iGL4M8IDgS zOR}bT0IVF%m3O_if5N1V3sbg#9N!14_>J&SCLfZw?vM;V#dB!<>P{K_NuTM6_-wG< z57=b${o>b{K>6m){n2JsBO+e3l2WY(Wgd*Oe&@guCf@26{rFO(d85Zb?UZWd9xVg| zoTnv6Y3@I`bEKDEEf-xNTuvWUUtDaIeyJRgU)m7snWJnuFVXDe{k%-kM_o==+oVh5 z;(3_;KAyoIWlQU{H)#Cjf;b!z+~S#IHqf>B{rEv}dBxys(_aOFvudk%k_q}mz{+y1k2W!SB-W@;ec^Myy>;=R(Ob)6o@iA@t*EeD z%8PbM5TAmZsj-|z-(Spq*M2e!HL~{-6amVAZz~*qTpA3;&rkHsRtpa*85U@H>>6}# zRFLmOhPyq=S?Q15FO-0X%TcfUa__upn5asEa1qO6MxFw1haarqCRSvcDY`>af;5>JI%Lq23FieM9Ajyq|yG~i+K)h4ETcOL^M$+ z=$?gs*yz7NDN|PXmDsLzMJTF1AmhUz(PjtH5BGJy?s3}YLE?KmJ5ed5r1#EaNGlYA z&wGg5ssTQsT+Yr43Dojo?35n+J1mHseLWcl62|2oG0cR%sM=h755g+|>uucF;{DUYQ{9v|hw#`$de z)A&A+zA1D;_Q2;Vse9F<)(nI)(7Y+D0;xSu9@<#GDd6WH&d70&;hkwox0$+ zQ%w^SlPg&TRk_5JxY(|~rc z2C28mt@bL>!z8}g!;dyeqe#dPL8e4@^bA!TzIR|od2Kx6G3h8l2@hcWfL6|`__)sdzyzO zx4N*@V~k3%+2xv$B%EkLCIPS3FTtp9C9enek*cr19T%z6-Cq~`C=wrV||*RWYG&Vbc>E^ zfN2IR?>=;yemqK~!E#Ux^IYJQOXo8|{`UpKQq!j{nc_T3mC$iVi@D1vIgX>nT#97i zu9Zs$PDA0*7r4nzS~bhD&3?MEE*w5Z`1bb6bDd%l^Lz^F^X73l;8Pgz$?S%Da6Gu` zCF@Y7Tt-Xf^+jzb;(BMn`do}1-FE|s5u;SwWURm?(XKH*!g+7i()Q|<%BV-`ffZ4x zaM7vTtmJN`M?>@NM>)});K1V;HtW$3z&%A-<28NDV}-)m z!MUfD6OXOO*ZoOwMhRIJy|4sVsAjV_fC=wJ*bDRZ;SNfLN-=U0MCq{mj3bxcCYINi zPdon6)9UzFpAOok?mMW4Q=Tw|HVLDbT|xWrk!5_z1Z$WP3RD=s5SN>s3@kd8Zo=uD zISOP8B}Rk?wrI!ML|z(Rxy>j((NhKTEpF+cm2p{XZhlnSyn0^L6#H14&E?D?pzI*+ z1>B~7Rw>l|^2-+f5sfsaNAgv)2&MEU5{*QJ{Oqql^V$9B_x?|NFGo|k0Wn>cmc+);S$p0pr`O-aON9{j) z99z4fK~D$?P3e=0W|h$9?Nu&j?e)<`S|GB&Qs@ZEmI}Yr>9lop) zO}j%^_lwJq-ng=4WCkb%A8}--IjPC=)>XEk@P&3YQ8Z7Qt1hSY5!Vk*U(PU=^oL+O zHlAuY?e|Wh6B)EICMos>?TH^5-3LC?FPhSqe1u>XlNZ6B0B{sRp#zN9n$F9q7j;He zaA0VcR4NWhY7;5J>#Iaq#~(}ts^aGDG+}!|#|xYrYjr6UwR>J44M}BDLz1sFqrZ(G z#&Wp7k5mDA$f#LaS+~d2QETz$3Jg-#mxIfO;p?CIW;%{7kxBDV{TGmD57)}4-_>Hf zt~Z`sH>gh1#zJp7P6vhL!12L4I5qQ}2sqjNH>qp6JXgS&OQ7&ivpvPVZiZZA^@xn{$ z#7j0bpPxMFdQXwcW_t{#os7BhBc^43z~jHzB8`=aY0}R=<3I^#Mn++uVWn{K(aV_* zqy7ETW>zBMEJ{-dh2#7pBT$dbdW^~2A-SHq-g}RfcEK+ciqWI7{wA8f1oEB;j#4RH zus)e>m;l>9ASH!cXe1Ly!C~vYa`!#{bYtc%vrW}LR~CPV2sJhONKQ_9bmB?DnBKGE z2p{Ke89v?tWfE@f%iZJSy{wP29<3_f z*+%&G_F;j19~sG`4yzkZIIt`DQ3L3>tDar4gg}zyNf$uc`uz9-Ih;X@Q^MKtvzubiUEMq;V!X`gB zeU`C4Gc zdn)Kw&gUwvtVK`SuAXm?<$GMk$Xt3{08Oubtrj7*L6=!e);}Il-Qj*zsUaa1s_6?5w7EzQOg~%?ce==XP%8D326E>}%zP-l*@sNGAE+=$ z5Z&lSyGE8YOWoB@QF6Zasd%{ecUZ7cXx@&>_@^R_tFJI-y49Vk3kANseL{uMZSm20 zp;09V(n*8K)Pc9IyU*(L^IGTHS>=zblJu6QLDwKqXkVU^zpmq?g!+h#JLhL~4&7ES zlC|k`RXP&ESJunz(Fhos3#yKp>jPE#9ly*7&U7?x?=cxEc)PPMldtCog$+NBLv!sper;vIuAW1Z4gf)#W_pwT3tI~Hj zS=n7nnmmeUAp@i`)iu9F^$N?SQfR1~ZwR7Rp?xr`J{VD*7rt3fdxu6-HiEV$i{jT- zk*Sn83ct_)-j9CumYAW(BX^GZaU_c$0*h;wRXUb1NLGheE*fDPuHNKF8YcboLmYux zHmiPn>B!G%Sq{VZ$qEV%6oDcf>kM~5opd1m)}VJiCp(XLN|pYA+sB-{wO>n889Wr1 zzR7yyxIcv)5IBkyQ8lT{ex9xNTqRYqernfivTdmxQOMv6tuKu)<0vLH?_+Og;GewJ z-dWQ0F)GMPuu)3H8zi-LpUG*btKZf?#OIy*qyWXy;@ovTfx+o{Xx|q3*13TN5_S_G z;R-QYzfiT9@i9{gBnYqX=BxaQPR%{=DpWBtz!(pj0&@$kr) z$fSWWWzI6^O=zWArb&MD4tE zw$9iG_P|b=!lzi#I{2@h0_wvBbSiAzkGIya$_g?MsNvlgSQZZFF&!7<|BtM*fQlpA z)-V= z3uY1Zhr|3aesKpFWl@SWU8~KlcT0Y^S_-l90b9As?WLA#oebgFL$GAixjt;RH#>P{BQ zJdb8PyPRi~wwflJ3Sw)7OBxm!>D3t563)lGxR})dItr4wiK)9WT2RZJw@|SL%1rFB zlO=Lk;QDB{g9u0@CpNyqVV4{v(8`CtE$w+tI(k3G9lx{O*Ur~KVd+-dDtBCea1`}To)FwI;MP#S(_v5StS9>a>5eT85$4vvt=B~j`J$}D6YtN9n+K&l9&VffeN-jQ* z$O*l#3iQ!f77Od;A)mV*+9%gN0m9r2>;l))M49nAqbT%RKS)4!g(QC}J8?QWzabI*2zdBE@0K zf^0hH^z>46vf*jG$wkP{6n8VWGRThRVcJf8_#z!pVGViN#J~tfWR_Kl=o&;eI@Se+=YaV6eUKhor&+pu>`&=RSO6 zzs~>GdK%AW=49ep1nnGoa&&P~*KlZrsu_Khv!ODgBjsauttAOz;~i?v$o^)@E9vC7 z7}jbm%qwE-b+iF=vftT|p8`+^{a<#g&<dzWAd0!QBw@*Jd zTzthW^r5JIYf5#w=tjC)g2`^seqIx7GwBif?pPxeJ}-nYwhBNM+*gI5rlqDU%{_JM z$nqzhWc!#&_z_jwEPZ*bXw(WLD{;IQ*yBk%G>+m98E5=<#ot=aBXpkdS<9CIR!|+$ zC!0`GkeDx1#X3KDbtU;EH^c^^$W-I!1p)j0+2-Y@EjOEatCO25jP1bL{mxk*1)J+R1zI)3M~lI(PY>yg0F zel}iQjDH=ZCaJ%T=3s6A8q>A(>6XdCHB(9R9UToGFqo@5CJXHlEa#<6>sOX_$PL<> zQp@g{&&R2??-`!uFS`3sHWGzO@NJQ*YC5(yA_Y2USTpH7mZ|GlhuWIf=X` zk#4^|Y_8Q3xxb+#3$v;i`p{2GRp&#^%SKNY@ZF|-L@M;rCvT*ZVQXP}IM45d6JMwh zEg#nL;TU!xo;ggl!eo9?)LdZba6oWb*Z8;+ELK<)QANi=xkUy2(Pv{@zhE^Gm$1Nz zVyz(1*1;lz*U;|;ffZ(J6@Cv}otySL70JoCnAfh?#cCdQVXm57COZK`GOHEI5&N=P z@6$vQ(S3tYS&OFw!5!3RI++_%?$V>Y3lU!T`5N=7ho4uwb8r*T%oh?EY7B;Y77v{V zjaS-)2F=o6=tx0++Hjo7vwiO*&hr_(q2IAU7rhi5xrXUljQjdCUzR>JN zy0Xy}x4OzO(ErkNf?{n`XLXH|KT>C_f<;Na)Os>YSL=Aa(PMH+jy}AC`Ywr5PVoMn zm8~BQT%w9g9_wob{e0{&LC}0P^m%$hbfUMVSaSYo>Raqi1!9;*D3Y(SmZ9g;$$7W~ z*{UdS=him&A{NRX3sD~(MGc4hEll@HZA)3mTriBJ_~4vU1-+UTz-T?5#b8h0L}qn0 zhhCPvrwNJ#Gds+0E|XtMkLkiRu z?~k;;2ZGuU$|T=(K(AwCCI`ReR8x5xjn-kra%x~ZW0E_r!^O@g){uR5KJHZ*>9k-S zM6}cO&Rf{P-2GKbn*TJLe8T~kmC5(=fm)HfZCK_aR*0}#NHf-_9goJWIB zmzaH(=4N7aur-+gqMexg*r*>uKHNg&txB+73Fvu!$$C}qqzSrhy}b(Rbhu5Bx9RUB z8yK3(fR*Hi!98R77SG!p+A90JY?a*y3VjNf45qbbm+toiQr6-gmldhVJ--ar8O{8W z+b$2M+QN8-k0zSsbD6yQ0u&)jCM!I*bIAJ@07NQM&E-TEJ=IoOz5Sz=TKOf7Y0!$KpIO3f0Yv~Z z!KARpm+Rdr@ta9A@O7{YgAE%H9_Ni2@eIr;uxn*dT8XNa@6z53F{s|{0!vu#^I0pJ z9T<5c4b3YHXl7dGj5ma;5H32Vm>e)2=zFC`glQP74jmfdblE4E?5)p5{fv(um)M?Y z_y`x3C#z>oE)Rko6)J>2N$e`g2+vOUn5Gb%G!zPwP;OMmUzHY&mnYLYE;a3!q=Ys7 z;k^{HLG5jAb^c1CexJKEzk#h`?zMmfl!2%F{@Wq6>BG|jXHSlJllzb@*t>9T)Bci~%{((kKOk&@&WuMeL10pH|kpz*NhP74#l61 z9iE`+Ssl{M4k1(*LNEF^i%W0vhgkyIZMY8WJkxlc2u$HsswuN`6mM{?PmB30%D~r4 zeFu+EZ3HFA3$}ADnkQ+Dpb3Ri)L63-z~o?jkN31648(l|10A3V0<0H3^B0;=8xM7~ z99)A5v@;`SVdYlhX)QM3Y*5_VD}q=nr6SA`5&XCx`9AnU9)hSOcL-eAc@H4Ot0y2I zm*t~o^Zv)9)dilt;*G=GAo*n-oLCAmYk{(B;1;a@kY=cR3#y+MVkE7xl}ii>frn1sSC!87taAv=hwLrXERBzc21Vp@Pr%C++fjqD1H62dl zdwKL;ApO<{V7YMuHlEB519)IF)<(!vjS_<2=`P~4`tkMt&2)tsNxo8`*?N^y{T6#| zjnVQ@Y>bMu;15OZm-9jjk5~H>*tz2Rt-dKM72@5YaW||MZ*oV#O?KI&kyzS0tp=21 z?k4p}RL5jL#UnzW4D%r}HO?npR3oXiARH9K^4sIWeIAE*{s2`AGvFV79}vc6W`N-7 z^MXn9D(?`HKCrlE$Avg#?j@UyM$iR>j4ie^$s=AN1(_ERBCDgtf4SzDB7@m@{j+Be zK=U}R&GzaCZUJmL@UZ+f$2xUUz}6zXPDf_Zf|xS$^j!--K!(6ro%B{}XUN@YES%4t zA8Du~JO^dWbL6GYGy>^oyRP#6)07>UB)m>axar;Xpa{l$6hbtD!~{g)Om(sq4V0GAFP-o;lpqZd zi*7B-;Iy!>qBhMy{4X^m~N zHT`TyEUSVmP{@Tx&5i^DocEVa1hEQLZAyCs18Roa+fya|bZ>2hl-|wt_sJ05u^Vr2 zKDlt;y^pfxXBi_oe=vg#2L``Pmmq$Ay@f4h=vZzB~&! z$@Bwd%krLxOfP?ds5%pD{w6Q?uazoH!yik4#_1+=yH=J<_k0D9S~fqU=zJbf?#Kx1 zcFWw+KG7v|*+1#OIf}^w(_inCToT+SY}dRmA=9)neL%_ldi159=;dCor-xV0aAV$8 zeB)@lR34-X(Q>JUY{8iT>CWqVNxP_^+VdH`fX+UDCEYt}`7S7edouRB6zyp>*{>>f z!>9DvdoejG^=GeQ@UN4H7&y<%ZSZvoctQA*h#gwrD@G)}qNnQQE0|QOrrE!`~0N}683x1s!%>1zL#_u++1BY#GmaSA$*-3I*OPXOjV-J;Fv5=dv zwZ9*&(B3h-B{q~&WjoQ(S>KMU2wZGY8};j!t2=0w8Tc|z%`7X|+rAhp05uRRvf6G_ zUjSiiBwk8mj;JprEFeMpP0BvWQBC7^>T?cusi zWP^rdemEca=-J^a37|CO+v=J?a2H>wARx8xbwQ4#9G2$=CR69*gr~%^o9`HN-sXXW zsmYae2W6uF8n##8k$Ix6$vcdo*;>1m=@dyTFHN8^+RS@ro>~Ep{xpzFsT;aPY1R-} z8nNI^8cb12{LXe>sy$_Vp%pn)`Zn$2X((Fj`cOzCrHJvF?40*{wllVdsT0x#T0s|j zJ09UbrB9{azAS#l;j1xpJicMRfB#$*<~0319{Cs=n1}+-^NAwKMbd|2wIkGCi~ML zd|P1im@SmMe(0cz^(`7OUcQeUKabjTy|@;T<|DTgeKn0nc(+l>J6esky-x?E65;pA z8UY=Z?_&35-yCA?S^@k`Z%@{fl==hrOCN+MG#@qHnbJvgVpaBtCLX@l-6m@0=X)C4 z`3}QHJN5I-+ng{0I8sJ}qFMj!UoDeL(n3C=5DnTwQn7t1dRk-CQ?=J51NFP_hWefIQeZq z!r-wQ&Qhs4T;S<1=Q|2|Z&Huyw-FY@60&u6SdS<$bojeYcwEe!iDBvgl?$kCfLVa= zKqZtkuStE_d~v#-!a|{26Fx5XAXW zW(=ob)8qWG9)|9{mNLVj%8b9O@|w;$&0g}T&G*Zp!w%%V$*-c#n6WWM%;`5+9b`)q zW5|Y$cQ^h9CQDa8`EEWi`or*(L(!8}JbO93sN<@X5b*{38LCXOFj*+$hpHHib_FHe z;Olv@4sVP^XA&I$IBGT0z#lm52#M*2a9I`77iLPhCi+yb(1I1!^{H#aR%pKwNG8In zAf(>#O9{akl=^*8qyGxw@fj}-MgB6px~V(QxvT$_@b$dhcH@JL%hFF~>Q90pH_9>o zjg4sHxUCSjTTjD<&y9mB_MkhDtNZfz94trESGsN=HfGw<9}K$SBkz-e7zMGZsafQH z50B*u??)_!7aq^?=u=3SCpv)^{Hb_9);9zlNm&?W9SQXaQ`Lfm(=E_lMkX#@C2S82rz> z|7L#CVPE#T0}Qr%{|M#(+a-V*yTic!wmb9qeOMkVFH}YU#S4!|L0yRb^FI#18_c3m z*Zl>Q7c2Ph(f*A)|M?H#)>jJHWN2Uu_>~BKZ^UCn@SfndiXC0#qv>aX;H|ZuA=tY(vSFydH(|?RVR1xv zjKWy&`)-oB1vVdVAtu;=w()#q@kDY0<*^)O8J;g@{(o%bdc(qRtxg6%cSgi0!&8Df zBG`gV#mz#;{_IlyHyR)64VOy!4=BzCbSbuqU;V4SZY~-0cIf|(EJ_mKGE*%x8hy$Y zgx>xOJ^$~4CH)7G4^9CF&53OE+P{AF$NMR>pt7a^xQysqz#a*<3@JGN9wEXoWZ6*J z|MeqqJ=P?EYj{dI=>Ge&$R7c!<)35z=exN6?h(9^H1X+w5BZP3`YjRw~BUk>vc7zfZx-j66!yqrk42?sQiRgpz;%hCwA5A@eN&Xv7#q4$a zR9?=!S?PL(RIFXzrWf2AhBHhGM%&BD#ygJV5XB6Ifs1s@A7@+H+91IYH`>9^xT1TU zF&3Uklla#m-3r6JQK6V`wUOm-yW2s?H8~BXd{3A8J|lYUefCdAwHSUEQ>v^bUh8*U zqbGfe;G3W*n#(?)vy5!Vg9%@?`%-Fv(yg*QR@CD5vMtUc(@;$Kfg)OI&-~`AVn`Ky z0R&&1oY%ZwZ_l{NX$vZANiZ1P$mDO-R^ra@=uVQwaKl*SMj>Ci8NlXj1%~r`?07_> z0zC_p=B>&z@p5fv5GJYOb@y_-H>+DmQJ(TGc_Ot2IY>kritU{b z&{X|$eW{Kd-GqtHGUP*;_{G?#L>dY`Km5}^);!L)b-{(6_wfH2Nvj)Jt60F; zrc4#iKdd*CP9dzf@eaDQQhgE8!-h;3MtpM5m?6aV(R}POxUqx-m&$iY)L||i7qBW4 zpBd?56cIDNBP&3!rI-^SXG3AbJC{`)A+T!TGsjoF~3%PUXihlB89jpn*&ldW#VQ~?(? za@L5Q$UuU|%yQ;J1^nQ^)@0+K@$}LO3_;T;`Hq1UYfcO_L%~U%#<$1;jw5&@M&s$&XVFi_lB3<2gBT zA1`*`GXy&bA`(CCj2P-xIj^`s*{c-D!KWb5)?q{>_o+FmtAdX@(3l*Zm`|EdDjZLK4)OMI+P!^VYQ@}9uKtC zd0!=eaB-RBkOOWWv>vw8#o+naf{7RBTiq!%uQukorUw!jZK??F3e8_keKR1nKW2^N zkivaitcwGnht{2D9dw^`Kmo@GM|m4b?6bKt%)s45%b

BXv_L+EPbf50<5+_nog(sH0qspf zii&7i44ZRKGBfq7JPt?`k-ugrmL^^U0z)wX>{lukr#NEeB@V|Pjkv(8*qzUd&vx3s zKS{7q22HDegmeXCh$F&G*xN34pv^k0jPE^N%n5Nj9ka8WN#~t-{QbQ7V}>u5Bex~Y zipiMN__2Ii`<<{obUcJP_bt6akT5rb)Sd7=T)sZ|7i0r*l3sg z9vMI4n+N0ia3$eJC6+~9YgszqggIkiHtpzmIF;R=AKT*?;yNFN#E*D7jY7frG6nk- zRQY<@*5s=?%+iO(yjV7Mj4A715-yRij4dH73z;ERSXJ`-un*xCzlLjI;TmPuVZjw{ zt5_#E$ZkYhKZ*ixRAr&aj5?iFPo4p)!{s|u+GLKw(GcCU2lU~*8(f$Q-t@`h595H5 zpO6s;@lPH~t)9(X zkK$Yr@b!#20r57uppH$FBnAVv37Xj-#v+T`VDd`B3_xY{lgI}}rp%~KpC!_n!06MT zfsP;#^lNfwI6=8#tQL#U3$r~TigjQ@_2C@T>Q7CM7pTwNo+^tnMQY|nVHmG&Qb;`; z7G%*Bn^8;^vUo%K7&+IEpF+5EDp>WJ&0YMimk8|bS<7!kSUvr2sPy!fKinVEEVcNc zF@KqwG8jr>&aqhB`k6ptB6TG)4JueY+JH^tDH1qEH-~9Z*I(7X6O62IGYZ=&a;{Pa zfxrqNg0$~gxSG0Jh511@J*V`A=Z=Ij`2dZ^ZXw4Ah$naiIKQQaJLWy?dGYb_ar5x* z9>k(bML*?b4y7=b;2;pN=La`7agUaf#i3CmFyHQs^mQ zoIN`|Q#~TRB;R^^s8&7O+;h6z-)S7-S1r6;&|Mct{IE0X^9@`ln{+0XTKhh#VJ84C z7byVHiS6f`k0oOOR)Z42?91kJ&0c4tP6p_4g3HUJ?(GPgir{nWq#%K1Wn~|0r(Xl) zoX#88Bk^bBdZj9HJ7R8kVpUqyz>(q`G|Vl98b~?y{YQUS*<%62cF{+7g^w~=VlR86 z&0-cyE6`Gx0Yl2~<7Y>9YQm85E8Ymgwr3_c&MCl2ovVhymgMhz_c{qL;e(6iobzL9!OjP>`FKK&uW!K=sg?C zvx5+3QS@%~kz$JRUWA7$X02Z~l-#36)V9PG)&cTPB0f zN?M`cY1Y0YaY_4|ZUmrDS|!Pgt)HmcZ!ibW6IR3@dr*3WJ+2lQ(`4POvw-|VyJIf$8LX5Ycto$5Rx%Bhl z>kgm=6>fsBN82;N*JcVJ2+QU1^)7lJ$j$R(8dkfDmtnZ+xpwTg34+Ut-P2~He{{Or zVSn<`1FX)T&7hRDo=KUYD4^Y}7y6^g;2&$L6$NtdIspei$y7b5sLWvOG*GVOa`dgD zOGhvYvagKudkmUWD(}SZ*H!YnCmqODLTPk9`C{Zb4$CbFnp|AYdjjmS1f164+>>nx z&?gq~wibl}XDakG39TFIDF>$to6zNv8F5GkZni5_Gv7sQnKYuJT21(h&7VaVs}{dd^zH8>f#7ECc^Hm)C{MdTVR4wUOsc3XN_}cnbM-8Of=eW zC}h5x-%vM%w4m1-L8cteKnq}`O4q}byNm5Kymb{t>$YLtcTPQt znlu%aWr-p;sPfKp0JBczpwQ`4)_Jr5Gf}a)Fu&fKRUXIbf>Z!iCsJ;`^-EWv41ghc zK_4EQs)+dYO9VW%V|0oHn_M|%T?$T8TGLp)zZoL61HwJzX`7p z-}}4-JG3^4OI-OP_r$b}6dul%lJb@WR8;Ky47Jkh=T^Z@O2RuYzAY=iSdndy0hEBV z*V`gZjSkw1;ABR1?DOZXOzHd2Ha!juJ#3FzG$)2DA~B=#Kc7L9WB3m?!SvCI|4{enw+~mAv9{L2Ne`8tf7~ZOjCiPO~WwR&L zUgAmgrELo!w%w1xH`GqBJNn-Q3x@6H)-rgbW5CfjjSICS45igz|>ZI0t5vyGTvqd2SxskXcAL-0efhd+#zym(*jehO^ zB90z~ndf~*lBdQspeKIgbIfogk8NNzk^jBW?xD)P&s;#a6<{In-HHW@#R?;>sy&|+ zm~<-kDGk3ysz5Q==%a0E4!xPbQC&|!`7kXnfn&nBNc|;H;GMQUSL6B`$m4{!YNF-J54QkGjb)+$G-BT9 zK_Rs6qu~d~tHWdRV-WA>RPvd9*Rh76`yRx=&V&5pua(&=f`yWHPuF)?vTltj?jH~6 z6o?H%q++s+C#GH)M;ThM_I$Zod}FuH#^36MjOOqdFfIE!=R@q4+5!if5#{NADT?h* zzR0(BP!mZCyg{Q~H@0B$4y@lJiZXe+`OZnm8}lXPO-2nYApKs{9cu#sUq$#Txng!u$mh7`nI!7zS(m@_0dITk5u^W^H)yXm#l~%3QA-dhNzp~H zGGjjI0Pp&%Odz4>*fh|F+Db_Iip}X66x!`N*p_{^{tVuoucy%{e?S357h$E^#x3AL z|2ka&I_U#4EUVrpte1P^gaFUb75+{aOx3W9ArAjJ0`Cr)%KE(OT500f2GrC}Duw>eDUydfeiHd%0 zxN;ai@5k+N|49W?sBb~OmHeZuIYc|2UWK~v*hk8_!vXB$kHVnOyPA$*M<%zKYwx@P1YY|Bd#bM$ zOE(yF=vb-!`jgt}hPmt)0|Du8+dCFsz-;BWwF=^quF2Ne!P^ zL0tl5V!^d2YK^i<=rdK^V3Qgc>9oBF;p38;MA3G00oeI8t^x+maU*IyVGgr<a*PBe+K_e#z$;3oO zpL!U13p4qO07EJ-<#D*i?@c|Ct1{ByUTj*Vui#@eTGD&{PHjbO#zm;D(gtC0*GA&6 z<1!Md$h6W8?vIa&9hH+!d|!&)uc9D1`YBxtir>$v6jCZzgqg2KjLr7*M58oP*?q#a zY*Z_)&`2bCg%(6f@&oDj^lE)WT3~svi0#!Q!-HPVuM65e#|s7 z=^G3~-)VfavO0fnxsl5{85fp>e8%7fl$KrA8QE4gn`eeeo}sJ)4v1OM4?4{P9-yVX zf18B$S9OHWfRNLl*d`H&aN;ZU1w+8>01@ko2Y53B=Bzxzxr&8oUf*VCdi)(6b z7}xD_c`-uZc)u-xc9xj|go}9=n=bZ?=*C~To^^&_C@4hFXl=o&Zb#8|G6iSTF*Dll zPk-|lT-Nz&(<*5>3m~~_X}(*Jk3kU`b!X@28auIOSpo>rRxjQIw9~IzRn@l+TCic5 zKQxV2o(u&aE$>y-OH{f&%G&0?Xv{cGP$Rk!ZozsM7Su+hdz4rd5d{ZuSX62qZLW_e z?EGf40Rs;8vjw^r$NJSXlGwp2io@SP)v@ikhHwLo7q2vvNDUp4W(NA{E;dKQQOo!D zymvH_6nkD~B^crVHo!4Wa|gPT{OT|hu=(-f+OqkCMlE3u1h@~$8xB)Vs1MIpOV?`%<}*V!)zdwKia*x#i#2h7ui+?6_r zDJP4)eQYH{CAfx&-Sdealagx1kMx0dPHOn&k@>s*W}n%vMkKW^_(B@Q&#(c6F1(_G z_R5|1I3wwIRPnFnL;Hp~jdzO&*UiHcHNN4Ll05;~5Aq0@&?yBK1=o)fZtcw&|rM=XcpNrRfcEn#ei%BOoz{Afgf&Dsb z47JohUt%ICd4``W?9U}an-hl%0e#VauV_O@SdL{dY0z!#kbC?k(sbP(&8x%L)y=89 zA&S_Ixi2p}eKx3TNob#1@Hs~#e+3vOj91BaRbM2AIrp?3p_7br6gyeh`=ZK{(gXzs-3u3H+dzBt zNbd?%#efXXy%e$u;L%T?8SU6k!DsKy5N z($tbfV2_md5C#W4dVGXC!?|AsM$dGuBx5o&>fM|UAWUD|#h)b%Fv3Uw;mX;d!Dryj z4-@%kg_7JFsC!R91TuM@pldG2Aj$t-d<^)7al6~G3CnsOyZjI{FlbFG;1irenl+Hl zST@a4hc}kh`c*G&Y`i@W3{b|iN5U!vwg=1gK`P>$QXv{HXX`bG?E&P>md+?u!#WM5F_Ce+`_ zQbMsvf>AeD`f}_@Vr+A>5-uB-dxx#4UBOZj@YCc9FnBr_y^wrIK6c+=<0No#dK#Co zuyN#tc8TfeAru4*?Ft{My6zf)q+RgaLNW}-Z;U1%FzQJLQ;;Di?7ZCJZ#EeFfV_T{ z*;oVK0v1ns!W|iw%XPSigbF z<{#(zwioW!{dZ|Kln2h#Z+p*46Ta-jmqvGi=xqKnqY?Glb7iLHyNx*^t#+dnJ$M^j z7I0{D#H`VLY5Bra+QlomtZHn=8QUIU3B+cy!Z4!UrqMcRwEZ~UqVNv(VEp^`aEMHEMw5HYfIvjlXS^Xw5~%EACeB-jJH~jhECX}4Pu^sbRG^iXSw4(Fvo{|N4pE&Tojh`&3HL&P4U<_+GOgfSP@~`&F>H7u#Z(0vT zzKcH6;Ma=rFD*t5f85f)VtY<`p;c?eWm&hhH}sv-u#-i~`)tPHPko_zbl*)Cg_@JuSoQR_*tMu?&qxJ)3x!BcJY#8w zxK5V(3(96G#uA`yCgshaJUqP88$rt3B~Pii=S<2Of=Wmd+L%nwTTI88DQPUtDek@c z74jR8-8gQ?LZ9uMzb?DVugpG?-4_WchC`4~oD4#e(j!ibjb zVlFDsvLc987F%@ke)=+UfDr=t&H*;61(Pm-Cn&$~y&fZddGx;MxeMR{kkNWcwy8bD z@XJq=%T1tF^*ZVD^m58~X~_Wo++@Yw>w(VALa*(kzTZ#YRjJS3Nwj!)TubVarW)T` zrVU# zp~Y2)+%Y0nZwQlHrDhprIL=^Es{Vb_$ZFcD4{h=(zu}YP`0MfV@;9IY$3gZNI~11;g$1|_qKhF8*b2`>KF)l47fZ8I z1IM!~_Z>g{wcPym*%e^*uUY+mUJ|%F;cPgMx_>U-bYmk3{O8Do<$n8w%6V;!I8al3 zH~nuf$G=~T`@xJDeE^sc{AOM%XZ!x>68X=F{tqKHo(<&$fHtRZ0G#UX zO)>kZ8)mV7IWb_v^{n7`iOvrVgcY%Gdkkq>E=pjQ6= za=)EYzhD0SBj9fdm~tM>5_xc>u$!^x%Vli_Vp=*_g#6B9)`uSiQidA`a1<&=0eR6J zjrtmZ2}o?Q4+U6qZKv^K1+zLk0|cO|((r{a38cI1@rOGq1;$r_|6IX;_5=6(Qhs93 zK2v@{C*r=%5x#ANh-wu)dg+NaFm@q0lNzFqrcV{)zAboDrQXn@czK7e~r0N zbQtfKf38kVso#f1h8ce`#ti^%fyiA-O|=@ViMCk`(73qV)GGCK#H8c+5hwB$0`6qF z-UD~s>FKF>UW@wa()F-$j>~x^ARXv(RX{PzePiWn707MkfO9k$~rh6>F|i z?Ug25cu)@Nx?$!e1UITf5}tMh;k}T8D5ztZgMQryhd&O+x7EaWep22_!=Je8V z0l`?7OZFgB+C&ZzM`%^sBmB-r0?MAR0V=oq-$hX0l_m8ZE#9aL&H(;6U?Aeu*&IvI ze<)S2XEHS$k{ISPS&6^e=Vig{jX_vrAZ18mELpMo$YfAW;Dmz!Os3)i^!9K66L0%N z)&9xL{dI>s{I+J*UHwIG{PJ+Y4G!{PMsd927fP#`y>WO~25R2QCu0d026W_tb|CJ~ z57Xg&JP|Wp{j+$vo0y6$74{$0L2}KgmLv1gRw}e15fMZh6^qen7a9ncnCmWpy1UwS z!=4|o#)%;n>Wf#`b=)0G08Ru?jYaREAFCyo%*o*kn`+MpUrPWEkM^_33V>=LHB8g)%u8aWqc6eXIP(Lf+rN^SLwzi+6B+i;1pLoL4f7iiJY;_3%#99Y$uYig-4oW--2mkn<@<Rc+7%sAy<1RyeZ6ux z2T|JxmwHbZ-?dx|tRa<;5q+PAHSM*Q`3>8_(Fg^%KZ z9QU@LRUQ)fS@k*)WoK=B0;LoIn-4MDxf+DCbslT`9Txqb>P`il*EQG=c5>mkLlJ?6qsd{uMreDP;K;5(DdiwY@tKOFhg~7X>G2$&Sq{mssZ#tQrdi zR$9V;%?!W8VQaWl1Auw_K>Z%EzX=})5wJB@=Il5>?*{I|Gr(AAt`?H|j#Sy%4(y|~ z2y~9cLIqTKfd8vjqVdT79y1=hnBybst7;YcBU5-VV5`~l!&n(4Y*_C6HZVMXye-)x z91M6;rfHy?>BWm9uC8xa^b~4_3zy}!0GFvci-(Tm9i-KlXyf(RE7NOY{=%M2ZXbgk zv#yD#Cf6s_P1tubShSx*RJQGw8t|}+m?x0L`t(xdSN6BV5F=UHpD%I-O3B&8?~yam=7CH$}hET1PhIXBrg;;FX)- zeJ3o)d9k`?QVy5MDqW;9^zMrL03yN2DfaebdD6K8b}oC5!)%8Tg&*(`=}jma`&B36 z;%qgGj_JC!aHO;5x(>d$ec0qtQyv-tRNgt(eXNf6OE~<10}HF((ubR4*n+mQPk`&x z9@-7@!$Y0x!_J7rfRqDu$8U>fnfT9h%{yowOGJ`_##G5$!``Se*;atmn1fjpSgz~w zu&?RjC<3Xy%ZZad#zxDaH9)Gly0=%*bG5gEd3Uk>{aMUU#yMaFV3jg>+z8mQY8!{7 zy2d_sPfsuJ?*PVMzYA$vwzlYFgBa%u;)6DxfaP86%EqK=;u|09Z(6NCo&N@`yT!>J z`4N7_LjMi9Lk$Z%)5FH`kQrf42!5~Kg6(FfLu}&;u&UL3pF{VghMY_N0G^UaedC7n z`nd&e7+6083tk4iT_3z17M&1?yng8am38h>No9K+2PCqzC5Tg+sfhT@%CI}W4TVgU z@!4{;tfcT!PN~@y6(cZG$TwQ4-E8h?*^G=PncA%}OLAd4YC7W@d-+JyRyHbXNn?5U z5fH0?SZ8teIeYK3U%&6(`?G&Qe{O#NFu$MqB+w`%?wJ70jz^pidhqj8hfnR9`&$>q zFxlNTHo3h%Z5CyWd*uGugr2&G^O3DVe8hni!z>2-{h z4*+&M>L6eNo2R&9xM&`@#HSKJ{0Be6Xs6s~_RGUO_BBZg)9d3qpC@K`?4xj~N z`u;~ta02=q5gp?6_SDBBw zI}!2!08`Y7@!)iQ*5DQGpX*HnIx9ZaDLVssXd}GDEoD}6Wa0%A z$^1LfDT&5f?AWI6r-3k$>_K!m051WVK*Y&f42Y;Ndl6qw?XsSwm_c26`NcMoQqrt| zY7|+5e$k&G)p+_^b^+2o8%AD!dRYd63!pByJ7(_jZXLYXCHgW1kf+THigly3R1hQk z1G8CA{WR&MW%I)F@B;nMIS<&35WOXF#2xqI$iKispzp+$?uMWXwq-WZ>R)3XON<8) z^Dizzm`6@7{0qDRUkbajTUA!*ISBKZ8!$~&8L(=kMqMV{cr!G8ZJT6z6N!wn)u3Bz z%(lxOC@GpLb|mf@VV4(0^-k!+!I6hvm;oS1{#XCH9Sk*a16@r`x05=sM@kMu1(d9V z&{j+bOtooJialk8qxm{3om234Ad5fun2=oQ+4J=2>I{znMM(t~!wUVN$mw{T_==Q6 zU#piLEg%oCZ54#5%aVk6oX?T6@fuB=V<-1`r>3ObZ#^_qojm>t1>d`(Nd6!!y~Lz6 z6};j@ce1|1F$0(!ae!X~^8(F^`>#%Uk1=fsZmw?h=oV|CDZFBZMy5C*@c#(3=l?4g zIufdNax#agxKKnLK{c@PPa-Mrum)56*RUEqds?vd->%n3ofl4~Q<=(2%l{i$d~e(z zoYAx8pA;*DDRScGgn-Pg%o8qY+!Bj8>t zcMce@=Dt>t@1Z%8z;{@?W?mmSR8cHsg1UPc5$GI?pAk%KD5ddyOo5TATl@AH8Ef)! zISja<&=E^v4us`55;~{OPAt47-=T(?qUWaYPF8Uw^)h`_%QbXbZH#t^>&XWCN;7-xz2qOZc9 zRl_LY!wNkS+l8D9(?z1n z$ff{Y&0C&9!>o55VUy-Z;$qDjlnPMs0UmQW(j@U%o;1~Dvx{znE4gg0w(NfLVH^0X zOR5+LgE8oSKZa|;+yh~ToizanhWoZ)rr9iadLS+Wk)EdJA=M4kBdB5$x zaqYzQB0URm=wcvpev|SsO@9QwT|l!GUzR2h88G7$+U!TdWVDo|cy^2``%tc6lqA%0 zK>Bxa#oq{JobP?C<-pu?l!(a-jkVx7tkB-iY6={YL60%;V>m%+&$`ZNzd2GO`{7Xm zX4hqiaMLu0rz+gQB(H|IXqq_n$a2Sc;}dV+LfeIguHJFcb$ZAR-oy&nVu0W3tbTQ~ zr6o>WD*XX}N1$yf{*Uw|=CWPNHP?MT`BK3N24!Y!M%@#FQT4&T+_X607GY|`*6K#) zqA6HF%eN{!6XfR{Gs>{tx9_h|!tHDR%Ibh$2e$xDM{r(fbQq06+-Y7R>@5;m5HCrO z<6BN0)fA$rSrvj4Bb98!Gx@%IOf^&XA!Z}f1YcT z_)J|*;Mo0Wx_SvJgup_87V>R1e&Xwu&=~dwk129XMI5_OW=9HkP@fQTgdxj*j z>9B=kVkO1Mi#CV|xAhPFO`C6PK`fC*;mZYz)D4-EGqOB;XnevW&>sT&V|unf5!yM) zoODycjEZDr=%6hCa<`DbQdTG*yI_gIAUxiUrmr?IWkLk7=qG)#yw-%;kzLqV!X14fEW;lyua Qfjcl5b(bHd+A|{k|5H9q_y7O^ literal 0 HcmV?d00001 diff --git a/docs/images/dashboard.jpg b/docs/images/dashboard.jpg new file mode 100644 index 0000000000000000000000000000000000000000..48bd1e6d9fff08665cfc3067becb9c8c70ad6b82 GIT binary patch literal 63734 zcmeFZ2Ut^0wn|Xi@?Kg0GT*5LzIig(AHqfYMZ&9U&AA7)odoS_%OP z5SoBW4Wk z`1>>9l!30kE`W^<0AOSN0KX^L2K9Ax98AqjboC7(ercXub#a!md@P*L z{crfCecb79_|<*!JYmM?tI&CpFf7hbJZIixPPtv zw68Izs}I7Ql^E+A%xdYGuSHO9IKEMay43KAGN`T7%MS$w>5r7WhC_DQR_QOYy9656A*ijBnJ}%DV z$2kRgcu(?)3Y|MEDkLIuUP4~#yqK)Gh{z>1Y1zw)Dk>`Hq|~*pC|#9TR#Dm)!glP~ zG0x+hXSle|C|wY_p!A)M{N^B=7~lXm+d*!&-wmt=umbS#ejxrm9bh|n z=rH?{qbw@y6oBpEfrCfb4(&I9g&YR39XNQ1`!J6gJ1^e_MHu$Fn4{C3BLYgw=8yQr zZ@?3ZyHyc>{_$BVAd6QtiT9@i9v_ud(;ev12mkE__eHWFX3^xi z*#HOF*bX1v*XVB?Y}^M9^6+1PJvww)QQ7Ry`|nC(*By(yRkFVP9tLo-#2ny0$PKs# z*w~NK|BmVKtmoYN<3zMisB}kwzh1XP>$w%Rv(M$ql2sCllXJO-tc{lJNF5y}yIl+G zF=dyVeNAHTHD3t|kr%Q_S@be;c)F4~)d$S2!p9)b<$Z9^f3y@3-#!U;RwqktCBH(N z8>s^3H`8L{CECOlI@PMDQFPSFRKA9DaT8n!ONDu%Z{X!SUP0+KX#rP0+Ju z-ZR5u*=NuGY#jpmMCM?`tbTsVEzlEs_uWR;{TMaz=o~p2XXuJMCq%7GpN;-Jt2s<2 zULv`WNVsAhUPNa|VoRS1LIqwOMhh%aI-~mJM8FFqh@t^Pg!o@&lz0>yk&8wIvkGyS zfwGELEyCzHY)4J!wWO28gxR9wI7E%1w5iyS;tV zs1W0_3b1oz^R%wQAOZttTO_FDEroJZ18*=v>Ngou&MVOFxg#vxc2co zdI{$+aM3vaSx-SlKy}K`W}HH2emKTgKD{%864dPB`NpV<_=Go6o%rKY%E94Sh%;gt zVw{fENLg$SYIawpxa2j^9}UK8YaX|gl%Er7S&p4d9~d@4;rb^B<(3DIeC+mC9dQ;` zE)}*u(U&U$r^%vU-%M>SUv+?(eB9}D9pu<5Fn7EhJUld}W!T5^t0pC>QcCEtWDkIo z_`e(MTtQbi^hyR32B%P|P+%DhO{}!cYU*-QsL!*WBQ1RK);iD$@oELcYWfNwt3@1_!H=cdlkX(aW=*LuEA`5UZ;OLiS zOD5iy=_PVFxo_8RF1~MlUwS1W6^a!16o{i)Hj2oh%CRFLoW;6BNr!A=r9XSq4;y@e zmP|a>sqk43=z=r8m=Z#+^nDP|VQI=n9=>Y6D$iyeDe+Qw*4YxYCP)+8vJ=8kI=j_h zdS9P(*fAJB`sS_CIWFD>PxpKmKYgeyr@t#w-c5Zder!45irbJ=DYK%6UGe7d=B$Mj zXz=REaJLo=F$IC}Nt|i#Z_llMcYnHXDNjvD^svRb8>Iot*Ck3i>>v_xQ5{He2Vs#o zCk0T4>*g7&h)-XAcEv3t_AsB{kgcTBY2u^v>0kZjmwXbT0SJh&60+}b=atO4Bd2_L zUm$2e3w?-|rgV0Q#rzOAxw7)7gJ&=@tJAq6PD@*GHGnR>u&+_Q#1!s z5?fJYavc^sRY#kkA%9$@)%mJEar@dpQWOg2z!N<^XYvA7e0B5IHf1Lfh`^+>Lc13} zKYrZ~E-j46z!&2s?KNn24lSBI3Y-SVr^%a8-y* zc(xlf7}r}-bypo&V3(@%Wa5;TBAWeD>V4q+d8^Yvg{y@QcgaG(0olneol(r5R*Phj z73GJR!n+3BV%9xkl?Yg&y~oo+j&G3nS;J8-2EiIZ?Z3oGBo{MAv#Ui}aPF=`*Gshl zPk0!`*XL_Ch(iPCOeEDc{W9v(G=SSN^No!yo{X=Gc zdg?_Et}sU0GLbY`%q!meewy3+W49B{@lBU9=Oab^`Kdg7?hD4NVxYKBpo=PjI=lm3VX(Q01J3EKVNE5ot?q_ARYDjBzFfya#ZwN9*&vJ8 zAoLBE9;)x=ki$>Q2}q^4@|gtq(f1hHsV-Hai`TpC*xoss4^WLRo-~KapmRKl zM^mngtkB_pm2Kh(G!p_9LsDN=pVf?l+?ez1!ATPxpGmq<{aWRQlW2q?jddi((&dEK zh#iazZQ98vE#hBO_JWqT41(XpsjngY9DZuZp3NhaAIT+?ot^O>J$i~nVr-p?aX9@= zvfzig(X-C0PjuraLaGeSPbHkCpoAlO-kZ&)=)11x1|h)Zfq~oSBa7z-j3;JZdY||W z;BKPV8Z25A9(yCAo&GC)BC8v8c9yxEAY<{dIzd5mWoqyvX(T<1ki-!P4k#^EHZyOP z1h4jJ(Qd99wiG$-6^#Xvg3QJVM~jCB2gC@3+|2+9SLo|2{Ev!1Cr8LLhKZD{f>iN} zq^ZChxGsg?kBr0>xLxdXdeP8-#pCC7zV?WlJ$OBJUsSb5h%{;yzAAU3#db0!&MCnH z-eWlLWi8TL&KY{LuB5vo99=Zk0h1-(YL2!vbF}hyi8e8}sDS39i%IzRO}*eV@DwTS z^tzeC-x?%?g=rFsK;M<5~;!V5spoe-b;r7-x8}#lIVaycacNlNSzJa&yj6BcPP|kNc?t=X z!PhodBKQ`%eUczf7jG|I`A2f-1jR54ANO@H`0l-spN zeqwHP#qfyddaFbyND&&nCc>lcj z%bk9>v!WSPB{o5v=>)MXDsRj5HIW>G2Ded2V*H5gAz(<>mP~{w9vt zz8SA#%b+7|W>CeynBkyD^iYe0rv>PusE)BJl?335zs<4-CT+3i*p>D>a$QfJju;g; zth@6V1mBaM^0pK1k2=dl1RAJw3z4I{VZnh@eKXTruqKJ@(7so%K32cZg^bp00!727 zx&@ZOL+rbFi+z8sr%1VN0+$#J-%0@M(AML zthH2dN{R~yBJ!IuW_&x?PXRPD5Px_7 z$;b9(&EaMl6!RJR$FK#BP<|{?B+hSHEFn@?_Jw_=Jkueb;Lx{@2c>v?v7U^x#zm$7 zP?NExPZN`nQqwufgn@S|$+n$AU0>Tn<1A<%uvo<8$Oq=sND6=T5wU6$o8*yOPS-)o zGof6%*Tr1nCJhd*7IPik>US^C-`TMV73X$Icn`G{(QGQYBYw7_rV?27(Io2hikxKi zYtID=S6SA-+pL#Vl(*r-e4HrF87Ax_IN)k%47eN03j^ybU!TdnDpRSM+Me&fk!8ntZmt~Nos;f0rHAbwg445yMU1A4CS$@9KHkWS1C3p2` zbJ^_DS7y5&u&jj+b%XjLN4m_AZ;!}WzL-#iBF&F`nqe@xrQf2K{Mw{0k}Cz)P0Ctj z%Zp#g4>HX=tMNzuKG2kMTQDj1nCepFaFU}LlEw}?RMK(%P?;RcErh#MR}#w~A}G^^ zvD1vsaGiEeN zZwTKv1NUnZa&0Ivkoto)1#LIF**NCJL1)#vGmSr5FE4sZOV10WyBUD<5~T&*kqK$! zAwl0^2&69SdFi>EXA=2Kzvbr!@A6r~- ze==Qq(w`d<&KMf-RTxI@L^enFWUr&{qI;Pllx=&9Exr&(D6ka@0YY08^xX=wmt5ebv>{;?F^Io2m zMzRGQNnQd!$fM8jzbn#l_h|CYTgG=SFYl`>C(S>OZO~z_&$JL%m6pwW)y<4*tFdrN#-xnwEn;k0s zYoYk&WTK?aS++N{A>>(Gl5b1H^Vxiki06lM461N$yryN`3NozWB)ay*&hhw+FxaZe z(>~L$0dW+PuG1PPF~w{Eg(nlECWo9f*>oZ;7MGASDH8AS{qM#t!f3lit62k&%A`%w ze{^Q=*awf=aCpBUx-k9Z=dENE43iw_)Ni{c=)s+{=z}hA+JRXubEMa3UyIc$DNW&b z)(i%K9sa*L{bKXECr-+bKZbvS(WRVau0f!{Gv7WCT|nKsJk*pB;N4phK6Ed0$-$T#}SuQT%kVMvH^7^%P@$PFw;UnIrfZBJ z3N+EL9^}MOnTEMu22*~Yl!o}n5IpM;hdbf2N^ytttv>5gH4(-B4z8!kg0~(zO`Hk5 zD5C@758%qcyG_Nb1^5a6aYBuEOYJ@IP30w`=g7?jC|`p1a&Suwa>!p{()W#b`UiTr zy7jYeTKdHxPkw%cZS5KYq57~~+$08F%k{!YWDv*e7OzSX+FACRvZo@va=Swm#2Vci z(NkfDynT+QRX5KJ4!K(7S?s=oLuiDMj$e{-;dM@`zPyEDu5PFK5^99vg4{u;6^w5Z zlc1-)=sIi#qyi$bK6w-6coS%;oDkelhJ^`yqNQ!EsNJ$+BGCOOkt!>FUClD{=e@6z z7q(%9G2u<%PW#10`Hu3FfhT9_vI7k(E>F6N@;=ZdQ!gUmV|G4t+Rr-bH~)ZRewT}F#hwe- zPrh#O>(o1+2E z`B{0niTdZG3PL0Mg7XXEw0k<|0g9CGTefBx8;wC>&{dLq(=z6{a`d7ozitQObA_?N zQA%1GJkE=;K^xgK10j&Pfy&|! zZ!nxZ&n~?gcI)jG$F&M0(3qOwl-E|>{pg5;4$*3Sq5 zxmN=pcUks1@xvF%)UeR9V#XxAr6|Y+YR@1A(#WC@%UjY%Bx#L4%`@W2`B<1 zkd?tj%9l@Z1~Y)@8!0Svs#@V-h{CxxydZ!RUO&fECd0#?tK&SDlR+GBIRC-J6CofJ zRBQ%{lBKL8e0_m#4?)W4k0(tBQzV5E#SJgZxR=2#=|XgSkBk++#dUs_?lwtrM0d7L zMh^j~mfN$A4CzQd=%9Thf_W!oUiVrw%3AB&Qg*P|PRx4QnLd$i z7b3tN{Vn@J6RjF@Jnqn=(x_^bs1V3@s>GH^j_S;pP`$;;k#_nd;)JwdE{88NRiBgKyYr=wp7~%CJg42 zB(L+-^m_G}IViQF|Kjit{wzgt=IP%NyVs=?i)FUt?b-P{FE*Y8N=%(tZOmv)~_a zxTPDu=+nvUsluaG)xdLOrgVR-nO>kDP_Ach0;I`XQg$y4#5rhds|D&2D6?g{ZvBKn zW)?bA>JAs@1u<@!pw29+ip1pFsM6b9o+0?HeG;pr-VVXK-#_ne)Hi5Hg?nlhGPDnJMlDXm;h&ymU_@Zgo=tSFV|!qJ&+Vd)KY@@HPJGRQ+%mlESC_;Ov!x1PO)bdivA z5A|Zhlih@QEZwcul91v|*+Sj%p_=u|w}&eMZ0xf40!0uwe)OX#tsAT{By%<`ep&*l z_Vh!m6+=ar(HPrOpF z5)bt=m(0BQ10g~rF4)Wx%Z=?~BM-83o_&lvKiUgQed?!6q_n9GZ+;qgF*mE%Gai&4 zjEN$XV@;scfn6r_9b0Y_-Q>`)AR(163%%07baqJTu%$35uF(8SqppKS2--bSC*y=K zKPpRl>4{)H8JLsTn6Qie&5W-CnU1PH@o|KB5sWZ z9s&`w@SJjs_kenN!D<#HMdR+C_Z`M<3H*8>070{@3v;BWf@{>^NOO`hTcafr7q zwDh>U1?Mm2)Wp*wr(`VFZQ*{HM@25y}b9o0pFbWvKML@5BdhL?Jhp( z;u}m0|3AR2ik^0 z0sCN|`cGI`m$D8~c(@N9xh~cM*jV6j673gY9|+jDeFp4<{|w6}OBw(mr5#EBYo$*9 zee1_xH1*P3ES#hlhTCMz6y}c6CGcEI=enpC)z>vKD0!8mFRn1xIdQ3GI2JKA$S8l< z=X`JCb9=NL=bACr@|9PAHcShFKu2V4Klo$aaRH(avy}8!Vk)V+qT*GM3W({2Eh(qI zxC)V4(b6(pwWsA%`E%mZ;<=ch_64y#)S3z7Q+#3&twf}Lf>M-Ul*d{)cyiTab|Wss zK^GY85HDHzlTYLGZB4tf!7}#Fvdb0OFZf+~2AICWDWyNk%4zv`HHS<(!Z^vhBx=1X z*$bXcEXel}Vc_EUOdS2?-GN1wye~9`Y&SpR|C_>J=1HkE4c zvlk@IS(8MPbCjF2B*Z)I*D?l^RP94WI*`1efq%@bn+p~UizYp;C8T?LUGy5d4-6)# z^A;CmlZKDddpP9{?SaGhK3kJ}pP`h*R-Ej#Eb%)pO&8yHenlFOs&N~tW;`a-splso z&_^@Lar2Q=b%79C$g3X5>N^s`QW2KS?mmQuhp0+ViHuEuc2d<(Wbk>C=%9M;h^^M8 z1ur$f-+;MP-tNR1pguY!+>6eq>CAp1MqlSJ7c@KWu6e75E8Hd4*kiJ$TODW%Pn?CfQ#OgUc2Tz^&)aH6Cp_WG zIBQxbgXLRZsC_Z}dCSyLO#N&!ZxO84(hhkX0h^_ai43SsKfl_cY@&7~(C&@mUQ&_n z1m9rTag>9;enTa~)i2m@r;_vnx_aMYy$M%=6Xm2aYcW@*5r|MK94c9(o`3DW>g#-= z6YJ4V9!?)F&sPmM$vGD%nVJ|R?c$tdU?I6o-yzd?!_=2Pe2cpDh$?(ud`4N{w>U}L zE;~!0+|~+|izwalr09u@^n$ADbDmr(>Ibl@G0zp0H0QUL3byFS!lE!w){z*RvS9gT7H{ z}2w+L?w*;{cc8uShsc#qy)XrdvUS>_@?R? z%40eJajH_Jxy-xcWXbK|W~Vc9{LVce@d7pG+sexLW4|L5o6e4BL=yEaTK&zIvR}J`z%=#6(34}er`6U{YPfm( z_Z~X<`_&+a3^=ztePvQpxc#chA+Ln~th_JaKawcOlqKK5O9l!xzU`cSU@vQ)7Pttz zT4~^`Z9D6TLd_F`CWp9hHU!GA*%#Jq?&&(o+5J)vPJfg4VIKLMspLsHCt{q>XuFeA z2`tujFX4o(^_Nt z&GiMW?$UGuELocnKIW-PmaSf~sf&!Wr4kI77P-4Pq*&;(f#^~73VQkdXGS&^Mo#K4 zP{O=^)%Q~g*;sF%g<0nkUuntLE-$;`ZB|>kIR>h(mcHD|3Yx=178~;Ii3@Bk3O!sr z<~L_xa2l!Xs*I-tt|%Zf$=-_&2Cu1Ixp)m4=U7a*jA^Za(SBTA&;p zTKMC>|8!ae1KG{itm)--Id6&g&egd1$10?JUCDB6O>LWb)eOY5`}2c2VF_=4NQiJff?E`NF!nnPb9<{d8`6{#|)#i>(l0W@H(}B0p za&?N(vA)@fb1u?|a-%#-LB63-mAh%PNEu2BKIk>-d2i@35+h*J|&4J ztK-KiKFp@3w~7yapyexTYQfHY!jCqZK z!z*Z{FVORQRf_bRlp#B)wTX$=RdPm~i$8jS@Uu!TVH`a|SJXUX3rY?%pNz5^Vv?C- z>a8I_`}wGOB2dqe>qMUuf=?yjjv4D+mbXhldsq=pV4n3Hg4(LS;3a?PJ^E$z)2^{N zmr6?fhI7$__!UX6?^owvc;4}f@&4ZDRN9A=D+th!fWIr~spYIWNZ`lKrpnRX@SdN-N}8`df>L>iADmwo zJo^nGO$>1^!nchcl5k_-YW_hl$~vxx3ip$jC(SBwWOY zX;jL0CF5-AnYX6B0~UG)SPfi}Rksb967qBU(}tu?_4(T+OY%~Wz4;ORaKxFkfK}NH zz3alrB>c2r?zWVU!l|tr^&S~fBE0b8m#TSpo2W*8K2X*Z>?hKncG^wP{=~IYTi3I* zGRlOnTf&UMSPcR}0)i42hjlnSkd(jXdui@9H87A1A+b7A*Qbx}@#Tjl(nj!v{0ey7 zo2M~)#W3^9npj=7(%a6hrcEEVf-&yduFsYs7Gv_!0h2_ck7d?X_5=Q4Q&^p=HpJ-s zKkFW0pIpNsH*$O4x##OZO7DsK$jOVsuG?`*LipNb>{O1+uwv1sWqM=Thkz^c@&`M> zrBXDrG7sOW$ZMGxd|Mg_s-SX~@y)Kw*xjJ;l=V-?Sg!aX$>jWs4+*m?l=<~@PfB=m z;(=awZ%h%h^vFQ!&tg~$)z6_SzvA0ZsQ|%*s?%Nu{uMzTeU8=F&t4F?@b>wmxOk7L z7s{WiYb|H$B&W+K%N5t3xZMeL5XmjfZzMV8gNrQ`YXicjt zqP-ANwKlgmqm!cfx;?$iHb&(130v{Q+4cQ)^S$7~pHpr4ar&wsV&UD zS^CcCjcTsO*@+0cuGSAD@SI4B*=Sn4)QU#TpQ=17et&ItnPBLqF)$G@-{izqSk+_i z4wPGw`5Bfm8p8;~kbHWS`H2nEqu440$Tj{kj-abhqJC_An+GJZ>woO#v zWR#2QW)0{p)W-F@r!qxtk;fXo67XK|93r}OyDvm&M!nEir6fVQI$!&_mO?j9 zhqVXLI{J@QOq8d{JjQ6Ml04(&21Zr>oDY`&Q<)p1uYPVUmiBmC96RWo`Lv#tjPs|! ziu+`=mw3~eFoflpi|UC%MhZQT(;j^zQ-WVcQve8$yDLV>TUrqt>X^w}aG4Hb`J1KI z;Z}onZL6(&zX3nw9ri2~CiDA-5~|KA?5g+l8LcMBOgy$0{?NDt!jQU+=JpJyKL12( z6g_O1li^SKbw5QOI-+VRHvAia9YlnQLtzn?W|9X~4KQ<3F57KYSI&vuoDKGe_(@&M zly?_>XAwSk?fbyii324Y`o94YbzRfRQLaO+<@bCpsI*1jiihhC&-C}ufoajg>8P?c zoWGRBqNAmELvqYuXm5AkdJ=DZ4P#*FTuQqwF>wl=et%6p?11(jzpf07 zE?4P+C>cF#nR{S#P2f=nk)~)*ZEwD4ADY^FUud|7=xre!+jAT9sVA`2B5vH_?vlFS z@aWdsq~|@3&M~Py*8axp|Eev=DL9PLS(*@#%$KBZ4;5X-KV=gIesz-SPeXx{>w-=T%GBS=1(mRcdfP7%JdpfKY8`D z%(Z~bU#Q(9Y}VU3zXA0Qv#Pfy7H*A9_bQ*nUsX4&KG~2<&BM8^E45H9i#y7%(V_z2 ziu8OG>Bq3DSx@f(xMQV#4_Mhdl60|5HZn44p~KksVZmqxcd@`jUG|TI8l}qi!*4vTIWKeO~p;FK>{r#nR)8 zP0{16&O-4O^jCk5u^S%4U5A&WJNW8qUm!!%xGLY8xDcAwV+SKHu7#<<=eSg0HSHE; zGKGmaoxUkG>6JC;)gmn|%|nyA4ms>Cr`3&U4`8%2Od^+zEWZ?LCe)iv0IW3M$bp=Y+-{Ei}#)_Ip(aR{VlYJp}t{dBu z_*h&+x`3K->*cvQ)O=6jmH8}7$=$T}yV@C@lExnSj{8=LT@4#?n1kw+&)3NEh-*DmCwenW}+X5T=P1vvf=-=V>NgP?Jr!ch! z9;}@KGSB0S3Q zV{WTOYkJK8K&$Z*9s!-=)SAt~B}mClLiWh$oS7TYK`Gp&Jf<(hu=%jxy}agrr{tV7 z1r;KfFK=C^^z8znR?3<%~Ci@fE9EhDS+&j)IK66Y4| zi0fz-7$?@8ygsgMc3-&3%S5tLPgJ{}spWO)LL>6*F>^DVQqQu23!4Zla4Xm~*8EGQW!AAsYLK&786Tcj?hEeEr+6l&c!3f3=9DJ+8g z2q^tM6jAyL6!62muTes{**dlb(`+YOdEXzR`pcrN`B|YvlW}1L?aEh_-;gs(ymg>& z{2R+Lv46@?sknGpUbMvfJ<2P^QdrzdXkE9M+>8=q?UeL2H!7(>kgUGGO}62eK1zFTT|7cluWK+-Vf5$+E->j4CVD?q89^_6Mm#PH}D9` zOy(wntV$$|AHmFac5@IWcbvHg8H?tpAJejVa>34Du@0hVt?|_Pm|)EWL>NE+{WVR| zzl+j+?llVpk}i2ki(9*8QRDK1#*4h{3c(vwenpc`&=M?=LFKF|!H@~n&FA>crg;6tKxvfF{zd5*~(k5eLU0Xf|MYem#| ztb?k4DgOnrQ$zbY6v?-!2i;(4B-2JaVfS)hzm>%&NiO@^zO6pL&vg242!QQRj@+)l z6aJ+I+5e9FKP2*h$wUkKySd{2=UV+Kg2Q&-4*x9iA&SjTn#$L6w3pi(Da9YfYJX7Z zOt~~16DyPvd6SV<`QrP?7xw9E^}bIp0wVs*g_Buh%i2@P2mS@|ao$n=b>r62KlAH< zRyg+X!hfHNas`S&WYDL>Q6$<$k8i$krFTO8R0erai#tJ!)H7*`C)`B}L1u4Rhvfsa zAEd;;1V^>T?sTP^%fNWMNH`t;b6?6oaJ4^(_;l2e?Qgl7;+}Bmh0BNj%+HTD|MLF+ z^xdgH^Cy3@^8)@`R6|;c!MEvqH=R37stwz#%9)yik6`?=uTl8&6G%RIn|SKbjfKy45l3f{(m!O{(X4U-+)OSu{$rD@Q0&^y~yR8 zg1I6ocw+5qRJRYT%_*uCp(%Y)Oxq0_QaJ2>qA*|9?!h&aG`5 zW1VHiIunw2M^NZRtW7PE{eT?frBi~+!++9AvH9Hiattse^~V&0$y)r80v}`DcAmF} zRZZ;YuXQuKTiV?H2OCh_!N5YFh(_G1@AJ^%3Q+-cWe?cmQ`t7&(EJFsAVKJ+xfH)~ z6L_a31*efut@bxDN0^iZn(0o3@!wE2AY1B-LO;{!YQRdrrpOU%ZzjUKTh>O;aqC8` zbbOZA=TYF0#P~(T&K7@sZJca1gJ_K(@@Fk-troWqd`^zONc5IFZh>3#!IF4x6oIXV zlYCZ|OuSgbF1`?Di;94l0)q3jq1>$(n#oi8@>4XRpE0TPb#9>I#Hx}-pP6c&)|w`4 z1!S9s4KpMHFOXs-9(zp125YE^Con`_B}|kMY$-0_Dx6#&q%JQ}W<(QFzL|ydl3vH-30AaTrfP=!Bqq0; zpu4sq_sw;g$RZ>X?tQiKB4)=!FzPu@F8=X5g_xQovJml7ahaR&AFqD)-p2Ino3;*B zVq@2I3DXF;2^=bDrQ1=fswWq^CLds4Yc-dsyF6Tul*7uuO=7~|a+mI^L$?Hzb8|-3 zJG?@)nam+`+%_q}&WgbyNX~&+fJYfoeN!XPC)%DiK9oN@i2@2u6^tw2t=8w>0N0}X zp*CY-MtRat5GL2{n=BFVf*%y5iC9w%bW0K+J*9UT{iQ)#D&Km@oTsU;dh`WEcW0oh zQq)eRBeok3UT-1CSk&-~kow2Aq!|{AYi5QUX5G4P#Bce1h(nnnoC@96>x|!4@pQRjS(`z9tze^p{cO#u)8!qEq9e*@|s{CcJQZD>#8iZh1N!VVt&z7N_PW-LED++paZp4VE{_bWV+b&fl2%yjhNS zRxI1A(k0$P{aZN1NgPIgTSK943k`GO3tUBt`(TL5rQFd zq_)~~b9(x>f@juOQ!7RX2!yJ!-vBYgM$dBJFmQ2iwo3?ah`6~bqHJsDZQ9!_suHhW zYTK%xeKSCJYnpsUtGY^NTDsa|${jjRwdM_NJ$LFdVx4Dv*NXbIM6vUpjIod_nveXJ zL;@Z$+#0s(R|zwevkqSpSvC9(ke}u)9@c|iHN82M`&hSGTxgm=Ag@i={=|}pc05-v zo6#|w(y79B0u!^7e$wbtav&*H~EDR3f0oDYR9gbyN%I?Xt@J;!h>ZOd`(tvM#%y4W4vUd zySQI5KQAaPnt4B%+j<3Fe@Js~$_2J&sm1!}M8{1d2mM1_c!Ol5@Lqk|D)??|n6dBa zyoMja!D@M;mL?zCQo7wAeZ(q}?aD2clNz0QWI1T@8ft|CTZ8e(zJ8fXlnjs$`6;%F z@}4D*r}NC(=)sI)V4Zr0p)+xZ()q21>|+tdFUww0oB%KCkI)R>0XCJk66RDneoMh!4(SLZX6vx@(jXcmpFD05HpHy~-{!nXHjmilkN zgu~_ov2*lKcp=?(fa@RrziLO(9}k+>JKgI5UEM1tQ_?b3z{qg7PR_>fJ8sxA))8fp zsepnT_9Q_D`MVv9e4;@!smxsPZVq9w zg#jigOBozLcZjr@l9qUEr+qBzY;BQvy>{fA&~vMWigetYiMaR`4fpJ9v}cEH&3#hd z7JGz4f#!7(``Tq_wo_+_Od)+|X{T!{XkboEi#{yixQ0{iLXRqi7{!uj+I$h<-vF0Y zEj<_Qm+SJG?8N%-9KjLSBYLWG+egVV_;JbuOX)UEL}P})B0bvqjVL)xvzW||kke2X zYoEDj+g}GUZn>vBjxIb1EC1!zf~E_L@KUf|Q_g9N<*U;al+?A3lU8|KuzH!=xCJ`x z%EbtvyM_b?C7v#Z1WHzo*=GI*oSF)MdYJ<=bcd^~5$0fNC!TsdEdx{AoA7foQ_T6^ zXaRl1Vb%9(Umxu2m1zZ~TggVsXXcH1$TBvGh-!i}O6xLukz0Naa6+HKCLm_*&cqk?~ zebAz#UgC0BQzj+Apig)F6F#ldR!_jW`AOzhSc65Tll1Hv%G(06m$ep*6lel`?)&_6 zN%x|tHWx(09NzrLs~;Vx$#AGRm30_sy3PQ)%l4N z^Wz`pQF*n~cqxM&QXpf0M3(Y}n`7ltEk@PZB?!t}FkCrNNQC=WK4*&?~-s-yeQ;MrdT%sCu|!a@YogG0Myj@*th}shu0PLjz6=JO4%ehy5nXRGgFP-<#dMs{il!S>BnV?RN zcBNr0=Di$q^W*PNqVFh@XSCW}z>1ec5fzQ6PxJ9Ycr%ENIN2hv06!^L6~#qrz?xBR z1EoLz=%b;DX9$0-Fz$u<$Uv&IRRapfO-uxXXYV+wPpK)IPSne;q$U$R=jjnbT6wwe z-JdN68lJ>UGtI@O=0BxZ;zzH|9XUYG!vr%b-t6vl#--zwSf~C#L;hkxh*=$9x=-D? zdQ#?7sK^zZY$YM2zZJ*YPQhA1SBTmb8c?#(dtR2%({Wu?q_~RIEsm%`prca+Hi|pB z&sAdwovMAGc7MB-Tl6}kN*XCVH%Y$9tR1mgpe8a>5|YR(AS=rrh&BkLoq+*D@?3q7 z*T$C#ZRFw*OaFtt?*MCKN%F_}+2BAn!HH-@GC^R#U>guQ7$lKIcm|0i!ZXo;0VgmL z!4@E7vIzo7Ah1L-LEu2nV4_GiCP$M^^k06vxBGT)cW?LZzPtP1?Rv37}1lTX-OY=F)Hwx0=(E|B)DGfP$>tWcJw&py7zcm zTFh0r9s#edU^wMrRwj;E`h_+4yaU_a=edbj+^(VBA=fT3Pp8)=9G(wpR9w5M#>Yq7|46TBFG@NnGZm%UZg893Gf?#}ZCx&}D}q zLDmI`8OerI_1254(}E_@<_J`;c_*-ODl3UVsOkduFwDkB}+?ZpfaG|-@E41AH*|87T8pH{&cU=9Y%|UEH^EC~bWhC2@7f>2W8hF9w zHKz1LnNppHggmrmntUJ|hkuT=?VoC%u+BqJN1Jv2a?}6DCl!jDLX~L^GHLa)Z1b|= zR1v!jM^+bpsu}(kx?glNBLsIkBg0%d2A!1JuCH_#m0#1|Xq$BZQ%jYXq8x@?Ln_o1 zXX)x-(`CoB+M*vfFtvpjCeH>8!Js#!vf)ZsP|}HcIDm9y+QV;y5*{&Hzy&XWQh|PE zE<#;0Nzq`&YQpk-M1d#rMOwirCqYMSJ`|aTTNusTwqcfYV4pi+-tTz!ooPsJhe!U6 z@99(J&NH*Ky<roV*N7OfqCwB8=>1!>AtB#N3Fvy2LY@=@Cu^V?5f zR`bqWP(czCM93-Y*iz7<*RJcUU`4jUEDPDGQ9%sMW9#r&39^BhVeK;S>mdan7`_r` zHQ{Xn&U|!-*)2^;C{{s9P6%v8HIGyCf+@BI<*Aih?+A7dP&4_ZIhPky^ZcS z1F_FyTzjJvG-C|syWD4RM#!a#)L*!igb@#lp`20)WKnbycYPnNF>!G*NUSM%CTHkm z*yUP^JOs->iRp zu|DJb)}nL#brbj@SPg=;?bqC9MNilgeJ339NTE7G(s|?8bA;S0%YtIeD|~hpr>d5G z7!|(g9RTF9y^=W==^huXadFr}Pv16;?1C78mKHrhO+nk!eJ0X(dLa`w?Ok%XK~Mt7 z@wHDOJp!3f!UZ+2SxVUJ3fw5^RZ4C>8@!{W*sTm|UnmHaTph#NBG>wG&}y#_b`zWjo)6+J*#b?&GP2Kcn6zH*JU4J8cbKbR4o?QF|s))WUQyl z*$a^3_R{dUDcn4Y5CmY8lO` zmhSe*O|c;!^k#`ncu$2 z`i&2>P>G_$l%}H#Z=pdGj&9^%roSQkE%0el4ml6XUpau~OLZe+rG1R_e3>jhj^E6V zMf+;YNp=rCwHa5K4GLQHKID=J1u|cWEqCcAHWy-+v;{m;yIK~l_G;vE4>$0c3kSJm zp|}cf1ob#+V=VDRG9YGX=4D%-T)u^5GndqOc> z%C6zH1EYO3{=MdPuS`2>XbCwao1)O87e~eSdZcxEz%Ih=@o};wQR$MTt*r|WxqvTI zWQ}3B`RwEbS|{UtHr%(pDl)QjtSteK#nhFnO&$&6y=7%ZO~LWg9?z_qCQUj%VO?@+ zu{WW$PTp7(=|x11Jj|DJ-JU>5`>ETiHhcU6#*E=(lNbotfYf+1T62X@#7|I*Drw(C z4d7zH@LBXU&s%cip4@awH>ow`FG$nMcDI=y-2nOaZ`y2{4@Ww@I{fwrn$RDQ-`k@c zL|{>GT>g0ZU!NU5YoytHl}Elva3I>QIQ(`V1ma6CwVkXbC05}WXcqD0hu5w*`Mkat zJNt7+8p|138wq}Rg~e5;Cr6@i`X_j#5>D}adizDV`(3?0{&D#I{~3h?P71jhn2Uv% z!~uL0%h&%Ig;$5WvkfXX)-_mN zCm|(C=|YlHP6bW?68iXv*?g}?L8Q;z(=G{4DTw0-fL=?k9Yl(^xV5ge7h*aGZAKx1 zAm_CGoh??(+rm zjWewDk8dgVjS^u8KhAz!V*dX;6Usa+Sd#nby^7U+F3H2oKP|d^`X#0>?O5auv?cT2 zAF?DYkG-h*QHcM?CHuM;|64N$$dQry0-~Y<3FReFP6<3G6B9-Mj0Z$hpXe(FhfGb} zk{J9ADIQx;;9Tn8nf^#OuG?_wuM7&F_;KlfN3MERVgNoT)t*#ls88Ew1XRGEwG~ef6U(_v>G49DDog&%f*6 z;W~)(l%wY6i)wY_O8=%??_7tK+(4;Ec6hZA63%^&kj?mN^_!KRO(zrY{CEKKz`s1k z_U*r%F@Cw<{3h{lo*r(~)cMwHqMAJwHqOwhz0D>f-K74ui~iH6jXKAhlsST^6q`XQ zsPZv=d{kZ?_+o=hl{6bYFq-UlLZ1^hOCWaPP|dwC!EL}AHLhMEkF9`yI`PpEq*W@W z@brUTPDr~Zis3$jGid69p58F;3KwNx;k9OWz3}Yw`KXu&>mJ!|q+Wk|BV)>M{dN#_ zZD1w;4AG9i_&SuCFr<6q>i**LVjk8l(ziw<_Ve{O6^7+>p0E7CdD~f5nbDIaQH~H& zm~?xCJIbM?I9>0X-}w@~R;6!lMJ>GhDQ*6AyT*Yg3ot&w51Z;tp}D-M(LkYlbTu6md0B#fr5^$NfH64G99*7!ybCHv%2ueS?@(X0~a$T3WV z$v*#h_m}%N9;(GHD2*_hok4xKf@t|DdTRt)i+WW5`I3&sDA01lG@Z?YVwG>|NQRdd zb1YJ`U?VMvvhPggBcBHf1jg`z`DgT+5E)NS#b{9u3fmA~yFG6iYhD+W7xtlNXC<&p zwW$%^&q~&Qco9&)Q_u_)JQiG8E{nhmp8WE!e?ZgA$Vf)Q|$&7aKAtywG3)Y z5^>nP;y<&+Q!5j&i4rGWkOkXIg>%>q+KXzGDbb<^6&4bB>>yF(3HR6!a;xceI$ zWtkVQB(B`~Q@!oq9E*M}ASgOgIguUFwB@mVm3PYL>7qOVHUT&T|bN~%s2dAU1u?U{Xx4O!vAdr>TTQV_0B9Mp^4RwB%ocYfYf_^<3+80b4 zn=9r(#U5tsb}nZ!Yi6d|pP#Q2SMYW*Kb2c&SCyX=Ea>D)3#5@qiO_(;w!qlPncTBs zeu46gn8`Orz7N3fpN~UFpk|v!A(Ki{?2%D^!;8!ccIQUy3rOxFGN&cZ$MLxDdC)#m zQ~SEU+z;dK7ODUI6;d9LZ6jK}{Fuc+^OIf$yB8<(JX<{8s{2@mEnK2W$T-(tWO263 zMAcC3>}!iWg`~&}bk$62s|RYLAZ2`mUZQ>d`E7OqQY2K64^F@mR7wAg7RDK-JCVj zOUjqBsygFpEyzjuOql0__>KiB$_29;Jl!$0Ur2ID&$2<(-ap;9Hm)^AqfIGXScifQ z!C|*y#a;~jPeWr)ZH*7W812!;*4;HtJO3xVcA0=_4Ph48oz1&9LN9O%bY)=++I&HN zaHx^((&WA!mn%P5N|q@jwpmv>FYq27g$L*mezH_J)}JbK%TzWE)I5(Ho1g^t8#fhBvnwzhej z4sb+n=J!+Cnc8ONjJO!E$(?~@E>KaCQh=}I7no%6`P}XjgddNM8awmrgj)JUu3LUk z(VBhcZo>4m#0m2J!stnoWa zvApCoB92=Z@t!11z?fq5b$b^zse(|}h=i@=wxU5SeIzOUgkCj!;I}c_$aJZ878#if zgAuRW9cuj;EGo(WGE7bBsqt;$3%q7PF3rFe-c_Si4l;MQyD{sVy4PaP7gRa0#jez> zSThP~AsEV`o$8zeu&QI z&tBlqE3}mgVs~aHEw786nDZe7O#$ddoki2J#O!xYmSQg7gxr&8Ce|?(g+|CwOycoF z(F-1jZa@7QYeY=QV($Q|gN3tDE&eu~Ht83n%0@FeNo+_>rXK~pRX3*QH^>~kQXf!s66`7f!515tzXJ_tBK?44u8rA#p-8`H=XUh!Ub`l7m7UFXutufIsYiC1 z({=+-<9B3<^c%X}Eiqbj20%EbI_!CZL^Y;~T$;gf z(SX>`P{0%zVy-6NE6_3lx}B_AJiu>`lpz6T{Gy9}*8O}S)H~S=qezIk0ATMUHY&0D zxtL6a6_8HK9Uq7Ws8V`sW3r2^X|h`LftOL}$m{w0T;ng(M2kw&@p(4pU}HO=+C~#H zi>u+vv;8aMLu&hYQr?HTd;H2cZf_yNXr>U0H6dElwyJ->efnzRJoe7Vr$;rD+<{nfSAAGbR9BJqbuyE?=B%6FAqF@(W_?+*`pi+J=;DMpzi zT5*dZPM4oQR{n{^KeFiBu~RpOz!bJ21p@afxvylWF@mkV4ak_1<|@-BH$n^uQdTIQ z;p`jG-CU+GYN}JFpYmZY%kd=Y{VYawtTCd_zoad|@^nf`$sKT$t0?^{v~jPg&7^F$ z=MMcyY$Q)DkG6?8-eH;jVhl`b(R$sL9M{F_*+KDrgEs(D)l2fz&QFHr31cK;ptQx7 z*BKwh>AjjzP3R16PG>5i!nI-@E~_=Ei-#LiCG&gAc3)U~WV*VDxJ9zVDxbuemBxyK2YZ{JV&DX)q#~wQ6)akWW8EY%`jvq_ zL#ll7aM&QhV9(ik~2RtyKK5GHVV5JQ^D`>i;Z`(*HHPP@*l(X8~ zs7_C8fpU;ZK8e&* zTu@O0PS2G#8h>UHqb=O!5yrX*g@8biIoyo1xS8f4z)4#!6A~kAF6sd)>w(~dUOo0J z>4r4$Hb;S=Ae2q$CWrDJZ;p*`Q7%WjdxRRdMW6XaX;v?*;-*L)TcS_Lv1=-Fi+7$s zyC0vLss1^7xhSRY&|vD#W&zmx}1z#nm*umt)cp@EG@RPzzS%?-)Qod=GinRwXMlBPT^ z$1mn1gxtg6?GvEZvf0)7deCT%;7pB)!D z34|d+#>XevI*M|YG7u@wx%%PwcE5uOKPeAx$2Eb#KUnK;&9k|n43e{VJLBFvLx>F7 zTW*=S>WOW0%>-lu*0-=qdb&Ai1@WBmi&!t=6r9;0H-<+B-RN>b+;vcH0v`)STBQ@c z-91}U%NbJI=`!n@n^T*clTUJs(y^3PZH#U%@Wz1mUkso7*8}%YKeQo+n`rP*wchb^ zhDGQ&LoagSJAbfPBP68tt4W9lDkqHfNH8Knxu%6?kspuOZMjGPtUDKs_nOjdr!jZi z`^qqa-%R_K;Bdnnh<+m))lpNp%k|92#5!Wam1Gkb)E@k@s^D|hm|WmaqM&!MSC%!g zWmV%MFhBL0%OtS_r~wZ$m&Bkcl3{1EdEG9ZOJ~Kl`%@dUv~&`_Gp%_p-Y?rtS|<_l z!Ct~nie^IQfueIsH6@Hq)UNspbGqTc*jQ4IC6RaTJCnV{?Ym<@RyNHM(%SlbWB$w9 zRr|B3BB#gN`dlDaVQS|3##|acqlW=y#>q8PO)yzTvU0X-Rt!op;tlTcCV3o7UUyep z5*Z|*%WP-+-%;xrJ{po?Km^2brQK|jF=Dvao$h_q+F)Zn#~YAM*189-xG`5-YzXq- zC1aE@j#nZenOe=+9-6ofLaoh|E&8hOG5!fxo92krp6c4=VAQk;ulcj@*O{{$RKaYw;el#cUYsKY zCK{^VbH7jUZI2O^(vN@8wCT|xOokg9g=9Ya8B~a@ZqD3`e@D=r{Uu;{yU!BFhf^hY zOdw|v=;!Ai6=KdebTMMc++tuOx*CC0iZYG3Wa@hGEd%?9N*^Qr0WfnsHr#R}o>Ih?w-T3apnb3s;Ow>Is8 zGpVf7y-jr{d{fC%PEKo+R#rkUQ$P9eU(5Mlz4Gt)O345NUKc}J7+kw|amyA`?u((S zOuC97pfFMBlFp&K>yVCAQ6JWPV%JW=E zt-boBw^a|-#`9oIwIu+=|nOYy#a-K~StCXmR$4bH# zK^0~|hY~8ui^BC$eF&gc3xVZI!(iLo54mE~gj$^GiBRj53c!ZlmMO;c-I0`;Ob5m` zG31h#x6EL$hKc*SL2!km{=hFQj%}6hOOZi=x#}~ATg1c$1=AWWo`=%C6q7LSoN|Ib z?vLdi4nCR?5wc8N8M)|GPJ!rr&|I`WtCcb$J><0(b-o!V{qS<*K%G|!Au(YdK8G(e z0jv3>e$|=uKz}plw_2ZtB!Z!@HGG(gUR##_3@GxWcD;CLora&4U)spzjp%^-^qt=wd zPhBzkN`xM2#=Xw^Z)HXfHW$cPFy@y-$nY>mY%G17>}fk;8j~vIgJN;s!lx`$^;9!G z{iT7eKnqx=X{5{8aW?B!^xUL3FKMEvAXWb%Gh5fk_6eWY^&6U0@#GMYS?mde!2}VB z=!qD!HTAEDn@!*M=XHw+V!Z*Gfm;G}fa7$_QJ1QTU7%`V;tQtY9p`MQd3^&QQ(#0n z(_4M!FLa^5Yxbbk!pa&jj4j4C1d3lMJ2tdC$+>wfJJLWIY*_vQ*%;0Tr3ExL9beS!c=CQ5n+T zTE#?p_i@QovNfob+*KYRuqE~m?8#+V8_?@f(2N6LKIHHux4nPKp|!Ti9N;BYp-|q& z5Yjfr)-Ihd4!02R`l#SE>z?c73XpTc2t97@nyv_jUv+BaD`xk{45!0SI)Q z$&h)p(8s`>WNA&FBG@ol2202acCE-XerGE2+YlNMgsbFGhzypoVM6!;u0SmB0oPCuG~Yi%n3?j^sp@(UJ^o>Q_er(vHYl4a8g=- zYadkbQmgywtba;X@jE#Ek&9G@Gm8KW1kmB&kB%xR;e-QE=C z)XZ#rMl4CIDORcP7$abmiZJkp7mjlsvBEg{>7jS zVt_tyPMcW2Oh2NPY{)#DmB0%q7s4#N-ip=?E$W4G2AuRuOHiO{-M;a9DN&wEVe67< zft%ZS&d{Wrm`xQQ))$f^&txW$AYN33DTu2Lq;KEhxld^p!73i-6F-ZSt>~YBP+C^? z#P76p-}OoONqub-uzbEH7nh((tE(hV0iikHQkZdaGwG@ce>~9B9*oySMM^#xXuCPzV4ZQ`c>^~Jv3J*#3R;B)ovkkv^imZ?RrOH zC1`gw^mI+<4AMV5Ex|k7J>u3WO*$Vljy^>trrsdKKM z@#_e(?OyckR$5GemJEAcw9#z;2cG6H&qj^G$IH&wrG{y+HQa{U$aggH%eR4`Fu({D z%1Ue@5hKs^-szy8p_|EioPMq5YhS{;>AsXeEA)ZPJIs8XQhYc)yIFAZnqG<-DJC=K zCskn|5AXgWFQwt-=4Pj#ulMlXe)10WQCq^q08MDZur?t|uu}L8DicGm`g>-=J50yL zDZFOmlCLb2Ys*RN&-C;_57}lN z*)v+beNDrEMWyk_>Ctzm$7+`aezoPNwoX>G||G3B@S{C{3kl|_O zRW(?-Eip(S(EEBx$-VCP-YxI>E0mRxPQ3TOwUoXARip`S%sYpMFpR8M62si56ket> zP^lF=X*2yYR;VVk5u*gv*(^C$)8)w#-W7JmOM$9Und?7QKMK60yGsANBZppj*^p8@ zMb&elZMUiO$BcMX0IY?0-ziNauqA)=-bY1Kmm(vij2~Vd0Ii)NcEOst%B?4jH-9;Kmr*=t8pr<%G$Jxh?CejtLMHz%@SZNWYpk* zTf%#(mn9o}?tCH|pH?^sm>}djF7n4U_9?N)0w~-7a%IM|XA7w!{KLXuZyR0!@tKGu zT*nmP2Fwc8Xl8ruQEUx27S5EOPzY3xfZZ7s_Mhh^^&%M?o!i$Y%iN(OvU7>iVz~_9 zej`6A1k%&jlg2P7rru&a(f&M#FJ!|#+~rGBMyXeIz{U2mQjjl(d&I%UxCknLFFU?4 z`>~@TE#~|eNLW=q0o9+_bI2jlFg5S7LdD5Xu}-xVh!EKexaCzbi>R=K8n|pKExSp! z>6*l1FONcM3%oBRwwuePOP7pwDW%@$7BmDu1gWx6mPr@nF+xTU;u@z^VugEgE$t&w zyyO=-78C9q9K}P;!7UhyfFMp*vTr2t!gb-s$-9HVktY5Wi91aap1!~vt^*iehBeyI zsI4P+P<`=CTW=Sjt-e%`p`MJ0f2l@5-Lc?OvrK6kWcId)R_>Aa>0zZ=3X>?gwd`v^ z7@v|iRXT*^{_yhert7tu>?-BbirlkSR+c$hGTR9X0Q%gs6C0u22Mc)zY-tC?d)Tw7 zg(Cwxqew8faUMcqt z;_S1B!FKgGD*-i6v*r$D_6MPR!m8hy04xMWt#p#%KMVZdr9;klCWlRL4#O-wAqnk# zH*9cto0n{iS>U;2G4XMUc0)F*H1`Q_>M}8nJ*E{@41AS2+FXi?Oob}6fYv{!I>C(1d0ZSV-(SG6$Qu&TE`i zet~}}uK>XjYs|m+{3MD}+b_e(>P0I6+4|4UJky|BunZ|a(tg$f6Nb^ZRPL{9&XG#> z1^7Be9B=*b3;A1q9+=-*HW7?s|H0$fOu)15^gWvs-!ANR7~4L387%glsjf5h(OB!u z*7Ln>pMx=titkKw4TbA#9|WEpI2axvLzB)=fllnbPu}bM&Ln@MUG3V2Wr*y~6=kgL zqr>Z}S?^2!q17@d6_5zMorR~n>d`G6A1hhUI$0>XM#La71-1|Ny;iqlaV}yGS Q z=FJvEVKcMet6ZxvlIRbUyw-j;+}kiI9540?BN&U96}KWq+qxtJY1z~EzAD;=`Wlk3 zFIdY9>hG`wG>O;?9!B}@MVcyJeykG{eKFDpQgQx;{R_h~7WnlEqY_Z*FEMu9rrgM8|zy*iT(ApT0J0}}Q+GxrJR|+;cz;e0HXP4Lo4*~HT*!e6WW_g|--y|x4X&J_9TJJXSx#M$}IwX^t}-zv_2RoPCx zxcZ$bT`GNO#@}w=V&fp~KKxE-t5{8&=b8q8BcsAgaA?xl)U(8f`ipte^>&)C{$Y{6 zHBkvwc{&4WE^qX+xkC6L>b7mCGv64Ig4b7KJ6LZuR6vT$IA-tVIrx|eStJQsCDeRp(g|rP4RAM6Nk|n=&~)ImDhriD zE*Vex+_H`~Jaekq*`=&oS7Lo(q||U8!^s|NQ${7z>V*{!K^88ivyk%)>r@2YJaP>R zDkYI4oT_J@p7XxvHfbYdqMSgpG2xqlJZJ-fkUQ%Xk30k-DdCjGbeLd_Nd2JArFr`@ zbD5?&U#7Cle-1mgHkO?tRmn>H&=-?n*%cM2WPvN`J;OIwufTX=HwNml?y}#TlIQ8< zk#DZkO0D0j)18(L)_uMh$;6yJW{7LxY(#}aY(m%vNy zPwU^iXZauA;CPnviutEEoQ6K`+D*IyKyG}m$<3`BB_S{T!l348`v+MXM7oM&TDVD! zt<`aiK=58Us{%eCFocf=-u9h{VbLuQu0BH3+DEBrfZA|6j;;7;>{TU@BTXe zUt+&>K6s&qCz$O~;8Etz&F823>J6U4+b;j%z5l|Ax;k9w2t4&vyiu6Je?l++@dT|O ze&D*SaZ4kscYv)kmpbKMg=**wG;FzVUlEYm9}3aYX@2+@UgaOTq!e6z(9rjQ;dv5! z5fEfM#`u_+S9AN$)O42P#&S?V*Wv+9+B<-KdF4>I!*`~^;S+n$LgkW`4MKNS9)4$f zu{`A~UsbhVeRr&P?b_BioWfroaE_~8n`&9XbzS9tb&Gb^`78PF3g1@0rJ&1jXJHIB z|It?djv{~O0Dao}`l7_Z^W%L6=mKv(K+%SWcOV1&n=|eEjMOlPdV`I9wrJu}l zsTpYtmK)EZ@bf=H@;@u96kYQN4qW{w2cifZOiJ}X4?KlNHP89xJaz$GfqG4O52GVu zCY#t8NHq3YPDp@Efzq&=zaa-vZDo60OF)1^F2Qosrz&ui-opc#CHzBT7e30z9X&T9 zr!yj_%fPiY=w|#e&~878^*G<@0wecFWH{_iSqLb5C$>8H*Z2Q7c}XNkWgO>5Hn z!=swJd$)XEt_rv<9XZgbNo-3Mxw)VEHnh=yO*zTdHe~GD^4#{zpgsL$*{e6FIL4+J zFq$!c?AcE)o%cq3YngcppdCIo6Qmu6bP!ROb$sz>g8e7SD_3OM4pEK5^cdEV|M0Z& zyF33^s1W@1za0@7o>~llhK+QE(1~a0i~h8#na%G^wz%>t)Yi9$voZ{bgZ~gcYx{^rRmI{jZqt54E~~XWCRNn(Eklv7$j|P;d8=<(kuh&i#z(prQ8X zo*T14$&Klu5oh8q9khSCIia$hGWeT`hlrn?UR*nKP?-3vrnbE34>R^}HMZ+AE6)_{ zqN-81{wbaRl+HgHg#L}hKc(~6Fq=P`UykU4W15a%k#NVi=^tGIw<(0U2X&QF12di6 z4;&Xh_IwgEiBJm>TX<~gkmSrtbfP6Rm(8Uk%C--cRS4MVIeBD@!%N~3p75{7C-7jf@g?$YLaoYf8OBVLxUvnTaNY-9R?#fn^XUjh!y^vUgD>m*(BJf?ftyh+Rp8=iMibi!CzF9PkwOxHZ&b_CU+|! zv@=&D=AGPT&=KTa4D@w5Cu?3y;=*o@i|3-Xz1Q~877<0mgx!GWga>E%HN=S?1*2RU(IGNoU-of z2-zZpr=6Gcu!jfq%Ta-B^CO_hSHFR{WeJ=~2Xf@cIK7{A-E}L4Z}S(~2isl<)1)UA z$a)m&J;T53-Ao}wA{IN!2LJ;2_$POxN$5TxCtD*S_1qaJ)+8lyF_$?C&v+J;VBT0@ z;kG&1JUm7toYDw}eVe)pyfeKIraH;M$&B7O3=<9Bdz>68-Vjm+p;%j^5FZ zJwM1NPa&WV>Ua!UJTP5#d}g4^Ub1Fu^}!@1XK$Tl_!p`HgRm9)D8$|eVHP0YIDpI@q4lZ7T_tYq;(Hhj!n3$O23N*>Hu)FEy2 zOav*s;Z4UcaI7PJQI1&aT~ZOYZ_yr6CcfL=B=%Xi&j*#z;fugh^$8ws->=7S$$V?9EPPZC8hLQ&#UBeX!H-;uimdp?54Y)m z`_Bsh#Z|*|2nabRd{F8HrH$1D8K-d{gV#==ZVAUQUL5ekNj7`xclohaM^`0-UL^Hz~960 z{CVmB*7UI#wS-5!$@RjO83`s$ek5rE?>Y=C`TE~LEgTMe|Es0+hqpGuI*qUFn!Sor z)R)uQ*r*Q|RY*@^W)44@chAZ~Fjbm8UBGg7{#eT;%ic1{xOTlaHSY4-ut6uEHpUG+{@LfrKK10KJ$b4Yb8I^T(%Ea3VkemUfeuQ zB)~nRp<}KcFbl17FnVgqb@pGHWJK+Yi6Fi$kL3G_CL@KzEgvXW{I=16uX6;_&$hZk zMitiEUrxQ1xnXSob&k&K0?)~8a7wr@CQEDV)yhkg0G!n*By&*s)% zMj*6S^2v?;%8g}SGDOGGebw3&`^zYV+jO@YcVsrDzXyD$SJ*nfOMN?_Mw}elI%Dqq z+9x|N&sr;0Y-X&IfU#*7^_rmJk}k6CC)ZzkVt-%V4XfVYH(r>)a@&Me(8JfdnFhwB z7hjie-vQZRa)RJl4fvcqGV;O$6lWjAwt3#}9kHHmw5)jdZrDQU@m4D;$VXO*9L>{b zFj6BGD&Xaw&;jl=?d!4cq1=WHca|^j+6fx3XFY~EsGNXm-`2tcB_4R9MKW#M0s-`c z32?{elS8$0lq7zKbt6&EqXVvBysCsu`B$10x))h4AnhZ^%2s7dvn;vJbrD0h71uwd z%e9T3Wv3$+85`Z>T(4cf_HxDYo}O8!UMYNTy?Ju$BI1#oje&|ebru%ZqpM*UuGha3 z03Rv0ozyBfkqi>c&3>m4d3w-VD*BB?i&3j7v3oMtE%tp~DagU5nGrVdrt;doAo0@N z!w446f;tz|gpB3^{HTaINw}9aC@IBZmfPtUa(WyO!pGNVF;EL-Gy(Ol6kWSfd2*2d z?l+IqhOnm)2{$^AQ-YF3TC3hzF(HM#I^p;wh#O@b?mi;Pb&e3%t1ZdSTdNr;kb`iV zQ24n#Nb8cFKKhOZOk2(|7c133@j%0q+R_pM*Lamo!`2m@l$}a=DTT-@Mb}>aw~hio z9{&4HPAqwT`3F{!%`>nJyh0Z+2Vws(y+tmN3QSe9w2eLhHLeV#n^af@V3N0P(;x=r z-Fr0ycj$vEyu&c3*HONdGL8=}r0$!P>S%uQwx49#MdVC3vDkZQxD>IFNZSvtD|KiR zwpP00f9l%>@oc-nGP{Z6?dZpGr?X>X(Mf>7``xW*5kaf5*efqwGUXuMO?qwTYUAh#}W{&FlAS4||scjIvn9>C`W2_o8(JfE=DjW;z8uuQ*a|hgD zW3p9p+TJO&EifT9rZ%o5*(9R3+X~x}AVC?DIozwNM z0^9!Bu|nSU2a?Yu)n{vmHT`0cRV_NkX545X+kxyi)^mC??lsjiZ$#iMc#aFAEjtyIKvg zN&d99z&uvAh34w+t9%cU1eyEulOv@(4$x>Jdzxfo-dy-GGet<_d~flR(`hh=yKOnf zx&>c3*nHyFX^oX{4q+`qvQ)=LjE4;z>V^N9ZWOL*kuP`md4a>le0wQJQ3z!2q0 z)R$y}d6*Zl@taAvYI#ym;HFWY2O`PMJ|Iy*%}+&G^IoaW7f5@|I<-(b(b?zL)W`G0 z;gSvvZ*r~@lqbYR1j1O#S9-aC1XoyyG{$*w!9=Xb!v}a@-W$jzJc`au6il6K#6t`U zsM0q#Iq#G1^tZ-I&sj2CN1)oQ^X%^o9Wfet^;joPNVdW^&cECp+h4M?K4Y2PU^~+v zaFlAWgR_HOLe1O`I!bmLH#iAl?d%K7+#g=kQW#W!63E@AAjW#Oi` zUHU`Cw;w@Kldfg{lA7(flN_5yW1wkKE-p1nYcJMT_phtXIfY2<`J*c-ZByR zS{q${Qx$P)`7Ba^{+&t1drpSm_686fz!BqKFfE+lv-fu3iRnThM{28HK!QT!zz9G9 z;absm;B1-9=RM~#Xv~6QpNqY)7B}dNHEB%aew$Hsyy#r*&~2!HUT<_5+zePo!?G7n zX-RmAHVe0VetgOI3vvi8`#{8s?x)mTUpZh~ zW~_1Q2@lTO@PdJCwTmUq8$*bwAMLjdK6iRXezy3UN1yh`_Ie4UBvB=`6#@mf);I2|Q3Su3pCwUu($?fx&&5`YI9iM1x z$0&7}K|sO@_^%3ZWBY9^Z~8)dB6`n}_<9;nSnipwO!Nq9rKy23J|59)Mfz<#_Xldm zBxsf7GvdjIMh*QQAB%n-p5}tFA5%S_GB6taK>G3NH^A%RS7*LjAZO{N7<(^;eg;g- z2ch5;`bI4NVb2P-ow`XZGB4&fWW* zGy6IBx#!+Hd7kwot0ZgX``5qz|NqwazOTdujKo7DWrQy8&7w$rNR2<_DMF;{r|ueQ z?Z51qA`SpLI~hWGae^c!sr5`2AeFuCfwqYHKJ2o2NFC}w9$o0kJRcDl~h$f10 zf^sQ!Nss$%nL*wB&%ySgvS6>%D85I}M4-n4Vfs3gtmU%LCmH|6*DXZ2@TtZG?(vTYNdDgw5&Z)NYnIYG zC`Uq5k7%Mv`JoxY0l9VC4k1!7NU?etXLY@HUprAh8(+2Ye23i=B7nDTAENq`$u*W# ztknU}BjOxK3g8JuZm@#U4P1Wy01N6Dw80G<46MY6%Fb5S#~}4qek(I$7hM1NsSMQwr26M zIjpMDJ05ns#|3T8Sof=8gSru;I`f=dnU?Bc`UYGRDQ)8m7F7dcTd0uaNSnvx?AmlacK(I zxapDNfS#RKvirJ`SGfm+d3OpZNJXVs8=H+f0)xR|L|wR=E6h_~$k#WFcV)4ro{CWW zDfsvW`vGn5w}ksK1QP6NBbjJbbuLaQ+%0xPId6t@(Ck&1*?( z80iic-h9Z?!MgqVVX>4dfnlv|d8Y!6f@&nxn7er z<}d#^Sb7K_wIQC?reU(3-zRWW53v7v++ftx^jw;^t*l%~`mhpL&WiN-kuqj7n(-+C zG^tn8kw4Z=j?bd`eXOaCQM{umn1RNwdHHN(G7uAu!;3`|hsAqe^{pu;gnx$1%Ax4N zwsSSV$QQja8GUQZ?o57f)r8i9%ch|0a{YiE?LGFx!_(@+tg=W#5GPen(G%(RHFogI z`$2i=T!^h#g#bdQ!l}<@%Kx@_@tfG87cIzV*?mT}c3^=J?bkoK{nQY@^5sx>h1n;z zgoHhX9p5$j)5Q244=b$$l?6emwBGD^fT)iT=3<*0Vh2|~gWTigDdFmP7=R8K&R$FS zthAwh!M>&R$;`GT`elrdO9l}v5WwnU-fCA)S9+b&Q<_nUwtp?3h+1`<4Y)0y`|6CG z&?TnykV#^3mptvuJzE}Ob2^I-nCv)#KnzJdGSD@FtsHQ>;y%y*t50r$8l4{+%RB`s zx&@0&H|NK#1UJ5fy{M({7WUlUZ#J0=y^m9IrmH`y7egQnZQ1ZBzd-2DHJ>@tKF{DF z5*m(P#DCPfV$b$KY7D*7NIkZd((~2J4{e2dk3DU0XjGEQW3)8($%zl|E|Z^`R>QnW zsovL~>A$xuf~cDzbC1)84Zx_d84rHebYN72=or{7v z@&q7XIf>HQf}Z{uuJUp{>NA=dp-|F3dsuw&OzaZpv7Toz)p^^&237V2k8=IqX1|2g zk+A$c`<&TCF=`J=t*aQUd&Nn-W?JuyNbV2Vm+q~d19~>uqd|)fN#6a)kXdTP0yg$z zM0$Jzny%oQs(m5#Yy~%NlK+^mg@c^6x+FVMJgcuO!v68QL)q4h)92$Z$BApXAzAUc z+3Vfh<7pw?+i{GjR0=_n_ z(IyY+a#JeKD8V#w!R{Df!>Y&P$$#+$;-3@7+JmCIzLTHS>K_~xqSsHH z%Y8$$1gKX)hk{6x>)Itus)iHOWfn8hxiz;Qb;0CVHwc&%E~QH8vo~^oYx(V+S9WX` zWh_I&(PQdL;fzkoT zPAFfOQ6{RV&1Pi9a^a#Y5v>nzOmH$1Yvw6at!vzN`&3i48cRaP9)^u_``y|5_g)KIMpgff36Dru0khyy_kcoi{f9d@2p{WpKs)`8%=<@l^R3jH?H?;?EBW>rv^h zQx(16+wl7G0J9OJW&R!EItUq$R`T* z{ed8Z+0Ym6Ng~fF)B|Q;&B$}JzP(>mCbA-#%4;p zxSco5Q{Oc+(}o_yon=*YAc;_1cyN#$1Z7DH(P-kQJ!yWg+kw#ICxzs7nODkg0D5CT z$s;X|f07o6X)|s%zGyz?-?_?yOr${@SN&WUv$`oV=!$wB@rNG7mwv=WvY$_|#*DyV zK%vm0>lou9e`1%UwajE-Dtr^_cGJTbQ^C8-rx*cMIF8tLJ%+ z-}0{e3%{_w7Vc?{dVWaciW?W9kv;W`*xY2JoL6W*y-rt0toe@5 z=qGol$6t@BQF7fAYOMKTCH(t)c6X;4A2cv5UQC>Qs^45(De?>Zu;4kMAsEos<(2gO zv8dol0K^&zi|6F#pe?8FEnc6%l$O@#RTf-Eyy~!NE-(SMm4pML$Z2}KB}zZguz>)n zIJ@uGSeKA9ZO27NG{e+NvNz)qY!nUgn|c5VNz|1oquqNxqvhwL&dN-9IY>iS`CT3% z+KYeLwzPoN@7Qk5XC*g2w2GY(dcze^YUYU4IZnRKkRhR6BU3IfpXN@kXk(@^)XH+l zV4M_JVi&oR)zv?6QSWVJd57#wsL#Yai7Hwjiiivlzm-x-Eo~_>8?|u^6RHQys=+daib zBPDz`QO>sPvkF)6x9Udq>HE{?T#p+mjbXNfE52SviU**?s>NbRF3#G%VJNRwXrlDl8Z z`Mh9ftr%QzkFk|Zz5YODYeFcFRb!bmvwCE|_tc7>&s&qFrHP&Lk67%c=DacB5cp+? z^(@x^`tAecqe(o*)}}E?l~CItNv5Ia%?*@MNq+OS6&>d);H`#o!dWl^D}vY&N(^VWM<5^fDhv9Hs&RD3}x+Y!r~{aMCX;b*bla{TG9HvmE)& zN4oh67R&k2P}gC`>KeG}951b)jqLOO&`j$LE#kC08NG^c0zz_PX->heW^QAA-oniS zRV8icRJyJ)s*@J2Kq_2`2j(C(N)7_Z71E0{KiKFkT)n?_SjlD*JF6=2VtCb6jz)Crzjk$5`h+m#x<20ZWN*OG@GmA#z*q^6 zeYmBw{n|UI3}Y=0pXeM5l-dC*X%n~`MXZZ)X5EaCfeR!3^3e*FAh5nfC{RPc=mZJZwvg+7`)FHnVow0?29xHzHBhfc9lr7;p4>3AGcFM7~yFATH z6gT)0DY)bhPR|sY2mPM;2D;0;UQMXzAADO0AV{r?*^NX`bjcRtC)?~}n`Y*=)ecLf zhu@QNvLo!7NhJZn)h4ai+E4RR6nK7yBU(-G^%+FL8*qBU6OCJSIkB;+mOp1y%#TYs z#9e;1GA=@s%cnVbOa%6E8O6!uwMHv5b&nRGtN#4)glzmB)KysPpb0yCZjQ@^ldRys z=Qo(={unO1PU*PgCv~RSqc(GPF}jR3t~M4N@p86>#+|lsMJXdIo1~R>lJWQ)R@vgn z0ms@!zzOy9P}k+oGrDOn&Qfq&uBo%+G)RrdRjmPAxNZm3#f-Wu1q$2-ixa7dsz5_h z8B|YoA6|O+UHO8)pP*i78cfZ0b7^hS+5=Y_>|*qp>@^ z)1L^MHt}zT3JB!r_&oRcc&!THGedN>%rCvRHJu=ffAbTN#0!KJ!u6eG7%IaluU1bM zMj!0_bd{iOwz8ol9$OZMTO*nI$Y<&l#AtU(ZRqzvZC}&UUZv73nKQ!Ya`rJfQxCK0 z2HN_9qx3rKEp1Qo=_iE`!>stOb>Kqn$MEnn6kXKh*VFl+Q1#Jat1A0TR|k}F>U2|C zD`UWE)GgPjTE!sqdqwnmj5b^xqyfcn$;7JNE(#g-_-qkquV#eL==jW1TW5-irjMhf zSIxUT=h(`gdcM~o5#_J@FVC5qp8m;nHqaU99vaB}QW|E}a!QVH=*>6s zek1VUUwFR0|K>Zr7hEh+?*4{v+3%hoor*m2?ps!9j}f=-`L`?=WAta!KP~Y8x&`*? z{&S?9r{PGZUFaMQmb};IVPVc%z>U4@Rj-lrJt#TJ4wM z&rZt^DWfH&sTM(9JL`)iFJ@V5C$x-064F-Ypj%B74LlrtRWqf+;QM9Wd{49nQcK=G zKXHpStk-9!8D$60&_g`viM|qy>~CdKrRUo$gocx5gGb3lF;@AT&R%9krY4-53j`7} zlW#*d6JHMw@>4}F_j3Z@_dI@hg4Qxr!QtrI=|b<7hM?9NnKe-898U6poP>>vh1Oo> z>&B5VR6N${7g{H~JFd50&4#Q4Qb{m3D9w!_cNtqba>!Sm>YOsFZ6|MjoW~Vpe!cyl zR$tlM^HywL>Hg{wQBz^Q6Zrrj@m-3bDOX#m6qi1@nwD7>Wy%St@VzcZcCK|6fFl=V zot|-PTuWVjfVffJ)a#M*BiXRyTzQ5!K!2h2aF17A^RK!oH9qlrOJGj8DtKQciRKDEZ&{*b+a*T_L94?l1Fqjy+ zSTO=Ssp&J_st&qW42|-P0h@Cc9Cxx(-AC1MD2T2_b~EQ)mUGWY9`-ujy9rbOx~J% zqXQY6u+H}m#oUl3ML#`GSFX%YJ!BVJ=4wI)sD3go2JT6I>Yy)2NH^^G<4J^yO$N@o z@kXfZ%>@XTL}OCNjkud(?Vr3v7xOxNY_nfQ$bMMf{Cl|&S%ft!p+}EY3>U~7)tyj0 z=r4xD3LxwA*g{3q+8Qn@Nox5B0hPeEH~1XZTcNk$?=y}=`95;B$_taUW^&e$c7W^F+@$VDBSqeh>m@<^Zh6X$I0wdrRj}3YYcXEh zTO+ZvNXtf4__5G-^U3VyQzxkpV_Otn!*ntJ4Ee}qKyt&6k zMTuqN@pu@6vKZ_9tESw)c!1lphfmUxRR8#gh<9T!B%6!kV9Is2Hm&MJD7u;e?knv| z2H?uGVkvN0G+RJ+s&hR>bTZ0ol9>Ty_lSr_+g`Ola{S&4+wLkR(OXHCu9+*h(y)8y zNptl~{ZpUmC*hYt@beW0MCA#nrc>WK$_b>#)ct6EPIc1G{oJa)BrPx8a*1eWqIvyT zY%2*VaTOdTL{`U7i&?1GB~e zkY6XC+BtrDD$7;0e8c+f;3vFuvek6Xj@aU9qK-m1iKGj)1Y)TJzz01eO}Fwrho1ay zt|N-eu4=9CKgS!sV!FLRSJr$DXy}Qo?&(r#(!5m#W^W!dPmSBW%#hmUWDcvY9SGxU z3Fdq|&*`Ug_RGNpFeT$Vnew_eQXW=6 zLzuUBtCEkOPf_$~^4PqF&P=a(K5~VJd6^eW8hf|G9{Lv)PY39^d@tPEi@Dt-I-Bgu^j2 zFXKk;@dWHaNB^ev@m|N>N`Idx(ilaZ5r38;{&4r|pS}GLn2AJoq+e>0>O=WYs>$3gOu-)K~ z%17>Rrtr^Ns{X+K+BBqC`C-NO{3U{y{(JVNtqRpS|5)8Zj?`ny z9bKhB(&fn3C1(OMdo{S8>Gv5#jv2iDcmC4z*4ww)e0p<@~p_SKVDC$@<2( zwC~Ejg25afZrYQ6(ig0FhnsFI9CEZUee*4SAN^;;Kh5)h?ki~`*zKQRT1HqK}7$MoUpp7?8rjv zoJXSS&^-M0LZ^S{=9`hMe#;pz>)yuy2*veYfxh}*?*SD3z9PuFLCwchg4#Hk5MtqDCX?P$oZ*;uFZ=)`Wg zX|_bRS7GYn{M2hrRa9e@?;w~{hzp6~B$XSfIS_o6BXQc^o(Tnx0U zQjWe!Ubcy8bBG^!RvBKd;}M7V6O~a2@@17RV`mdOA8Y76zsZ0`8A@wAI9S7_ilN4Dd^HlVdELe>gPSmk98}0xk_LpoW@AA)Bf!r_qbu zE0E4w7Zj-4))bEXz9x0=rCdi(>$=l%BSPt!ylf9U&I4XqKVa~Xzz zE!+CnG*5fywsC1vGaN&+sLW~l`ea(zW7T!}RlktM&~Xcs72`bs8Z|H=E4I)$*voZ3 zgKuR`+GLjnMkW|xRd43Vf?aHT{ZVqJU^M$|577}q!JfY)cN&M~YQvyJhUc$Z(@=pz z_cZF1sJ>S-Nuc<17^w!oK0A<7BBB>otn_-yB{{rPCn%r8s-H}X`{*arm;{#)F`f_Q z#_ZRG0po)lQ+2aFhkiXS`{tAUmOabdEmEP5mhvU4y#HNE~UMG?5@>uo4^FdwW zBw*4dh_SPwbE}k0ivkl+1BH-BujCAOi%|7vNsN>syy*B`&&`m=pP>)!YtJWP=1U4f z_2Z~Y#U=MV(vDDLaQ?{m`C@g>gHAHkN*7`MlFsmkBOy=lz2;@Mq&ht{a(dpizus~l z{i7s96$QKbcYaBH%k?02G))DCmpbp#r-rvjCUsEaKpd_{4*cltwU^0t_OlN8kp$>p z_H6Vuq;lyKWuD`0d#(yUGUeMe50T68=0*XB#mB}klO{zv?@_jl_fnz{p(HGbP_>^ zp?5l(J2Nh7oe=LZ?@p9OIi)M3YRj< z7A&3jYU$~W1!kd8l&eYBE+yz_8)mxYDmgwy(*VY>x$$j~ILjJISi7URFYpE+D%9T) zU`h;ALvSBFTnt!mQjP|qpLnf@FC~gEfE~AQ6+)r)ZbQvYJ~Ayeh+aW0?+VuvEG-~W zpVKfrVex53$JYrCmp1}P4)(F+R0vlMINMkt@K(}uj$pQ2%UfhIUIs6K5)jL!SG)Rt zP~}Q!p07=~e(+U0Lj{S%u?3gpY9nsg zoWy`g?XjtJkjiebvh^xh-u6e9txQzKOAo0A3;=`$ti>@+6b70T^xSxZWbyrIba(iu zumHlpAuWC8!AQBJ>Y+(V^ zXJKGyFGXYB-Vnikpw6$l?K5E?leF%U3u;0d}@*G!PucOyU zf}={X197E&NPsJgvVLg}wl@Z4Gk6g!dqN5lr%VPKrcx`*78{bd!UDcgz?G(aL*r`} z(>0IBy_`osdNr=E*MUc~q^=cBCqP~bl)iibMIu9~Q`_6$A~}VQw48dE2`S9GQ{Eg; zP0^XO1AsEHjq<-9%|FP24Q;PI?0;Q&PT_h+rFZ?fw)s$F*?L9B5h+tE3-I}C`R*An zvH1E?);eQ;{_=i1RAstF06$oA(;s_`275k;!f;O=b07gb*U8XgT}gV?0@*zvcN8e` zX|tuG!aqQp3~hoByZ_6>WnSW8Lp&MzJ9Wk(ri4P=dy~NkX|`YHY&}Q~X=7zfPgnU) z_>2t#oH$q+22#qZ@y-vVVlDlc{IB+l_-heKVltT`jgNH7@STUpt{fzC7tqz+H+#?=NOM5|87a8uo)G^4_3?4=rD@C#DYUtzri-7{lxA&ZC zoR_iBBV%mqJab&$ zPA!g+8xmClQ)gwL&piD8jfB#`_>}BeY-*_AhGlHprdC~2Js)#&&rr|3F6Bhau?~&G z2c1jWKIGZup2I773}<1rlwRVgY+<&5rV9dfGx9;huR}w4xYO8V>W2D>3rz*q^wSe< zw)xWCWb&+KuB@g;*hMa^PrN=-YLHbyr+op%n%cUS;3JX^;MO96WfOfsmwd7A#E1&X7*&8so+(C)CDBk8!YZ(&d1DnQ?{SBDu!s zw;sf>=R$lXK+m9=4h0T!5s8A!Xv>^9^d;0FUrDh^;<6xPajPyiD@8Ohg*zUuefa%H zoXav80=5Ra4<6Svg|dx{nYYZtdva$cb>9=!L9%}GmEyokqt0%`t4XdgH5IWUXE#>9 zSU)pX?uAC=P`ir!xm3H9no0ULD{V*kHd->0TZ`0!ccYFM&R|*y8t6C`YniijPH`K}+#;*@s)i7MyRr3H#vf(6B zUW!d<&na#H{(FJhZ3DH(CM(VO$Is?D6<^1$@o2!mSmq4->U~ z^XHfTG{m1X;=j|WAuRKe)rQ&xm15-Ch@;J9@{|JXRxWCdCw*7+5JTCd{~z3T{Z{sjxJgcfdC z&b+zwXXF3<)^G;#P=7PI%dE~`%x^YNGu!*;vHI^a8F?^eD3sg~vF`G}f1>TReSUYS zlc$q}?HcN_1kiSCpZwYIkDG_i@nP%8KQnV@N|11AnQBli9?BTu9&?D?R-ishHVdN`_7TIJBW8&xNK%2vggl+f80DNMR=Q)@!6uY6}rO9p=D6bPP4(t zK+Nuwik}m%T};6K2MV!&BONkRyikN92eMZ_1GVUOvi1&rZ78p%$e=Hhdu(id_=upGG{gVoP@vDIew)H}5{w^VAADR@O?D zDa@d^U9OE7m+VcXdew8{<5M|Vr2#;GuJl;Iof7kqW3K8CtAGdiRTkT6R5g~`7mklp zS+NfVUrexY;y<>eoNS{kRG6ZnuUl-GQGQk8R27omm^Lw4W-2 zko7X-6aH?5nQH#fVHIynC*Ved3?p)>mPg z2G>VB^3k;;#{C@X7B<)9!$AL4je(19N<~CUsl!Md&ZVCOk{NHJ&`a72ONzK!yQ$cg zXVi+77&{pN#3d61HnhuAKMt=(eb;U>*XrRisVy=KLAeZ-X7kg8(PG?~mgL^4i{}G~ z+GC@c6s*mQE0jz^5veG9S+dVRQ-@$lZot!cTn($lev!CmT!k7u+T@v6n)nLj(myHq zW!D9zN?A<($ej!2(#I>ujjJNAd`)RSK|sE$4>v6)1I5)FD328Jcj+*?S9uMLF!&+hQ*$x)v#>fx?ym390Lf|9lMbQ*P>gM3yRVU0;qN=MHh_B_+nKLQ|-i0d|8R!UB~uaoSD zK@Q@n@3aoz;yto|@vg65b*=}R(y9=-F~DY8j=K4;yURMWagF}VQN~eT?nTMpgjfH$ T*Kcs-&aR*T7k%p!e;NHBue^DH literal 0 HcmV?d00001 diff --git a/docs/index.html b/docs/index.html index f68b0bc..9e73796 100644 --- a/docs/index.html +++ b/docs/index.html @@ -4,7 +4,7 @@ vBilling · Usage-based billing for AI Clouds running vCluster or vMetal - + @@ -283,7 +283,8 @@ max-width: 520px; margin: 0 0 40px; } - .hero-ctas { display: flex; gap: 16px; justify-content: flex-start; } + .hero-ctas { display: flex; flex-wrap: wrap; gap: 12px 16px; justify-content: flex-start; } + .hero-ctas a { white-space: nowrap; } .hero-visual { position: relative; @@ -463,6 +464,9 @@ } /* ── Features grid ── */ + .hero-new { margin-top: 22px; font-size: 0.95rem; line-height: 1.55; color: #505466; max-width: 580px; } + .hero-new a { color: #CC5100; font-weight: 600; text-decoration: none; } + .hero-new a:hover { text-decoration: underline; } .features-grid { display: grid; grid-template-columns: repeat(3, 1fr); @@ -728,13 +732,13 @@ .section-dark .section-title { color: var(--text-reversed); } .section-dark .section-body { color: rgba(255,255,255,0.65); } - .neocloud-grid { + .AI Cloud-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 64px; align-items: center; } - .neocloud-copy .section-body { max-width: 100%; margin-bottom: 32px; } + .AI Cloud-copy .section-body { max-width: 100%; margin-bottom: 32px; } .checklist { list-style: none; display: flex; @@ -889,7 +893,7 @@ .section-title { font-size: 32px; } .steps { grid-template-columns: repeat(2, 1fr); } .features-grid { grid-template-columns: repeat(2, 1fr); } - .neocloud-grid { grid-template-columns: 1fr; gap: 40px; } + .AI Cloud-grid { grid-template-columns: 1fr; gap: 40px; } .main-nav { display: none; } .footer-top { grid-template-columns: 1fr 1fr; gap: 40px; } } @@ -985,11 +989,13 @@ Open source billing controller

Usage-based billing for AI Clouds running vCluster or vMetal

-

Auto-discover Tenant Clusters, meter node capacity, GPU SKUs, and egress, then stream usage events to the billing adapter of your choice.

+

Meter tenant clusters to the second, keep a durable, verifiable ledger, and ship reconcilable usage to Stripe, Metronome, Lago, or your own pipeline.

Get started + Read the docs View on GitHub
+

New in v0.2: Stripe and Metronome, a verifiable ledger, vCluster Private Nodes, and DCGM through your own Prometheus. See it on GKE with shared and private GPUs →

@@ -1046,11 +1052,11 @@

Usage-based billing for AI Clouds running vCluster or vMetal

Adapter-agnostic

Ships to your billing backend

-

vBilling is the pipe, not the billing engine. Meter Tenant Clusters once and route events to whichever billing adapter you run.

+

vBilling is the pipe, not the billing engine. Meter tenant clusters once and route events to whichever billing adapter you run.

+ aria-label="vCluster and vMetal feed vBilling which routes usage events to Lago, Metronome, Stripe, or a webhook"> @@ -1088,8 +1094,7 @@

Ships to your billing backend

- Metronome - COMING SOON + Metronome @@ -1097,7 +1102,7 @@

Ships to your billing backend

 - Custom Adapter + Webhook / Custom
@@ -1123,7 +1128,7 @@

Three steps to tenant billing

Choose Billing Platform

-

Pick the billing backend you already run: Lago, Stripe Meters, Metronome, or your own. vBilling ships as an adapter, not as a replacement.

+

Pick the billing backend you already run: Stripe, Metronome (with Stripe for payments), Lago, or your own pipeline through a signed webhook. Send to several at once.

02 · Configure

@@ -1143,7 +1148,7 @@

Configure Pricing & Packaging

Connect vBilling Adapter

-

Deploy vBilling and point it at your adapter. Tenant Clusters are auto-discovered; usage events start streaming in minutes.

+

Deploy vBilling and point it at your backend. Tenant clusters are auto-discovered, and usage starts flowing from the next closed window.

@@ -1155,57 +1160,99 @@

Connect vBilling Adapter

Features

Everything you need to bill tenants

-

vBilling meters dedicated-node consumption across Kubernetes and streams usage events to the billing adapter you choose.

+

vBilling meters shared and dedicated GPU capacity, commits it to a durable ledger, and delivers it to every billing backend you run, exactly once.

- admin_panel_settings -

Node capacity tracking

-

Meter full dedicated node capacity (CPU, memory, and GPU SKUs) when tenants own bare-metal nodes.

+ inventory_2 +

Durable, verifiable ledger

+

Each window is fsynced to a hash-chained ledger before delivery. Restarts and outages are backfilled: no usage lost, nothing billed twice.

+
+ +
+ hub +

Stripe, Metronome, Lago, webhook

+

Fan out to several destinations at once. Each has its own cursor, so a billing API outage never blocks your data platform feed.

+
+ +
+ balance +

Reconciliation

+

Recompute any day from the ledger and compare it with what Stripe or Metronome recorded, dead letters accounted for.

memory_alt -

GPU type-aware

-

Detects GPU models (H100, A100, T4) from node labels. Price each SKU differently in your billing platform.

+

GPU SKU-aware

+

Detects H100, L40S, A100 and more from node labels. Stripe gets a meter per SKU; Metronome and Lago price on the sku dimension.

- data_usage -

DCGM integration

-

Query NVIDIA DCGM via Prometheus for real GPU utilization metrics, not just allocation counts.

+ grid_view +

Fractional GPUs

+

MIG slices and time-sliced GPUs are metered at their fraction, so shared GPUs can be sold by the slice.

- chart_data -

Network egress

-

Meter outbound network traffic per tenant through Prometheus or CNI metrics for bandwidth-based billing.

+ health_and_safety +

Fair metering

+

No charges for image pulls, provisioning, sleeping clusters, or GPU time on unhealthy nodes. Downtime is recorded as an audit trail.

+
+ +
+ admin_panel_settings +

Dedicated nodes

+

Bill whole bare-metal nodes by SKU or instance type, without double-billing the pods that run on them.

- swap_horiz -

Pluggable adapters

-

Ship usage events to Lago today; Metronome, Stripe Meters, OpenMeter, and custom adapters are on the way.

+ dns +

vCluster Private Nodes

+

GPU machines that join a tenant cluster directly are read through its own API, with a read-only kubeconfig vCluster exports, and billed whole. Auto Nodes and Standalone too.

- tune -

LoadBalancer tracking

-

Track LoadBalancer service hours per Tenant Cluster for infrastructure-level billing.

+ monitoring +

Bring your own Prometheus

+

DCGM GPU utilization from any Prometheus, Thanos or Mimir, MIG instances included, or from vCluster Platform fleet observability.

+
+ +
+ functions +

Billable metrics in PromQL

+

Define your own metrics in PromQL, such as inference tokens or GPU energy in kWh, and bill them like built-in usage.

desktop_cloud -

Spot / on-demand

-

Detect spot instance nodes and apply configurable discounts for accurate cost attribution.

+

Capacity types

+

Every event carries on-demand, spot, preemptible or reserved, so your billing platform applies the discount and quantities stay physical.

+
+ +
+ block +

Billing-state enforcement

+

Stripe dunning and Metronome spend alerts label, annotate, or soft-suspend tenant clusters through an admission policy.

+
+ +
+ input +

Ingest API

+

Push inference tokens, Slurm accounting, or storage usage into the same durable pipeline as collector events.

+
+ +
+ public +

Regions and sovereignty

+

One vBilling per control plane cluster: every event, ledger record and billing state stays in its region.

autorenew

Auto-discovery

-

Watches for Tenant Cluster StatefulSets and Platform API resources. Customers are created in your billing adapter automatically.

+

Finds tenant clusters and vCluster Platform projects. Customers, meters and metrics are created in your backend automatically.

@@ -1218,19 +1265,19 @@

Auto-discovery

Architecture

How the pieces fit together

-

vBilling runs as a controller in your Control Plane Cluster. It watches Tenant Clusters, collects capacity and usage metrics, and streams events to the billing adapter you choose.

+

vBilling runs as a controller in your Control Plane Cluster. It watches tenant clusters, collects capacity and usage metrics, and streams events to the billing adapter you choose.

Control Plane Cluster
─────────────────────────────────────────────
 
-
Tenant Cluster team-alpha · private · 4x A100
-
Tenant Cluster team-beta · private · 2x L40S
-
Tenant Cluster team-gpu · private · 8x H100
+
tenant cluster team-alpha · private · 4x A100
+
tenant cluster team-beta · private · 2x L40S
+
tenant cluster team-gpu · private · 8x H100
 
vBilling Controller
-
Go binary · Deployment in Control Plane Cluster
+
Go binary · StatefulSet with a durable ledger, one per control plane cluster
Choose → Configure → Connect
 
↓ Usage events (HTTP) ↓
@@ -1265,9 +1312,8 @@

Up and running in 3 steps

- + +
@@ -1278,7 +1324,7 @@

Up and running in 3 steps

1

Deploy Lago

-

Spin up the open-source billing engine with Docker Compose. UI at :8080, API at :3000.

+

Spin up the open-source billing engine with Docker Compose. UI at :8081, API at :3000 (vBilling's own dashboard uses :8080).

# Clone vBilling and start Lago
@@ -1295,14 +1341,15 @@ 
Deploy Lago

             
2

             

               
Install vBilling

-              
Deploy the controller via Helm. Point it at your Lago instance with your API key.

+              
Deploy the controller via Helm. Point it at your Lago instance; the API key stays in a Secret.

               

                 
-                
helm upgrade --install vbilling deploy/helm/vbilling \
-  --namespace vbilling-system --create-namespace \
-  --set adapter=lago \
+                
kubectl create namespace vbilling-system
+kubectl -n vbilling-system create secret generic vbilling-lago --from-file=api-key=./lago-api-key
+helm upgrade --install vbilling deploy/helm/vbilling --namespace vbilling-system \
+  --set adapters='{lago}' \
   --set lago.apiURL=http://lago-api:3000 \
-  --set lago.apiKey=YOUR_LAGO_API_KEY

+  --set lago.existingSecret=vbilling-lago
@@ -1330,11 +1377,103 @@

Configure pricing

+ + @@ -1376,7 +1515,7 @@

Default events, zero manual tagging

Storage GB-hoursPVC sizesPer PVC Network egress GBCNI / PrometheusPer tenant LoadBalancer hoursService countPer LB service - Control plane hoursTenant Cluster watch1 per cluster + Control plane hourstenant cluster watch1 per cluster @@ -1386,21 +1525,21 @@

Default events, zero manual tagging

-
+
-
+

For AI Clouds

Built for AI Clouds

If you run an AI cloud on Kubernetes, vBilling gives you the billing pipe, without forcing a billing backend on you.


-

AI Clouds give each customer a Tenant Cluster with dedicated GPU nodes. vBilling detects the hardware, meters node capacity by GPU SKU, and streams events into whichever billing adapter you run.

+

AI Clouds give each customer a tenant cluster with dedicated GPU nodes. vBilling detects the hardware, meters node capacity by GPU SKU, and streams events into whichever billing adapter you run.

    -
  • Detects dedicated H100, A100, L40S nodes per tenant
    • -
  • Meters full node capacity: CPU, memory, and GPUs
    • -
  • GPU SKU-specific events, priced in your billing platform
    • -
  • DCGM integration for real utilization tracking
    • -
  • Adapter-agnostic: keep your existing billing backend
    • +
  • Shared, fractional (MIG, time-sliced) and dedicated GPUs, metered to the second
    • +
  • SKU, region and capacity-type dimensions priced in your billing platform
    • +
  • No charges for provisioning, image pulls, or unhealthy hardware
    • +
  • Durable ledger, backfill and reconciliation: metering held to financial-data standards
    • +
  • Stripe, Metronome, Lago or a webhook: keep your existing billing backend
@@ -1414,21 +1553,21 @@

Built for AI Clouds

──────────────────────────────
Customer signs up
↓
- Platform provisions Tenant Cluster + Platform provisions tenant cluster + private nodes (8x H100)
↓
vBilling auto-discovers detects dedicated nodes
↓
- Streams events every 60s: + Meters every 60s window, per hour: 8 GPU-hours (H100) - 96 CPU-hours - 1 TB memory-hours + 96 CPU core-hours + 1 TiB memory-hours
↓
- Billing adapter aggregates - generates invoice + Stripe / Metronome / Lago rate + and invoice
↓
- Webhook → payments → Paid $ + Payment webhooks → billing state → Paid $
diff --git a/docs/metrics.html b/docs/metrics.html index bf03c56..0293d7c 100644 --- a/docs/metrics.html +++ b/docs/metrics.html @@ -253,7 +253,8 @@ Getting Started Configuration Metrics Reference - Private Nodes + Dedicated and Private Nodes + Example: GKE with GPUs Troubleshooting
@@ -266,354 +267,297 @@

Metrics Reference

-

vBilling tracks 9 billable metrics across CPU, memory, storage, GPU, network, and infrastructure. Each metric is sent to Lago as a usage event with specific properties.

+

Every metric vBilling v0.2 emits, how it is measured, and the dimensions billing backends can price on.

i
-
All metrics are aggregated using Lago's sum_agg aggregation type. Values are multiplied by the collection interval (in hours) to convert from instantaneous readings to hourly units.
+
vBilling meters closed windows (60s by default) aligned to the wall clock. Allocation metrics come from object lifetimes, to the second, so missed windows can be backfilled exactly after a restart. Every event also carries tenant, region, project, resource_id, tenant_cluster, tenant_class and a deterministic id. Schema: usage-event.v1.json.
- -

1. CPU Core-Hours

+

1. GPU Hours

-

vcluster_cpu_core_hours

+

vcluster_gpu_hours

- Shared Mode - Private Node Mode + Shared nodes Dedicated nodes
Source
-
Kubernetes Metrics Server (PodMetrics API)
+
Pod nvidia.com/gpu, amd.com/gpu, nvidia.com/mig-* and nvidia.com/gpu.shared requests, and GPUs allocated through DRA ResourceClaims; node labels (or DRA productName) for the GPU model
Unit
-
Core-hours
-
Formula
-
current_cpu_cores x interval_hours x cost_multiplier
-
Lago Field
-
cpu_core_hours
+
gpu-hours
+
Quantity
+
devices × GPU fraction × seconds running in the window / 3600
-

Measures the actual CPU usage of all pods running in the vCluster namespace. The collector queries the metrics-server PodMetrics API, sums up the CPU usage (in millicores) across all containers, and converts to core-hours by multiplying by the collection interval in hours.

-

In shared mode, only namespace pods are counted. In private node mode, the full CPU capacity of dedicated nodes is also added to the total.

-

The cost_multiplier adjusts for spot vs. on-demand pricing. If 50% of pods run on spot nodes with a 60% discount, the multiplier is 0.70.

- -

Lago event properties

+

Billing starts at the first container start (image pulls and scheduling are free) and stops when the last container finishes. Pods that started or finished mid-window are billed for exactly the seconds they ran.

+

Fractional GPUs: a MIG slice counts as slices/7 of a GPU (A30: /4), e.g. mig-1g.10gb = 1/7. This works for the MIG mixed strategy (nvidia.com/mig-* resources), the single strategy (slices exposed as nvidia.com/gpu, product label ending in -MIG-<profile>) and GKE GPU partitions (cloud.google.com/gke-gpu-partition-size). On time-sliced nodes (nvidia.com/gpu.replicas > 1, or GKE cloud.google.com/gke-max-shared-clients-per-gpu with time-sharing or MPS) a device counts as 1/replicas. A DRA claim counts the devices allocated to it (a MIG device at its profile's fraction), split across the pods it is reserved for. The SKU becomes <model>-mig-<profile> or <model>-timeslice-<n>, always against the base GPU model (the -SHARED and -MIG- label suffixes are dropped).

+

Dedicated and private nodes: the node's GPU capacity is billed in whole GPUs with billing_mode=dedicated_node (control plane cluster nodes reserved for the tenant) or billing_mode=private_node (vCluster Private Nodes, Auto Nodes and Standalone, metered through the tenant cluster's own API): MIG slices and time-slicing replicas are converted back, so 8 GPUs shared 4 ways bill as 8, not 32. Pods on those nodes are not billed again. See Dedicated and Private Nodes.

+

Fair metering: GPU time on a node that is NotReady or carries an UNHEALTHY_NODE_TAINTS taint is emitted as vcluster_gpu_downtime_hours instead. So is a dedicated or private node's GPU time while its GPUs are installed (GPU Feature Discovery's nvidia.com/gpu.count) but not allocatable, for example while the driver rebuilds after a reboot or a MIG change.

+

Dimensions

- + - - - - - + + + + + +
PropertyTypeDescription
KeyMeaning
cpu_core_hoursfloatBillable core-hours (with cost multiplier applied)
raw_cpu_coresfloatInstantaneous CPU usage in cores
cost_multiplierfloatSpot/on-demand cost adjustment factor
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
skuGPU model (or the node's SKU label), plus the fractional profile
gpu_typeNormalized GPU model
gpu_profilefull, mig-<profile>, timeslice-<n>
capacity_typeon-demand, spot, preemptible, reserved
billing_modeshared, dedicated_node or private_node
zoneNode zone
- -

2. Memory GB-Hours

+

2. CPU Core-Hours

-

vcluster_memory_gb_hours

+

vcluster_cpu_core_hours

- Shared Mode - Private Node Mode + Shared nodes Dedicated nodes
Source
-
Kubernetes Metrics Server (PodMetrics API)
+
CPU_MEMORY_BASIS: metrics-server usage, container requests, or the larger of both; node capacity for dedicated nodes
Unit
-
GB-hours
-
Formula
-
current_memory_gb x interval_hours x cost_multiplier
-
Lago Field
-
memory_gb_hours
+
core-hours
+
Quantity
+
cores × seconds in the window / 3600
-

Measures actual memory consumption of all pods in the vCluster namespace. Memory is read from the metrics-server in bytes, converted to GB (bytes / 1024^3), then multiplied by the interval to produce GB-hours.

-

In private node mode, the total memory capacity of dedicated nodes is added on top of shared-namespace pod usage.

- -

Lago event properties

+

usage samples metrics-server for the live window only and is never extrapolated into backfilled windows. requests is computed from pod lifetimes, so it is backfilled exactly. max bills the larger of usage and requests per pod.

+

The tenant cluster's own control plane pods (app=vcluster) are covered by instance hours and excluded unless METER_CONTROL_PLANE=true.

+

Dimensions

- + - - - - + + +
PropertyTypeDescription
KeyMeaning
memory_gb_hoursfloatBillable GB-hours (with cost multiplier)
raw_memory_gbfloatInstantaneous memory usage in GB
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
capacity_typeCapacity type of the node
billing_modeshared, dedicated_node or private_node
namespaceNamespace inside the tenant cluster (with METER_BY_NAMESPACE)
- -

3. Storage GB-Hours

+

3. Memory GiB-Hours

-

vcluster_storage_gb_hours

+

vcluster_memory_gb_hours

- Shared Mode + Shared nodes Dedicated nodes
Source
-
PersistentVolumeClaims (PVC spec.resources.requests)
+
As CPU: usage, requests or max; node capacity for dedicated nodes
Unit
-
GB-hours
-
Formula
-
total_pvc_storage_gb x interval_hours
-
Lago Field
-
storage_gb_hours
+
gib-hours
+
Quantity
+
GiB × seconds / 3600
-

Sums the requested storage size of all bound PVCs in the vCluster namespace. Only PVCs with status.phase == Bound are counted. Storage billing is not affected by spot/on-demand node cost multipliers.

- -

Lago event properties

+

1 GiB = 10243 bytes.

+

Dimensions

- + - - - - + +
PropertyTypeDescription
KeyMeaning
storage_gb_hoursfloatBillable storage GB-hours
raw_storage_gbfloatTotal PVC storage in GB at time of collection
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
capacity_typeCapacity type of the node
billing_modeshared, dedicated_node or private_node
- -

4. GPU Hours

+

4. Dedicated Node Hours

-

vcluster_gpu_hours

+

vcluster_private_node_hours

- Shared Mode - Private Node Mode + Dedicated nodes
Source
-
Pod nvidia.com/gpu resource requests + node labels for GPU type
+
Nodes dedicated to the tenant cluster (see Dedicated nodes)
Unit
-
GPU-hours
-
Formula
-
gpu_count x interval_hours (per GPU type)
-
Lago Field
-
gpu_hours
+
node-hours
+
Quantity
+
seconds the node existed and was healthy in the window / 3600, one event per node
-

Counts the number of nvidia.com/gpu resources requested by running pods, grouped by GPU type. The GPU type is detected from node labels on the node where the pod is scheduled. One event is sent per distinct GPU type.

- -

GPU type detection (label priority order)

-
    -
  1. nvidia.com/gpu.product -- NVIDIA GPU Operator (e.g., "NVIDIA-A100-SXM4-80GB")
    2. -
  2. nvidia.com/gpu.machine -- alternative NVIDIA label
    3. -
  3. accelerator -- GKE generic accelerator label
    4. -
  4. cloud.google.com/gke-accelerator -- GKE-specific (e.g., "nvidia-tesla-t4")
    5. -
  5. k8s.amazonaws.com/accelerator -- EKS (e.g., "nvidia-tesla-a100")
    6. -
  6. node.kubernetes.io/instance-type -- fallback: cloud instance type
    7. -
-

In private node mode, GPU counts are read from the node's status.capacity["nvidia.com/gpu"] field, and GPU types are detected using the same label priority.

- -

Lago event properties

+

Downtime on an unhealthy node is not billed.

+

Dimensions

- + - - - - - - + + + +
PropertyTypeDescription
KeyMeaning
gpu_hoursfloatBillable GPU-hours for this GPU type
gpu_countintNumber of GPUs of this type
gpu_typestringGPU model name (normalized)
billing_modestring"private_node" when from dedicated nodes, absent in shared mode
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
skuSKU label, instance type, or <n>x<GPU model>
capacity_typeCapacity type of the node
nodeNode name (also resource_id)
instance_typenode.kubernetes.io/instance-type
- -

5. GPU Utilization

+

5. Storage GiB-Hours

-

vcluster_gpu_utilization

+

vcluster_storage_gb_hours

- Shared Mode - Requires Prometheus + Shared nodes
Source
-
DCGM Exporter via Prometheus (DCGM_FI_DEV_GPU_UTIL)
+
Bound PVCs synced from the tenant cluster (provisioned capacity, else the request)
Unit
-
Percentage-hours (utilization score)
-
Formula
-
avg_utilization_percent x interval_hours
-
Lago Field
-
gpu_util_score
+
gib-hours
+
Quantity
+
GiB × seconds bound in the window / 3600
-

Queries DCGM GPU utilization from Prometheus for all GPUs in the vCluster namespace. The average utilization percentage is multiplied by the interval to produce a utilization score. Requires PROMETHEUS_URL to be configured and the DCGM Exporter running in the cluster.

-

Also queries DCGM_FI_DEV_FB_USED and DCGM_FI_DEV_FB_FREE for GPU memory usage tracking. This metric is primarily informational and can be used for billing adjustments or overuse surcharges.

- -

Lago event properties

+

The tenant cluster's own data volume is excluded unless METER_CONTROL_PLANE=true.

+

Dimensions

- + - - - - - +
PropertyTypeDescription
KeyMeaning
gpu_util_scorefloatUtilization score (avg% x hours)
avg_utilization_pctfloatAverage GPU utilization percentage (0-100)
gpu_countintNumber of GPUs with utilization data
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
skuStorage class
- -

6. Network Egress

+

6. Tenant Cluster Hours

-

vcluster_network_egress_gb

+

vcluster_instance_hours

- Shared Mode - Requires Prometheus + Shared nodes Dedicated nodes
Source
-
Prometheus container_network_transmit_bytes_total
+
Tenant cluster control plane readiness
Unit
-
GB
-
Formula
-
sum(rate(container_network_transmit_bytes_total{namespace}[5m])) x 300 / 1024^3
-
Lago Field
-
egress_gb
+
hours
+
Quantity
+
seconds ready in the window / 3600
-

Measures outbound (transmit) network traffic for all containers in the vCluster namespace. Uses a 5-minute rate window from Prometheus to estimate bytes per second, then multiplies by 300 seconds. Requires PROMETHEUS_URL to be set.

-

Ingress traffic (container_network_receive_bytes_total) is also collected and tracked internally but not billed by default.

- -

Lago event properties

+

No charge while a tenant cluster is provisioning or asleep (no ready replica).

+

Dimensions

- + - - - +
PropertyTypeDescription
KeyMeaning
egress_gbfloatEgress traffic in GB
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
tenant_classTenant class
-

7. LoadBalancer Hours

vcluster_lb_hours

- Shared Mode + Shared nodes
Source
-
Kubernetes Services with spec.type == LoadBalancer
+
Services of type LoadBalancer with an assigned address
Unit
-
Hours
-
Formula
-
loadbalancer_count x interval_hours
-
Lago Field
-
lb_hours
+
hours
+
Quantity
+
load balancers × seconds / 3600
-

Counts the number of Services of type LoadBalancer in the vCluster namespace. Each LoadBalancer service incurs an hourly charge. This covers the cost of cloud load balancer resources (e.g., AWS NLB/ALB, GCP LB) provisioned by the vCluster.

- -

Lago event properties

- - - - - - - - -
PropertyTypeDescription
lb_hoursfloatTotal LoadBalancer service-hours
lb_countintNumber of LoadBalancer services
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
+

A load balancer that is still pending (no IP) is not billed.

- -

8. Instance Hours

+

8. Network Egress GiB

-

vcluster_instance_hours

+

vcluster_network_egress_gb

- Shared Mode - Private Node Mode + Prometheus / informational
Source
-
Per-vCluster (always sent for every tracked vCluster)
+
Prometheus: increase(container_network_transmit_bytes_total{namespace}) over the window, evaluated at the window end, or your EGRESS_QUERY
Unit
-
Hours
-
Formula
-
interval_hours (flat 1x per vCluster)
-
Lago Field
-
instance_hours
+
gib
+
Quantity
+
bytes / 10243
-

A flat hourly charge for running a vCluster instance. This event is always sent for every tracked vCluster, regardless of resource usage. It represents the base cost of operating a vCluster -- the platform overhead, control plane resources, and management infrastructure.

+

The default counts all transmitted bytes. To bill only internet egress, point EGRESS_QUERY at flow metrics from your CNI or fabric.

+
+
-

Lago event properties

+

9. GPU Utilization (informational)

+
+
+

vcluster_gpu_utilization

+
+ Prometheus / informational +
+
+
+
+
Source
+
DCGM exporter via Prometheus (DCGM_FI_DEV_GPU_UTIL; for GPUs in MIG mode, which DCGM reports per instance, DCGM_FI_PROF_GR_ENGINE_ACTIVE), matched on the workload's namespace or, where Prometheus renames it (kube-prometheus-stack), exported_namespace. GPU_UTIL_QUERY replaces the query for other exporters; none turns it off.
+
Unit
+
util-hours
+
Quantity
+
average utilization % × hours, per GPU model
+
+

Informational: delivered to the ledger, webhooks and exports, never to Stripe or Metronome (Lago keeps it for v0.1 plans).

+

Dimensions

- + - - - +
PropertyTypeDescription
KeyMeaning
instance_hoursfloatHours the vCluster was running
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
gpu_typeGPU model
- -

9. Private Node Hours

+

10. GPU Downtime Hours (informational)

-

vcluster_private_node_hours

+

vcluster_gpu_downtime_hours

- Private Node Mode Only + Prometheus / informational
Source
-
Node labels: vcluster.loft.sh/managed-by, vcluster.loft.sh/cluster
+
Allocated GPU time on NotReady or unhealthy nodes
Unit
-
Node-hours
-
Formula
-
private_node_count x interval_hours
-
Lago Field
-
private_node_hours
+
gpu-hours
+
Quantity
+
as GPU hours
-

Counts the number of dedicated/private nodes allocated to a vCluster tenant. Only sent when private nodes are detected via node labels. This represents the cost of reserving entire physical or virtual machines for a single tenant.

-

For full details on how private nodes are detected and billed, see the Private Nodes guide.

- -

Lago event properties

+

An auditable record of the time tenants were not charged for, never sent to billing backends.

+

Dimensions

- + - - - - - + +
PropertyTypeDescription
KeyMeaning
private_node_hoursfloatTotal node-hours (count x interval)
node_countintNumber of dedicated nodes
billing_modestringAlways "private_node"
vcluster_namestringName of the vCluster
vcluster_namespacestringNamespace of the vCluster
gpu_typeGPU model
nodeNode
+

Custom metrics

+

Declare operator metrics with CUSTOM_METRICS="code:unit[:key|key],..." (for example inference_output_tokens:tokens:model|region) and push closed windows to POST /api/v1/events. Billing backends create meters and billable metrics for them at startup.

diff --git a/docs/private-nodes.html b/docs/private-nodes.html index 6d5380c..5988fdc 100644 --- a/docs/private-nodes.html +++ b/docs/private-nodes.html @@ -3,7 +3,7 @@ - Private Nodes - vBilling Docs + Dedicated and Private Nodes - vBilling Docs @@ -252,7 +252,8 @@ Getting Started Configuration Metrics Reference - Private Nodes + Dedicated and Private Nodes + Example: GKE with GPUs Troubleshooting
@@ -264,338 +265,103 @@
-

Private Node Billing

-

How vBilling detects and bills dedicated bare-metal or virtual machines allocated to individual vCluster tenants. Ideal for neoclouds, GPU clusters, and high-isolation environments.

- - -

What Are Private Nodes?

-

Private nodes are dedicated Kubernetes nodes (physical servers, cloud VMs, or bare-metal machines) allocated exclusively to a single vCluster tenant. Unlike the default shared model where vCluster pods run alongside other workloads on shared nodes, private nodes give a tenant the entire machine.

- -
-
-

Shared Mode (default)

-

vCluster pods run in a namespace on shared nodes. Billing is based on actual pod resource usage (CPU, memory) as reported by metrics-server.

-
-
-

Private Node Mode

-

Entire nodes are dedicated to one tenant. Billing is based on the full node capacity (CPU, memory, GPUs, storage) regardless of actual utilization.

-
-
- -

Common use cases for private nodes:

-
    -
  • GPU workloads -- Tenants get dedicated GPU nodes (e.g., 8x A100 or 8x H100)
    • -
  • Security isolation -- Tenants require physical machine-level isolation
    • -
  • Performance -- No noisy-neighbor effects from shared infrastructure
    • -
  • Bare-metal providers -- Neoclouds selling entire servers to tenants
    • -
- - -

How vBilling Detects Private Nodes

-

vBilling discovers private nodes by searching for Kubernetes nodes with specific labels that associate them with a vCluster. The detection uses multiple strategies in order:

- -

Strategy 1: Direct namespace match

-

vBilling searches for nodes with these label selectors (where the value matches the vCluster namespace):

-
# Label selectors tried for namespace "vcluster-team-gpu": -vcluster.loft.sh/managed-by=vcluster-team-gpu -vcluster.loft.sh/cluster=vcluster-team-gpu
- -

Strategy 2: Name with prefix stripping

-

If the namespace starts with a common prefix (vcluster- or vc-), vBilling also tries the name with the prefix stripped:

-
# For namespace "vcluster-team-gpu", the prefix "vcluster-" is stripped: -# vCluster name derived = "team-gpu" -# Additional selectors tried: -vcluster.loft.sh/managed-by=team-gpu -vcluster.loft.sh/cluster=team-gpu
- -
-
!
-
This prefix stripping is a common source of label mismatch issues. If your namespace is vcluster-team-gpu and you label nodes with vcluster.loft.sh/managed-by=vcluster-team-gpu, it will match via Strategy 1. But if you label with just team-gpu, it matches via Strategy 2. See Troubleshooting for details.
-
- -

Strategy 3: Custom label via environment variable

-

For non-standard setups, you can specify a custom label selector via the VCLUSTER_NODE_LABEL environment variable:

-
# Format: VCLUSTER_NODE_LABEL=label-key=value-pattern -# Use %s as a placeholder for the vCluster name (after prefix stripping) - -# Example: nodes labeled with "tenant=team-gpu" -export VCLUSTER_NODE_LABEL="tenant=%s" - -# Example: nodes labeled with "dedicated-to=team-gpu" -export VCLUSTER_NODE_LABEL="dedicated-to=%s" - -# In Helm values: -# env: -# - name: VCLUSTER_NODE_LABEL -# value: "tenant=%s"
- -

Detection flow diagram

-
-
For each tracked vCluster namespace:
-
-  Namespace = "vcluster-team-gpu"
-      |
-      |--> Try: vcluster.loft.sh/managed-by=vcluster-team-gpu
-      |--> Try: vcluster.loft.sh/cluster=vcluster-team-gpu
-      |
-      |--> Strip prefix "vcluster-" --> Name = "team-gpu"
-      |--> Try: vcluster.loft.sh/managed-by=team-gpu
-      |--> Try: vcluster.loft.sh/cluster=team-gpu
-      |
-      |--> Try: Custom label (VCLUSTER_NODE_LABEL=%s --> "team-gpu")
-      |
-      v
-  Deduplicate matched nodes by name
+

Dedicated and Private Node Billing

+

How vBilling bills nodes that belong to one tenant cluster for their full capacity, whether they sit in the control plane cluster (dedicated nodes) or join the tenant cluster itself (vCluster Private Nodes, Auto Nodes, Standalone), without ever billing the workloads on them twice.

+ +

What dedicated nodes are

+

A dedicated node is a physical server, VM or bare-metal machine reserved for one tenant cluster. AI Clouds typically sell them by SKU (for example bm.gpu.h100.8), billing the whole machine regardless of utilization. Shared nodes, by contrast, are billed by what each pod allocates.

+
+
i
+
Dedicated nodes are control plane cluster nodes reserved through labels. With vCluster Private Nodes, nodes join the tenant cluster directly and the control plane cluster never sees them or their pods; see Private Nodes, Auto Nodes and Standalone.
- -

What Gets Billed

-

When private nodes are detected for a vCluster, the full node capacity is billed to the tenant. The following data is extracted from each node:

- +

How vBilling detects them

+

A node is dedicated to a tenant cluster when any of these labels match. The namespace is the tenant cluster's control plane cluster namespace; the name is the tenant cluster's name.

- - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + +
ResourceSourceLago Metric
LabelMatches
CPU (total cores)node.status.capacity["cpu"]vcluster_cpu_core_hours (billing_mode=private_node)
Memory (total bytes)node.status.capacity["memory"]vcluster_memory_gb_hours (billing_mode=private_node)
GPU count + typenode.status.capacity["nvidia.com/gpu"] + node labelsvcluster_gpu_hours (billing_mode=private_node)
Ephemeral storagenode.status.capacity["ephemeral-storage"]Tracked internally (not a separate Lago metric)
Node countNumber of matched nodesvcluster_private_node_hours
Spot/on-demand statusNode labels (see below)Used to determine IsSpot flag
Instance typenode.kubernetes.io/instance-typeTracked in InstanceType field
vbilling.vcluster.com/tenant-clusterThe tenant cluster ID (vcluster-<namespace>-<name>) or name. Recommended.
vcluster.loft.sh/managed-byNamespace or name (also the name with a vcluster- or vc- namespace prefix stripped)
vcluster.loft.sh/clusterAs above
VCLUSTER_NODE_LABEL patternAny key=value where %s is replaced with the candidate names, e.g. tenant=%s
+

Taint dedicated nodes (for example dedicated=team-b:NoSchedule) and give the tenant's workloads a matching toleration, so other tenants cannot land on them.

+
kubectl label node bm-h100-01 vbilling.vcluster.com/tenant-cluster=vcluster-team-b-team-b +kubectl label node bm-h100-01 vbilling.vcluster.com/sku=bm.gpu.h100.8 vbilling.vcluster.com/capacity-type=reserved +kubectl taint node bm-h100-01 dedicated=team-b:NoSchedule
-

Additionally, vBilling queries metrics-server for the actual CPU and memory usage on private nodes (via node-level metrics) and adds that to the billing totals. This ensures workloads running directly on dedicated nodes outside the vCluster namespace are also captured.

- - -

GPU Type Detection on Private Nodes

-

vBilling checks multiple node labels to determine the GPU model on a private node. Labels are checked in priority order -- the first match wins:

- +

What gets billed

+

Each dedicated node produces its own events every window (resource_id = node name):

- - - + - - - - - - - - - - - - - - - - - - - - + + + +
LabelPlatformExample Value
MetricQuantityDimensions
nvidia.com/gpu.productNVIDIA GPU Operator (any platform)NVIDIA-A100-SXM4-80GB
nvidia.com/gpu.machineAlternative NVIDIA labelNVIDIA-DGX-A100
cloud.google.com/gke-acceleratorGoogle Kubernetes Engine (GKE)nvidia-tesla-t4
k8s.amazonaws.com/acceleratorAmazon Elastic Kubernetes Service (EKS)nvidia-tesla-a100
vcluster_private_node_hoursHours the node was healthysku (SKU label, instance type, or <n>x<GPU model>), capacity_type, instance_type, node
vcluster_gpu_hoursGPU capacity × hoursbilling_mode=dedicated_node, sku, gpu_type
vcluster_cpu_core_hoursCPU capacity × hoursbilling_mode=dedicated_node
vcluster_memory_gb_hoursMemory capacity (GiB) × hoursbilling_mode=dedicated_node
- -

GPU type strings are normalized: spaces are replaced with hyphens and leading/trailing whitespace is trimmed.

- - -

Spot / On-Demand Detection

-

Each private node is checked for spot/preemptible status using the following labels:

- +

Price either the node-hours by SKU, or the capacity components by billing_mode, whichever matches how you sell. Set the other to zero.

+ +

No double billing

+

Pods that run on the tenant's own dedicated nodes are not metered again as shared usage: the node allocation already covers them. (v0.1 billed both the pods and the node.) A pod from a different tenant that lands on an untainted dedicated node is billed to its own tenant as shared usage.

+ +

Downtime

+

If a dedicated node is NotReady, or carries a taint listed in UNHEALTHY_NODE_TAINTS, its node-hours stop and its GPUs are recorded as vcluster_gpu_downtime_hours (informational, never invoiced), from the moment the condition or taint appeared.

+ +

Private Nodes, Auto Nodes and Standalone

+

With Private Nodes, vCluster syncs nothing to the control plane cluster: the nodes, their pods and their volumes exist only in the tenant cluster. vBilling meters these tenant clusters through their own API. Every node the tenant cluster's API reports that the control plane cluster does not have (and that is not one of vCluster's placeholder nodes, labeled vcluster.loft.sh/fake-node) is a private node, billed like a dedicated node with billing_mode=private_node. The tenant cluster's volumes and load balancers are billed as well. Shared tenant clusters are never billed twice, even when vBilling can read their API.

+

vBilling reads one Secret per tenant cluster, vbilling-kubeconfig in its namespace, and its RBAC is pinned to that name. Let vCluster write it with a read-only service account:

+
exportKubeConfig: + additionalSecrets: + - name: vbilling-kubeconfig + serviceAccount: + name: vbilling-reader + namespace: kube-system + clusterRole: vbilling-reader +experimental: + deploy: + vcluster: + manifests: |- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vbilling-reader + rules: + - apiGroups: [""] + resources: [nodes, pods, persistentvolumeclaims, services] + verbs: [get, list, watch] + - apiGroups: [resource.k8s.io] + resources: [resourceclaims, resourceslices] + verbs: [get, list] + - apiGroups: [metrics.k8s.io] + resources: [pods] + verbs: [get, list]
+

vCluster writes kubeconfigs for localhost; vBilling connects to the tenant cluster's Service and verifies TLS against <name>.<namespace>. Tenant clusters outside the control plane cluster (vCluster Standalone, or any other cluster) are listed in TENANT_CLUSTERS_FILE with their own kubeconfig.

- - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + +
LabelSpot ValuesPlatform
SettingEffect
kubernetes.io/lifecycle"spot", "preemptible"Common / generic
node.kubernetes.io/lifecycle"spot", "preemptible"Alternative generic
cloud.google.com/gke-preemptible"true"GKE preemptible VMs
cloud.google.com/gke-spot"true"GKE Spot VMs
eks.amazonaws.com/capacityType"SPOT"Amazon EKS
karpenter.sh/capacity-type"spot"Karpenter autoscaler
PRIVATE_NODE_BILLING=node (default)Each private node is billed whole: node-hours, whole GPUs, CPU and memory capacity.
PRIVATE_NODE_BILLING=usageThe pods on private nodes are billed instead, like shared usage (namespaces in TENANT_EXCLUDE_NAMESPACES, default kube-system, are skipped).
vbilling.vcluster.com/billable=false node labelExempts a node, for example hardware the tenant owns. Works for dedicated nodes too.
TENANT_ADMIN_FALLBACK=trueAlso accepts the tenant cluster's admin kubeconfig (vc-<name>). Requires read access to Secrets; prefer the read-only export.
+

Outages. If a tenant cluster's API cannot be read, everything else is still metered and the gap is recorded under DATA_DIR. As soon as the API answers, exactly the missed windows are metered and appended as late events, deduplicated by ID, up to MAX_BACKFILL.

+

Scale-down. Nodes deleted between two windows (Auto Nodes scaling down, a machine removed) are billed until the second they were deleted. A deletion noticed only after a watch reconnects is dated to the last time vBilling saw the node, never later.

-

The spot/on-demand status is stored in the IsSpot field of each private node. This information can be used for pricing differentiation.

- - -

Example: GKE Test with T4 GPU

-

Here is a real-world example of setting up private node billing on GKE with NVIDIA T4 GPUs.

- -

Label the node

-
# Identify the GPU node -kubectl get nodes -l cloud.google.com/gke-accelerator=nvidia-tesla-t4 - -# NAME STATUS AGE -# gke-cluster-gpu-pool-abc12345-def0 Ready 2d - -# Label the node as belonging to the vCluster -kubectl label node gke-cluster-gpu-pool-abc12345-def0 \ - vcluster.loft.sh/managed-by=team-gpu
- -

Verify detection

-
# Check that the node has the right labels -kubectl get node gke-cluster-gpu-pool-abc12345-def0 --show-labels - -# Labels include: -# cloud.google.com/gke-accelerator=nvidia-tesla-t4 -# vcluster.loft.sh/managed-by=team-gpu -# node.kubernetes.io/instance-type=n1-standard-8 - -# Check vBilling logs (namespace: vcluster-team-gpu) -kubectl logs -n vbilling-system -l app=vbilling -f - -# Expected output: -# [metrics] vcluster-team-gpu: found 1 private node(s): totalCPU=8 cores, totalMemory=30.00 GB, totalGPUs=1 -# [metrics] node=gke-cluster-gpu-pool-abc12345-def0 type=n1-standard-8 cpu=8 mem=30720Mi gpus=1(nvidia-tesla-t4) on-demand -# [metrics] vcluster-team-gpu: added private node usage: CPU=+2.150 cores, Memory=+8.45 GB -# [controller] sent 6 billing events to Lago
- -

Resulting Lago events

-

With a 60-second collection interval (0.01667 hours), the following events are sent:

-
# CPU core-hours from private node (8 cores x 0.01667h) -{ "code": "vcluster_cpu_core_hours", - "properties": { "cpu_core_hours": 0.133333, "raw_cpu_cores": 8.0, - "billing_mode": "private_node" } } - -# Memory GB-hours from private node (30 GB x 0.01667h) -{ "code": "vcluster_memory_gb_hours", - "properties": { "memory_gb_hours": 0.5, "raw_memory_gb": 30.0, - "billing_mode": "private_node" } } - -# GPU hours (1 T4 x 0.01667h) -{ "code": "vcluster_gpu_hours", - "properties": { "gpu_hours": 0.016667, "gpu_count": 1, - "gpu_type": "nvidia-tesla-t4", "billing_mode": "private_node" } } - -# Private node hours (1 node x 0.01667h) -{ "code": "vcluster_private_node_hours", - "properties": { "private_node_hours": 0.016667, "node_count": 1, - "billing_mode": "private_node" } }
- - -

How to Set Up Private Nodes

-

Follow these steps to enable private node billing for a vCluster:

- -

1. Label your nodes

-
# Option A: Use the standard label -kubectl label node <node-name> \ - vcluster.loft.sh/managed-by=<vcluster-name> - -# Option B: Use the cluster label -kubectl label node <node-name> \ - vcluster.loft.sh/cluster=<vcluster-name> - -# For multiple nodes: -kubectl label node node-1 node-2 node-3 \ - vcluster.loft.sh/managed-by=team-gpu
- -

2. Ensure the label value matches

-
-
!
-
- The label value must match what vBilling expects. vBilling tries several values: -
    -
  • The full namespace name (e.g., vcluster-team-gpu)
    • -
  • The namespace with vcluster- or vc- prefix stripped (e.g., team-gpu)
    • -
  • The custom label pattern from VCLUSTER_NODE_LABEL
    • -
-
-
- -

3. Verify with kubectl

-
# Check which nodes match the label -kubectl get nodes -l vcluster.loft.sh/managed-by=team-gpu - -# NAME STATUS ROLES AGE VERSION -# gpu-node-001 Ready <none> 5d v1.29.2 -# gpu-node-002 Ready <none> 5d v1.29.2 - -# Inspect node details -kubectl describe node gpu-node-001 | grep -A5 "Capacity:" - -# Capacity: -# cpu: 96 -# ephemeral-storage: 1844592Ki -# hugepages-1Gi: 0 -# memory: 1056718848Ki -# nvidia.com/gpu: 8
- -

4. Check vBilling logs

-
kubectl logs -n vbilling-system -l app=vbilling | grep "private node" - -# [metrics] vcluster-team-gpu: found 2 private node(s): totalCPU=192 cores, totalMemory=2013.12 GB, totalGPUs=16 -# [metrics] node=gpu-node-001 type=p4d.24xlarge cpu=96 mem=1006559Mi gpus=8(NVIDIA-A100-SXM4-80GB) on-demand -# [metrics] node=gpu-node-002 type=p4d.24xlarge cpu=96 mem=1006559Mi gpus=8(NVIDIA-A100-SXM4-80GB) on-demand
- -
-
✓
-
Private node billing happens automatically alongside shared-namespace billing. Both shared and private metrics are sent to Lago simultaneously. The billing_mode=private_node property lets you distinguish them in Lago.
-
+

Whole GPUs, however they are carved

+

Dedicated and private nodes are billed for physical GPUs. MIG slices (mixed or single strategy, GKE GPU partitions) and time-slicing replicas (NVIDIA or GKE time-sharing) are converted back: a node exposing 56 MIG slices of 1g.10gb, or 32 time-sliced devices with 4 replicas, bills 8 GPUs. GPUs offered only through DRA ResourceSlices count too.

+

GPU model detection

+

The first label present wins: nvidia.com/gpu.product, nvidia.com/gpu.machine, amd.com/gpu.product-name, cloud.google.com/gke-accelerator, k8s.amazonaws.com/accelerator, accelerator. Spaces are replaced with dashes (NVIDIA H100 80GB HBM3 becomes NVIDIA-H100-80GB-HBM3). The -SHARED and -MIG-<profile> suffixes GPU Feature Discovery adds are removed, so the SKU is always the base GPU; DRA devices use their productName attribute.

diff --git a/docs/schema/usage-event.v1.json b/docs/schema/usage-event.v1.json new file mode 100644 index 0000000..ea7381a --- /dev/null +++ b/docs/schema/usage-event.v1.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/vClusterLabs-Experiments/vbilling/blob/main/docs/schema/usage-event.v1.json", + "title": "vBilling usage event", + "description": "One metered quantity for one tenant over one closed window. Versioning policy: schema_version changes only on breaking changes (removed or retyped fields, changed ID derivation). New optional fields and dimensions are additive and do not bump it. Consumers must ignore unknown fields.", + "type": "object", + "required": ["schema_version", "id", "tenant", "metric", "quantity", "unit", "window_start", "window_end", "recorded_at"], + "properties": { + "schema_version": {"const": "vbilling.usage/v1"}, + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9._:\\-]{1,100}$", + "description": "Stable identifier. For collector events: \"vb1_\" + the first 128 bits (hex) of SHA-256 over schema_version, tenant, metric, window_start, window_end (RFC 3339, UTC), region, project, sku, resource_id and the sorted dimensions, NUL-separated. Quantity and properties are excluded, so re-measuring a window yields the same ID. Ingested events either supply their own ID or receive the derived one. Deduplicate on this field." + }, + "tenant": {"type": "string", "minLength": 1, "maxLength": 128, "description": "Billing customer ID."}, + "metric": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,99}$", "description": "Metric code, see GET /api/v1/catalog."}, + "quantity": {"type": "number", "minimum": 0, "description": "Amount of the metric in unit, accumulated over the window. Rounded to 9 decimal places."}, + "unit": {"type": "string", "examples": ["gpu-hours", "core-hours", "gib-hours", "node-hours", "hours", "gib", "tokens"]}, + "window_start": {"type": "string", "format": "date-time", "description": "Inclusive start, UTC. Collector windows are aligned to multiples of the window size since the Unix epoch."}, + "window_end": {"type": "string", "format": "date-time", "description": "Exclusive end, UTC."}, + "region": {"type": "string", "description": "Region of the control plane cluster that metered the event (REGION)."}, + "project": {"type": "string", "description": "Cost-attribution project (vCluster Platform project or the vbilling.vcluster.com/project annotation)."}, + "sku": {"type": "string", "description": "Billable SKU: GPU model (+ \"-mig-\" / \"-timeslice-\"), node SKU or instance type, or storage class."}, + "resource_id": {"type": "string", "description": "Tenant cluster ID for cluster-level metrics, node name for per-node metrics."}, + "dimensions": { + "type": "object", + "additionalProperties": {"type": "string"}, + "description": "Identity-bearing attributes for grouping and pricing.", + "properties": { + "tenant_cluster": {"type": "string"}, + "gpu_type": {"type": "string"}, + "gpu_profile": {"type": "string", "examples": ["full", "mig-1g.10gb", "timeslice-4"]}, + "capacity_type": {"enum": ["on-demand", "spot", "preemptible", "reserved"]}, + "billing_mode": {"enum": ["shared", "dedicated_node", "private_node"]}, + "tenant_class": {"type": "string"}, + "zone": {"type": "string"}, + "namespace": {"type": "string", "description": "Namespace inside the tenant cluster (METER_BY_NAMESPACE)."}, + "node": {"type": "string"}, + "instance_type": {"type": "string"} + } + }, + "properties": {"type": "object", "description": "Informational context (device counts, basis, backfilled flag). Never part of the identity."}, + "source": {"type": "string", "description": "\"collector\", or \"ingest:\" for events pushed through POST /api/v1/events."}, + "recorded_at": {"type": "string", "format": "date-time"} + } +} diff --git a/docs/troubleshooting.html b/docs/troubleshooting.html index 9868b16..7eeee02 100644 --- a/docs/troubleshooting.html +++ b/docs/troubleshooting.html @@ -227,7 +227,8 @@ Getting Started Configuration Metrics Reference - Private Nodes + Dedicated and Private Nodes + Example: GKE with GPUs Troubleshooting
@@ -242,24 +243,24 @@

Troubleshooting

Common issues you might encounter when running vBilling, with step-by-step solutions.

- -

No vClusters Discovered

+ +

No Tenant Clusters Discovered

"[discovery] found 0 vCluster(s)"

Symptoms

-

vBilling starts but reports zero vClusters. No customers or events appear in Lago. The controller log shows [discovery] found 0 vCluster(s) on every reconcile cycle.

+

vBilling starts but reports zero tenant clusters. No customers or events appear in Lago. The controller log shows [discovery] found 0 vCluster(s) on every reconcile cycle.

Cause

-

vBilling discovers vClusters by scanning for StatefulSets and Deployments with the label app=vcluster. If your vClusters use different labels, or if RBAC prevents listing these resources, discovery will find nothing.

+

vBilling discovers tenant clusters by scanning for StatefulSets and Deployments with the label app=vcluster. If your tenant clusters use different labels, or if RBAC prevents listing these resources, discovery will find nothing.

-

Fix Check that vClusters have the expected labels

+

Fix Check that tenant clusters have the expected labels

# List StatefulSets with the vcluster label kubectl get statefulsets --all-namespaces -l app=vcluster -# Should list your vClusters, e.g.: +# Should list your tenant clusters, e.g.: # NAMESPACE NAME READY AGE # vcluster-team-alpha my-vcluster 1/1 3d @@ -296,7 +297,7 @@

"[metrics] warning: cannot get pod metrics for <namespace>"

Symptoms

-

vBilling discovers vClusters but CPU and memory metrics are always zero. The log shows warnings about failing to get pod metrics.

+

vBilling discovers tenant clusters but CPU and memory metrics are always zero. The log shows warnings about failing to get pod metrics.

Cause

vBilling requires the Kubernetes metrics-server to be installed and functioning. Without it, the PodMetrics API is unavailable.

@@ -363,14 +364,14 @@

Fix Manually label your GPU nodes

-

Private Nodes Not Detected

+

Dedicated Nodes Not Detected

-

Private nodes labeled but not appearing in billing events

+

Dedicated nodes labeled but not appearing in billing events

Symptoms

-

You have labeled nodes with vcluster.loft.sh/managed-by but vBilling does not report any private nodes. No vcluster_private_node_hours events appear in Lago.

+

You have labeled control plane cluster nodes with vcluster.loft.sh/managed-by but vBilling does not bill them as dedicated nodes. No vcluster_private_node_hours events with billing_mode=dedicated_node appear. (For nodes that join a tenant cluster through vCluster Private Nodes, see Private Nodes Not Metered.)

Cause

The most common cause is a label value mismatch between the node label and the vCluster namespace name. vBilling tries multiple label values, but they must correspond to the namespace.

@@ -409,6 +410,49 @@

Fix Test the exact selector vBilling uses

+ +

Private Nodes Not Metered

+
+
+

Nodes joined with vCluster Private Nodes produce no events

+
+
+

Symptoms

+

A tenant cluster has private nodes, but no vcluster_private_node_hours events with billing_mode=private_node appear, or GET /api/v1/status lists the tenant cluster under failed tenant APIs.

+ +

Cause

+

vBilling reads private nodes through the tenant cluster's own API, with the kubeconfig in the Secret vbilling-kubeconfig in the tenant cluster's namespace. Without that Secret, or with the tenant cluster's control plane down, there is nothing to read.

+ +

Fix Export the read-only kubeconfig

+

Add exportKubeConfig.additionalSecrets with the vbilling-reader service account to the tenant cluster's vcluster.yaml (see Private Nodes), then check the Secret exists:

+
kubectl -n <tenant-namespace> get secret vbilling-kubeconfig
+ +

Fix The tenant cluster's control plane exits

+

you are trying to use a vCluster pro feature 'Private Nodes' (vcp-distro-private-nodes) that is not available means the tenant cluster has no license: register it with vCluster Platform (vcluster platform add vcluster <name> -n <namespace>). denied by loft access key scope means the CLI's access key is scoped and cannot register tenant clusters; log in with a key that can. vCluster 0.36.2 and later need Platform 4.11.3 or later.

+ +

Fix GPU-hours missing right after a node reboot

+

While a driver container rebuilds (after a reboot, a spot restart or a MIG change), a Ready node advertises no GPUs. vBilling bills the node and records its installed GPUs (GPU Feature Discovery's nvidia.com/gpu.count) as vcluster_gpu_downtime_hours with the reason "GPUs installed but not allocatable" until they are allocatable again.

+
+
+ +

GPU Utilization or Energy Missing

+
+
+

No vcluster_gpu_utilization events, or a DCGM metric reads zero or too high

+
+
+

Cause

+

DCGM behaves differently from what a query might assume:

+
    +
  • For GPUs in MIG mode DCGM reports no DCGM_FI_DEV_GPU_UTIL. The default query falls back to each instance's DCGM_FI_PROF_GR_ENGINE_ACTIVE; a custom GPU_UTIL_QUERY needs the same fallback.
    • +
  • Device-level fields (power, energy, framebuffer) repeat on every MIG instance: summing them bills the GPU once per slice. Keep one series per GPU with max by (modelName, UUID, ...) first.
    • +
  • The exporter refreshes every 30 seconds by default (DCGM_EXPORTER_INTERVAL), so increase() over a 30 second window can read zero. Integrate a gauge instead: avg_over_time(...[{{window}}]) * {{window_seconds}}.
    • +
  • DCGM on private nodes runs inside the tenant cluster. Ship it to your Prometheus with an agent that adds an external label naming the tenant cluster, and select on it with {{vcluster}}.
    • +
+

The GKE example has working queries for all four cases.

+
+
+

Lago Connection Failed

@@ -524,7 +568,7 @@

Browser console shows CORS errors when accessing Lago UI or vBilling dashboa

Symptoms

-

When accessing the Lago UI or any vBilling dashboard from a browser, you see errors like: Access to fetch at 'http://localhost:3000/...' from origin 'http://localhost:8080' has been blocked by CORS policy.

+

When accessing the Lago UI or any vBilling dashboard from a browser, you see errors like: Access to fetch at 'http://localhost:3000/...' from origin 'http://localhost:8081' has been blocked by CORS policy.

Cause

The browser enforces Cross-Origin Resource Sharing (CORS) restrictions when the frontend and API are served from different origins (different ports or domains).

diff --git a/examples/gke-gpu/01-prometheus.yaml b/examples/gke-gpu/01-prometheus.yaml new file mode 100644 index 0000000..aebf877 --- /dev/null +++ b/examples/gke-gpu/01-prometheus.yaml @@ -0,0 +1,95 @@ +# Central Prometheus for the control plane cluster ("bring your own Prometheus"). +# - Scrapes GKE's managed DCGM exporter on the shared GPU nodes. Its target +# labels win, so DCGM's namespace/pod labels become exported_namespace/exported_pod. +# - Accepts remote writes from tenant clusters with private nodes, whose DCGM +# never reaches this cluster otherwise (see 06-prometheus-agent.yaml). +apiVersion: v1 +kind: Namespace +metadata: {name: monitoring} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: prometheus, namespace: monitoring} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: prometheus-sd} +rules: + - apiGroups: [""] + resources: [pods, nodes, endpoints, services] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: prometheus-sd} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: prometheus-sd} +subjects: [{kind: ServiceAccount, name: prometheus, namespace: monitoring}] +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: prometheus, namespace: monitoring} +data: + prometheus.yml: | + global: + scrape_interval: 10s + scrape_configs: + - job_name: dcgm-exporter + kubernetes_sd_configs: [{role: pod}] + relabel_configs: + # GKE's managed exporter (GKE 1.32+ deploys it on GPU nodes) or NVIDIA's. + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name, __meta_kubernetes_pod_label_app] + regex: (gke-managed-dcgm-exporter;.*|.*;(nvidia-)?dcgm-exporter) + action: keep + - {source_labels: [__meta_kubernetes_pod_container_port_number], regex: "9400", action: keep} + - {source_labels: [__meta_kubernetes_namespace], target_label: namespace} + - {source_labels: [__meta_kubernetes_pod_name], target_label: pod} + - {source_labels: [__meta_kubernetes_pod_node_name], target_label: node} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: prometheus, namespace: monitoring} +spec: + replicas: 1 + selector: {matchLabels: {app: prometheus}} + template: + metadata: {labels: {app: prometheus}} + spec: + serviceAccountName: prometheus + securityContext: {runAsUser: 65534, runAsNonRoot: true, fsGroup: 65534} + containers: + - name: prometheus + image: prom/prometheus:v3.13.4 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=2d + - --web.enable-remote-write-receiver + ports: [{containerPort: 9090, name: http}] + resources: {requests: {cpu: 200m, memory: 512Mi}} + volumeMounts: + - {name: config, mountPath: /etc/prometheus} + - {name: data, mountPath: /prometheus} + volumes: + - {name: config, configMap: {name: prometheus}} + - {name: data, emptyDir: {}} # use a PersistentVolumeClaim beyond a demo +--- +apiVersion: v1 +kind: Service +metadata: {name: prometheus, namespace: monitoring} +spec: + selector: {app: prometheus} + ports: [{name: http, port: 9090, targetPort: 9090}] +--- +# Remote-write endpoint for private nodes: an internal load balancer, reachable +# only from the VPC. Restrict the source range to your nodes' subnet. +apiVersion: v1 +kind: Service +metadata: + name: prometheus-ingest + namespace: monitoring + annotations: {networking.gke.io/load-balancer-type: Internal} +spec: + type: LoadBalancer + loadBalancerSourceRanges: [10.128.0.0/20] # the default us-central1 subnet; change to yours + selector: {app: prometheus} + ports: [{name: http, port: 9090, targetPort: 9090}] diff --git a/examples/gke-gpu/02-tenant-shared.yaml b/examples/gke-gpu/02-tenant-shared.yaml new file mode 100644 index 0000000..77b3171 --- /dev/null +++ b/examples/gke-gpu/02-tenant-shared.yaml @@ -0,0 +1,7 @@ +# vcluster.yaml for a tenant cluster on shared nodes: its pods sync to the +# control plane cluster and run on its (time-sliced) GPU nodes, where vBilling +# meters them. No extra configuration is needed for vBilling. +controlPlane: + statefulSet: + resources: + requests: {cpu: 200m, memory: 256Mi} diff --git a/examples/gke-gpu/03-tenant-private.yaml b/examples/gke-gpu/03-tenant-private.yaml new file mode 100644 index 0000000..4263851 --- /dev/null +++ b/examples/gke-gpu/03-tenant-private.yaml @@ -0,0 +1,46 @@ +# vcluster.yaml for a tenant cluster with private nodes. GPU machines join this +# tenant cluster directly; nothing is synced to the control plane cluster, so +# vBilling reads the tenant cluster's own API with a read-only kubeconfig. +# Private Nodes needs a vCluster Platform license: register the tenant cluster +# with the Platform (vcluster create adds it when the CLI is logged in). +privateNodes: + enabled: true +controlPlane: + service: + annotations: + networking.gke.io/load-balancer-type: Internal # VPC-only join endpoint + spec: + type: LoadBalancer +networking: + # Must not overlap the VPC (GCP's default subnets use 10.128.0.0/9). + podCIDR: 172.20.0.0/16 + serviceCIDR: 172.21.0.0/16 + +# vBilling: export a kubeconfig for a read-only service account into the +# Secret vbilling-kubeconfig in this tenant cluster's control plane cluster +# namespace. vBilling's RBAC can read only that Secret name. +exportKubeConfig: + additionalSecrets: + - name: vbilling-kubeconfig + serviceAccount: + name: vbilling-reader + namespace: kube-system + clusterRole: vbilling-reader +experimental: + deploy: + vcluster: + manifests: |- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vbilling-reader + rules: + - apiGroups: [""] + resources: [nodes, pods, persistentvolumeclaims, services] + verbs: [get, list, watch] + - apiGroups: [resource.k8s.io] + resources: [resourceclaims, resourceslices] + verbs: [get, list] + - apiGroups: [metrics.k8s.io] + resources: [pods] + verbs: [get, list] diff --git a/examples/gke-gpu/04-node-startup.sh b/examples/gke-gpu/04-node-startup.sh new file mode 100644 index 0000000..198eb44 --- /dev/null +++ b/examples/gke-gpu/04-node-startup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Startup script that joins a VM to a tenant cluster as a vCluster private node, +# once. Replace JOIN_COMMAND with the line `vcluster token create` prints, e.g. +# curl -fsSLk "https:///node/join?token=" | sh - +# The token lets machines join the tenant cluster until it expires: remove this +# script from the instance metadata after the node joined, and never commit it +# with a token in it. +set -euo pipefail +exec >> /var/log/vcluster-join.log 2>&1 +[ -f /var/lib/vcluster-joined ] && { echo "already joined"; exit 0; } +echo "join started $(date -u +%FT%TZ)" + +JOIN_COMMAND + +touch /var/lib/vcluster-joined +echo "join finished $(date -u +%FT%TZ)" diff --git a/examples/gke-gpu/05-gpu-operator-values.yaml b/examples/gke-gpu/05-gpu-operator-values.yaml new file mode 100644 index 0000000..0754cd9 --- /dev/null +++ b/examples/gke-gpu/05-gpu-operator-values.yaml @@ -0,0 +1,9 @@ +# NVIDIA GPU Operator inside the tenant cluster, on its private GPU nodes. +# It installs the driver, container toolkit, device plugin, GPU Feature +# Discovery (the nvidia.com/gpu.* labels vBilling reads), DCGM and the MIG manager. +driver: {enabled: true} +toolkit: {enabled: true} +mig: {strategy: single} +dcgmExporter: + enabled: true + serviceMonitor: {enabled: false} # scraped by 06-prometheus-agent.yaml instead diff --git a/examples/gke-gpu/06-prometheus-agent.yaml b/examples/gke-gpu/06-prometheus-agent.yaml new file mode 100644 index 0000000..4de04e5 --- /dev/null +++ b/examples/gke-gpu/06-prometheus-agent.yaml @@ -0,0 +1,69 @@ +# Prometheus in agent mode inside the tenant cluster with private nodes. It +# scrapes DCGM on the private nodes and remote-writes to the central +# Prometheus, labelling every series with the tenant cluster, so vBilling can +# select them with {{vcluster}}. Apply it inside the tenant cluster. +# Replace PROMETHEUS_INGEST_IP with the IP of the prometheus-ingest Service. +apiVersion: v1 +kind: Namespace +metadata: {name: monitoring} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: prometheus-agent, namespace: monitoring} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: prometheus-agent} +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: prometheus-agent} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: prometheus-agent} +subjects: [{kind: ServiceAccount, name: prometheus-agent, namespace: monitoring}] +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: prometheus-agent, namespace: monitoring} +data: + prometheus.yml: | + global: + scrape_interval: 10s + external_labels: {vcluster: private-a100} # the tenant cluster's name + scrape_configs: + - job_name: dcgm-exporter + kubernetes_sd_configs: [{role: pod, namespaces: {names: [gpu-operator]}}] + relabel_configs: + - {source_labels: [__meta_kubernetes_pod_label_app], regex: nvidia-dcgm-exporter, action: keep} + - {source_labels: [__meta_kubernetes_pod_container_port_number], regex: "9400", action: keep} + - {source_labels: [__meta_kubernetes_namespace], target_label: namespace} + - {source_labels: [__meta_kubernetes_pod_name], target_label: pod} + - {source_labels: [__meta_kubernetes_pod_node_name], target_label: node} + remote_write: + - url: http://PROMETHEUS_INGEST_IP:9090/api/v1/write +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: prometheus-agent, namespace: monitoring} +spec: + replicas: 1 + selector: {matchLabels: {app: prometheus-agent}} + template: + metadata: {labels: {app: prometheus-agent}} + spec: + serviceAccountName: prometheus-agent + securityContext: {runAsUser: 65534, runAsNonRoot: true, fsGroup: 65534} + containers: + - name: prometheus + image: prom/prometheus:v3.13.4 + args: [--agent, --config.file=/etc/prometheus/prometheus.yml, --storage.agent.path=/data] + resources: {requests: {cpu: 100m, memory: 256Mi}} + volumeMounts: + - {name: config, mountPath: /etc/prometheus} + - {name: data, mountPath: /data} + volumes: + - {name: config, configMap: {name: prometheus-agent}} + - {name: data, emptyDir: {}} diff --git a/examples/gke-gpu/07-vbilling-values.yaml b/examples/gke-gpu/07-vbilling-values.yaml new file mode 100644 index 0000000..43c3606 --- /dev/null +++ b/examples/gke-gpu/07-vbilling-values.yaml @@ -0,0 +1,41 @@ +# vBilling on the GKE control plane cluster, delivering to Stripe test mode. +# The Stripe key lives only in the Secret vbilling-stripe (key api-key); this +# file holds no credentials. +adapters: [stripe] +region: us-central1 +clusterName: gke-us-central1 +billing: + defaultPlanCode: gpu-cloud # tenants subscribe to prices tagged metadata[vbilling_plan]=gpu-cloud + currency: USD + collectionInterval: 30s # 60s in production keeps the DCGM math steadier + maxBackfill: 2h +metering: + cpuMemoryBasis: requests +tenantAPI: + enabled: true # private nodes, through each tenant cluster's vbilling-kubeconfig +prometheus: + url: http://prometheus.monitoring.svc:9090 + # The defaults (namespace or exported_namespace, MIG instances included), + # plus tenant clusters with private nodes, whose Prometheus agent + # remote-writes with the external label vcluster=. + gpuUtilQuery: 'avg by (modelName) (avg_over_time(DCGM_FI_DEV_GPU_UTIL{exported_namespace="{{namespace}}"}[{{window}}]) or avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace="{{namespace}}"}[{{window}}]) or avg_over_time(DCGM_FI_DEV_GPU_UTIL{vcluster="{{vcluster}}"}[{{window}}]) or 100 * avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{exported_namespace="{{namespace}}",GPU_I_ID!=""}[{{window}}]) or 100 * avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace="{{namespace}}",GPU_I_ID!=""}[{{window}}]) or 100 * avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{vcluster="{{vcluster}}",GPU_I_ID!=""}[{{window}}]))' + gpuCountQuery: 'count by (modelName) (group by (modelName, UUID, gpu, Hostname, hostname) (DCGM_FI_DEV_GPU_UTIL{exported_namespace="{{namespace}}"} or DCGM_FI_DEV_GPU_UTIL{namespace="{{namespace}}"} or DCGM_FI_DEV_GPU_UTIL{vcluster="{{vcluster}}"} or DCGM_FI_PROF_GR_ENGINE_ACTIVE{exported_namespace="{{namespace}}",GPU_I_ID!=""} or DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace="{{namespace}}",GPU_I_ID!=""} or DCGM_FI_PROF_GR_ENGINE_ACTIVE{vcluster="{{vcluster}}",GPU_I_ID!=""}))' + metrics: + # A billable metric defined in PromQL: energy drawn by a tenant cluster's + # private GPUs. Power is integrated over the window because the DCGM + # exporter refreshes every 30s, and device-level fields repeat on every + # MIG instance, so one series per GPU (UUID) is kept before summing. + - code: gpu_energy_kwh + name: GPU energy + unit: kWh + description: Energy drawn by the tenant cluster's private GPUs, from DCGM + billable: true + dimensions: [modelName] + sku: modelName + query: sum by (modelName) (max by (modelName, UUID, gpu, Hostname, hostname) (avg_over_time(DCGM_FI_DEV_POWER_USAGE{vcluster="{{vcluster}}"}[{{window}}]))) * {{window_seconds}} / 3.6e6 +stripe: + existingSecret: vbilling-stripe + autoSubscribe: true +persistence: + enabled: true + size: 5Gi diff --git a/examples/gke-gpu/README.md b/examples/gke-gpu/README.md new file mode 100644 index 0000000..e65b481 --- /dev/null +++ b/examples/gke-gpu/README.md @@ -0,0 +1,161 @@ +# Example: GKE with shared and private GPU nodes, billed in Stripe + +A complete setup with both vCluster tenancy models on GPUs, metered by vBilling and invoiced by Stripe test mode: + +- **GKE** as the control plane cluster, with a spot L4 node pool time-shared four ways. +- **`shared-l4`**, a tenant cluster on shared nodes: its pods run on the L4 and are billed per pod, a quarter GPU each. +- **`private-a100`**, a tenant cluster with private nodes: an A100 VM joins it directly and is billed whole, read through the tenant cluster's own API with a read-only kubeconfig. +- **Prometheus** reading DCGM: GKE's exporter on shared nodes, and a Prometheus agent that remote-writes from the private node. +- **vBilling** sending usage to Stripe: customers, one meter per metric and SKU, subscriptions, invoices. + +The walkthrough is also on the docs site: [Example: GKE with GPUs](https://vclusterlabs-experiments.github.io/vbilling/example-gke.html). + +| File | What it is | +|---|---| +| `01-prometheus.yaml` | Central Prometheus: scrapes GKE's DCGM exporter, accepts remote writes | +| `02-tenant-shared.yaml` | `vcluster.yaml` for the shared-nodes tenant cluster | +| `03-tenant-private.yaml` | `vcluster.yaml` with Private Nodes and the read-only `vbilling-kubeconfig` export | +| `04-node-startup.sh` | VM startup script that joins the GPU machine as a private node | +| `05-gpu-operator-values.yaml` | NVIDIA GPU Operator inside the private tenant cluster | +| `06-prometheus-agent.yaml` | Prometheus agent in the private tenant cluster, remote-writing DCGM | +| `07-vbilling-values.yaml` | vBilling Helm values: Stripe, DCGM queries, a PromQL billable metric | +| `workloads/` | Sample GPU workloads for each tenant cluster | + +No file here contains a key, token or account ID. Keep yours out of the repository too: the Stripe key goes into a Kubernetes Secret from a local file, and the node join token only into the VM's metadata, removed after the join. + +## Prerequisites + +- `gcloud` with a project selected, `kubectl`, `helm`, and the `vcluster` CLI (0.36 or later). +- **vCluster Platform** for the Private Nodes license. The CLI must be logged in with a key that can register tenant clusters. A scoped key fails with `denied by loft access key scope`. vCluster 0.36.2 and later need Platform 4.11.3 or later. +- A **Stripe test-mode** secret key (`sk_test_...`) saved in a local file, for example `~/.stripe-test-key`. +- GPU quota in the zone: one L4 and one A100 (spot capacity is enough). + +```bash +export ZONE=us-central1-a CLUSTER=vbilling-demo +git clone https://github.com/vClusterLabs-Experiments/vbilling && cd vbilling/examples/gke-gpu +``` + +## 1. Control plane cluster + +```bash +gcloud container clusters create $CLUSTER --zone $ZONE --release-channel regular \ + --machine-type e2-standard-4 --num-nodes 2 +gcloud container node-pools create l4-spot --cluster $CLUSTER --zone $ZONE \ + --machine-type g2-standard-8 --spot --num-nodes 1 \ + --accelerator type=nvidia-l4,count=1,gpu-driver-version=default,gpu-sharing-strategy=time-sharing,max-shared-clients-per-gpu=4 \ + --node-labels vbilling.vcluster.com/capacity-type=spot +gcloud container clusters get-credentials $CLUSTER --zone $ZONE +``` + +## 2. Prometheus + +```bash +kubectl apply -f 01-prometheus.yaml +INGEST_IP=$(kubectl -n monitoring get svc prometheus-ingest -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +``` + +## 3. Tenant clusters + +```bash +vcluster platform login https:// # browser login; no key on the command line +vcluster create shared-l4 -n shared-l4 -f 02-tenant-shared.yaml --connect=false +vcluster create private-a100 -n private-a100 -f 03-tenant-private.yaml --connect=false +``` + +When the CLI is logged in, `vcluster create` registers each tenant cluster with the Platform, which is what licenses Private Nodes. A tenant cluster created before logging in is registered with `vcluster platform add vcluster private-a100 -n private-a100 --project default`. + +Tell vBilling who the tenants are. The emails use the reserved `.example` domain: + +```bash +kubectl annotate ns shared-l4 vbilling.vcluster.com/tenant=acme-ai "vbilling.vcluster.com/display-name=Acme AI" \ + vbilling.vcluster.com/email=billing@acme-ai.example vbilling.vcluster.com/plan=gpu-cloud +kubectl annotate ns private-a100 vbilling.vcluster.com/tenant=initech "vbilling.vcluster.com/display-name=Initech" \ + vbilling.vcluster.com/email=billing@initech.example vbilling.vcluster.com/plan=gpu-cloud +``` + +## 4. A private GPU node + +```bash +vcluster connect private-a100 -n private-a100 -- vcluster token create --expires 2h | grep node/join > join.txt +awk -v cmd="$(cat join.txt)" '$0 == "JOIN_COMMAND" { print cmd; next } { print }' 04-node-startup.sh > startup.sh +gcloud compute instances create gpu-node-1 --zone $ZONE --machine-type a2-highgpu-1g \ + --provisioning-model SPOT --instance-termination-action STOP --maintenance-policy TERMINATE \ + --image-family ubuntu-2204-lts --image-project ubuntu-os-cloud --boot-disk-size 150GB \ + --no-service-account --no-scopes --metadata-from-file startup-script=startup.sh +``` + +Once `gpu-node-1` is Ready in the tenant cluster, remove the token from the instance and from disk, and label the node so it gets an SKU: + +```bash +gcloud compute instances remove-metadata gpu-node-1 --zone $ZONE --keys startup-script +rm join.txt startup.sh +vcluster connect private-a100 -n private-a100 -- kubectl label node gpu-node-1 \ + node.kubernetes.io/instance-type=a2-highgpu-1g vbilling.vcluster.com/capacity-type=spot +``` + +## 5. GPU Operator and DCGM in the private tenant cluster + +```bash +helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update +vcluster connect private-a100 -n private-a100 -- \ + helm install gpu-operator nvidia/gpu-operator -n gpu-operator --create-namespace -f 05-gpu-operator-values.yaml +sed "s/PROMETHEUS_INGEST_IP/$INGEST_IP/" 06-prometheus-agent.yaml > agent.yaml +vcluster connect private-a100 -n private-a100 -- kubectl apply -f agent.yaml +``` + +The driver container compiles for the node's kernel, about five minutes. Optional MIG: label the node `nvidia.com/mig.config=all-1g.5gb`. On Compute Engine the GPU cannot be reset from the guest, so the MIG manager reports `failed` until the VM is rebooted (`gcloud compute instances reset`). A stop and start can move the VM to a GPU without MIG mode. + +## 6. vBilling + +```bash +kubectl create namespace vbilling +kubectl -n vbilling create secret generic vbilling-stripe --from-file=api-key=$HOME/.stripe-test-key +helm install vbilling ../../deploy/helm/vbilling -n vbilling -f 07-vbilling-values.yaml +vcluster connect shared-l4 -n shared-l4 -- kubectl apply -f workloads/shared-inference.yaml +vcluster connect private-a100 -n private-a100 -- kubectl apply -f workloads/private-training.yaml +``` + +vBilling creates the Stripe customers and meters on the first windows. Give the meters prices tagged with the plan, then restart vBilling so it subscribes the tenants: + +```bash +export STRIPE_API_KEY="$(cat ~/.stripe-test-key)" +curl -s https://api.stripe.com/v1/billing/meters -u "$STRIPE_API_KEY:" | grep -E '"(id|event_name)"' +curl -s https://api.stripe.com/v1/prices -u "$STRIPE_API_KEY:" \ + -d currency=usd -d unit_amount_decimal=220 \ + -d "recurring[interval]=month" -d "recurring[usage_type]=metered" -d "recurring[meter]=" \ + -d "product_data[name]=A100 40GB GPU-hour" -d "metadata[vbilling_plan]=gpu-cloud" +kubectl -n vbilling rollout restart statefulset/vbilling +``` + +## 7. Check it + +```bash +kubectl -n vbilling port-forward svc/vbilling 8080:8080 & +# dashboard: http://localhost:8080/ +curl -s "localhost:8080/api/v1/events?from=$(date -u +%Y-%m-%dT%H:00:00Z)" | head +curl -s "localhost:8080/api/v1/reconcile?from=&to=" # ledger vs Stripe, per tenant and metric +``` + +What one 30 second window looks like: + +| Tenant | Metric | Quantity | Why | +|---|---|---|---| +| acme-ai | `vcluster_gpu_hours` (sku `nvidia-l4-timeslice-4`) | 0.004166667 | 2 pods × ¼ GPU × 30 s | +| initech | `vcluster_gpu_hours` (sku `NVIDIA-A100-SXM4-40GB`) | 0.008333333 | the whole A100, also under MIG | +| initech | `vcluster_private_node_hours` (sku `a2-highgpu-1g`) | 0.008333333 | the node, billed whole | +| initech | `vcluster_cpu_core_hours` | 0.1 | all 12 cores | +| initech | `gpu_energy_kwh` | ≈ 0.0015 | ≈ 177 W for 30 s | + +![vBilling dashboard for this example](../../docs/images/dashboard-gke.png) + +If the spot A100 is reclaimed, the window in which the node becomes unreachable bills only the seconds before it, and the rest becomes `vcluster_gpu_downtime_hours` until the node is back. + +## Clean up + +```bash +gcloud compute instances delete gpu-node-1 --zone $ZONE --quiet +vcluster delete shared-l4 -n shared-l4 --delete-namespace +vcluster delete private-a100 -n private-a100 --delete-namespace +kubectl delete namespace monitoring vbilling # removes the load balancers and volumes GKE created +gcloud container clusters delete $CLUSTER --zone $ZONE --quiet +``` diff --git a/examples/gke-gpu/workloads/private-training.yaml b/examples/gke-gpu/workloads/private-training.yaml new file mode 100644 index 0000000..3d26873 --- /dev/null +++ b/examples/gke-gpu/workloads/private-training.yaml @@ -0,0 +1,22 @@ +# Apply inside the private-nodes tenant cluster. One replica uses the whole +# GPU; with MIG single strategy (all-1g.5gb) each replica gets one slice. +apiVersion: v1 +kind: Namespace +metadata: {name: training} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: nbody, namespace: training} +spec: + replicas: 1 + selector: {matchLabels: {app: nbody}} + template: + metadata: {labels: {app: nbody}} + spec: + containers: + - name: nbody + image: nvcr.io/nvidia/k8s/cuda-sample:nbody-cuda11.7.1 + command: [sh, -c, "while true; do /cuda-samples/sample -benchmark -numbodies=2048000 >/dev/null 2>&1 || sleep 5; done"] + resources: + requests: {cpu: "1", memory: 2Gi} + limits: {nvidia.com/gpu: 1} diff --git a/examples/gke-gpu/workloads/shared-inference.yaml b/examples/gke-gpu/workloads/shared-inference.yaml new file mode 100644 index 0000000..360833e --- /dev/null +++ b/examples/gke-gpu/workloads/shared-inference.yaml @@ -0,0 +1,71 @@ +# Apply inside the shared-nodes tenant cluster: two pods on a time-sliced L4 +# (a quarter GPU each), a CPU service behind an internal load balancer, and a +# 20 GiB volume, so every metric type shows up. +apiVersion: v1 +kind: Namespace +metadata: {name: inference} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: nbody, namespace: inference} +spec: + replicas: 2 + selector: {matchLabels: {app: nbody}} + template: + metadata: {labels: {app: nbody}} + spec: + nodeSelector: + cloud.google.com/gke-gpu-sharing-strategy: time-sharing + cloud.google.com/gke-max-shared-clients-per-gpu: "4" + containers: + - name: nbody + image: nvcr.io/nvidia/k8s/cuda-sample:nbody-cuda11.7.1 + command: [sh, -c, "while true; do /cuda-samples/sample -benchmark -numbodies=1024000 >/dev/null 2>&1 || sleep 5; done"] + resources: + requests: {cpu: 500m, memory: 1Gi} + limits: {nvidia.com/gpu: 1} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: api, namespace: inference} +spec: + replicas: 2 + selector: {matchLabels: {app: api}} + template: + metadata: {labels: {app: api}} + spec: + containers: + - name: api + image: nginx:1.27-alpine + ports: [{containerPort: 80}] + resources: {requests: {cpu: 500m, memory: 512Mi}} +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: inference + annotations: {networking.gke.io/load-balancer-type: Internal} +spec: + type: LoadBalancer + selector: {app: api} + ports: [{port: 80, targetPort: 80}] +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: model-cache, namespace: inference} +spec: + accessModes: [ReadWriteOnce] + resources: {requests: {storage: 20Gi}} +--- +apiVersion: v1 +kind: Pod +metadata: {name: cache-warmer, namespace: inference} +spec: + containers: + - name: warmer + image: busybox:1.36 + command: [sh, -c, "dd if=/dev/zero of=/cache/weights.bin bs=1M count=512 && sleep infinity"] + resources: {requests: {cpu: 100m, memory: 64Mi}} + volumeMounts: [{name: cache, mountPath: /cache}] + volumes: [{name: cache, persistentVolumeClaim: {claimName: model-cache}}] From 2ac394a652c87fa67671f5505264e6b1bbe1b7c8 Mon Sep 17 00:00:00 2001 From: Saiyam Pathak Date: Mon, 5 Oct 2026 23:42:28 +0530 Subject: [PATCH 4/4] Add the Apache-2.0 license and CI and release workflows CI runs vet, race tests and a Helm lint on every pull request. Publishing a release builds the linux/amd64 and linux/arm64 image the chart defaults to and attaches the packaged chart. --- .github/workflows/ci.yml | 28 +++++ .github/workflows/release.yml | 51 +++++++++ LICENSE | 202 ++++++++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 LICENSE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a923c59 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - run: go vet ./... + - run: go test -race ./... + + helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-helm@v4 + - run: helm lint deploy/helm/vbilling + - run: helm template vbilling deploy/helm/vbilling --set adapters='{stripe}' --set stripe.existingSecret=vbilling-stripe > /dev/null diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d057a62 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: release + +# Publishing a GitHub release (tag vX.Y.Z) builds the multi-arch image the +# Helm chart defaults to and attaches the packaged chart to the release. +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: write + packages: write + +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/vbilling + tags: | + type=semver,pattern=v{{version}} + type=semver,pattern=v{{major}}.{{minor}} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} + - uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + chart: + needs: image + if: github.event_name == 'release' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-helm@v4 + - run: helm package deploy/helm/vbilling --destination dist + - run: gh release upload "${{ github.event.release.tag_name }}" dist/*.tgz --clobber + env: + GH_TOKEN: ${{ github.token }} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.