add generic metrics for external API

cmd/stackit-csi-plugin/main.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/stackitcloud/cloud-provider-stackit/pkg/csi"
@@ -22,7 +26,7 @@ var (
 	endpoint                 string
 	cloudConfig              string
 	cluster                  string
-	httpEndpoint             string
+	metricsAddress           string
 	provideControllerService bool
 	provideNodeService       bool
 	legacyStorageMode        bool
@@ -34,7 +38,10 @@ func main() {
 		Use:   "stackit-csi-plugin",
 		Short: "STACKIT block-storage CSI plugin",
 		Run: func(_ *cobra.Command, _ []string) {
-			handle()
+			ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
+			defer cancel()
+
+			handle(ctx)
 		},
 		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
 			f := cmd.Flags()
@@ -67,7 +74,7 @@ func main() {
 	cmd.Flags().StringVar(&cloudConfig, "cloud-config", "", "CSI driver cloud config. This option can be given multiple times")
 
 	cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.")
-	cmd.PersistentFlags().StringVar(&httpEndpoint, "http-endpoint", "",
+	cmd.PersistentFlags().StringVar(&metricsAddress, "metrics-address", "",
 		"The TCP network address where the HTTP server for providing metrics for diagnostics, will listen (example: `:8080`)."+
 			"The default is empty string, which means the server is disabled.")
 
@@ -85,7 +92,16 @@ func main() {
 	os.Exit(code)
 }
 
-func handle() {
+func handle(ctx context.Context) {
+	if metricsAddress != "" {
+		metricsExporter := metrics.NewExporter()
+		prometheus.MustRegister(metricsExporter)
+		go func() {
+			if err := metrics.Run(ctx, metricsAddress); err != nil {
+				klog.Fatalf("Run metrics returned an error: %v", err)
+			}
+		}()
+	}
 	// Initialize cloud
 	driverOpts := &blockstorage.DriverOpts{
 		Endpoint:  endpoint,
@@ -110,8 +126,9 @@ func handle() {
 			klog.Fatal(err)
 		}
 
+		iaasHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameIaaS)
 		iaasOpts := []sdkconfig.ConfigurationOption{
-			sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient()),
+			sdkconfig.WithHTTPClient(iaasHTTPClient),
 		}
 
 		if cfg.Global.APIEndpoints.IaasAPI != "" {

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.32.0
 	github.com/onsi/gomega v1.42.1
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
 	github.com/stackitcloud/stackit-sdk-go/core v0.26.0
@@ -87,7 +88,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.0 // indirect

pkg/ccm/stackit.go

@@ -117,8 +117,9 @@ func BuildObservability() (*MetricsRemoteWrite, error) {
 
 // NewCloudControllerManager creates a new instance of the stackit struct from a stackitconfig struct
 func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteWrite) (*CloudControllerManager, error) {
+	lbHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameLoadBalancer)
 	lbOpts := []sdkconfig.ConfigurationOption{
-		sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient()),
+		sdkconfig.WithHTTPClient(lbHTTPClient),
 	}
 
 	if cfg.Global.APIEndpoints.LoadBalancerAPI != "" {
@@ -138,8 +139,9 @@ func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteW
 		return nil, fmt.Errorf("failed to create lb client: %v", err)
 	}
 
+	iaasHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameIaaS)
 	iaasOpts := []sdkconfig.ConfigurationOption{
-		sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient()),
+		sdkconfig.WithHTTPClient(iaasHTTPClient),
 	}
 
 	if cfg.Global.APIEndpoints.IaasAPI != "" {

pkg/metrics/http.go

@@ -3,19 +3,24 @@ package metrics
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-func NewInstrumentedHTTPClient() *http.Client {
+func NewInstrumentedHTTPClient(api string) *http.Client {
 	return &http.Client{
-		Transport: &InstrumentedRoundTripper{http.DefaultTransport},
+		Transport: &InstrumentedRoundTripper{
+			api:  api,
+			base: http.DefaultTransport,
+		},
 	}
 }
 
 type InstrumentedRoundTripper struct {
+	api  string
 	base http.RoundTripper
 }
 
@@ -26,15 +31,26 @@ func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Resp
 	response, err := rt.base.RoundTrip(request)
 	duration := time.Since(startTime)
 
-	LoadBalancerResponseTimeHistogram.
-		With(prometheus.Labels{operationLabel: operation}).
-		Observe(float64(duration.Seconds()))
-	LoadBalancerRequestCount.
-		With(prometheus.Labels{operationLabel: operation}).
-		Inc()
+	statusCode := "network_error"
+	if response != nil {
+		statusCode = strconv.Itoa(response.StatusCode)
+	}
+
+	labels := prometheus.Labels{
+		apiLabel:       rt.api,
+		methodLabel:    request.Method,
+		operationLabel: operation,
+		codeLabel:      statusCode,
+	}
+
+	HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds())
+	HTTPRequestCount.With(labels).Inc()
+
+	isHTTPError := response != nil && response.StatusCode >= 400
+	isNetworkError := err != nil
 
-	if response != nil && response.StatusCode >= http.StatusInternalServerError {
-		LoadBalancerErrorCount.Inc()
+	if isHTTPError || isNetworkError {
+		HTTPErrorCount.With(labels).Inc()
 	}
 
 	return response, err

pkg/metrics/http_test.go

@@ -2,10 +2,14 @@ package metrics
 
 import (
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
 )
 
 var _ = Describe("Metrics", func() {
@@ -22,4 +26,148 @@ var _ = Describe("Metrics", func() {
 		Entry("get load-balancers", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers", "get_load-balancers"),
 		Entry("get load-balancers instance", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers/id", "get_load-balancers_instance"),
 	)
+
+	Describe("InstrumentedRoundTripper", func() {
+		It("increments HTTPRequestCount for responses", func() {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer server.Close()
+
+			labels := prometheus.Labels{
+				apiLabel:       "test",
+				methodLabel:    "GET",
+				operationLabel: "get_request-count-test",
+				codeLabel:      "200",
+			}
+			before := testutil.ToFloat64(HTTPRequestCount.With(labels))
+
+			client := NewInstrumentedHTTPClient("test")
+
+			response, err := client.Get(server.URL + "/request-count-test")
+			Expect(err).NotTo(HaveOccurred())
+			defer response.Body.Close()
+
+			after := testutil.ToFloat64(HTTPRequestCount.With(labels))
+			Expect(after - before).To(Equal(float64(1)))
+		})
+
+		It("records HTTPRequestDurationHistogram observations for responses", func() {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer server.Close()
+
+			labels := prometheus.Labels{
+				apiLabel:       "test",
+				methodLabel:    "GET",
+				operationLabel: "get_request-duration-test",
+				codeLabel:      "200",
+			}
+			before := histogramSampleCount(HTTPRequestDurationHistogram.With(labels))
+
+			client := NewInstrumentedHTTPClient("test")
+
+			response, err := client.Get(server.URL + "/request-duration-test")
+			Expect(err).NotTo(HaveOccurred())
+			defer response.Body.Close()
+
+			after := histogramSampleCount(HTTPRequestDurationHistogram.With(labels))
+			Expect(after - before).To(Equal(uint64(1)))
+		})
+
+		It("increments HTTPErrorCount for error responses (400, 404, 500)", func() {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.Method == http.MethodPost {
+					w.WriteHeader(http.StatusInternalServerError)
+					return
+				}
+				if r.URL.Path == "/404" {
+					w.WriteHeader(http.StatusNotFound)
+					return
+				}
+				w.WriteHeader(http.StatusBadRequest)
+			}))
+			defer server.Close()
+
+			labels400 := prometheus.Labels{
+				apiLabel:       "test",
+				methodLabel:    http.MethodGet,
+				operationLabel: "get_",
+				codeLabel:      "400",
+			}
+			labels404 := prometheus.Labels{
+				apiLabel:       "test",
+				methodLabel:    http.MethodGet,
+				operationLabel: "get_404",
+				codeLabel:      "404",
+			}
+			labels500 := prometheus.Labels{
+				apiLabel:       "test",
+				methodLabel:    http.MethodPost,
+				operationLabel: "post_",
+				codeLabel:      "500",
+			}
+			before400 := testutil.ToFloat64(HTTPErrorCount.With(labels400))
+			before404 := testutil.ToFloat64(HTTPErrorCount.With(labels404))
+			before500 := testutil.ToFloat64(HTTPErrorCount.With(labels500))
+
+			client := NewInstrumentedHTTPClient("test")
+
+			response1, err := client.Get(server.URL)
+			Expect(err).NotTo(HaveOccurred())
+			defer response1.Body.Close()
+
+			response2, err := client.Get(server.URL + "/404")
+			Expect(err).NotTo(HaveOccurred())
+			defer response2.Body.Close()
+
+			response3, err := client.Post(server.URL, "application/json", nil)
+			Expect(err).NotTo(HaveOccurred())
+			defer response3.Body.Close()
+
+			after400 := testutil.ToFloat64(HTTPErrorCount.With(labels400))
+			after404 := testutil.ToFloat64(HTTPErrorCount.With(labels404))
+			after500 := testutil.ToFloat64(HTTPErrorCount.With(labels500))
+
+			Expect(after400 - before400).To(Equal(float64(1)))
+			Expect(after404 - before404).To(Equal(float64(1)))
+			Expect(after500 - before500).To(Equal(float64(1)))
+			Expect((after400 - before400) + (after404 - before404) + (after500 - before500)).To(Equal(float64(3)))
+		})
+
+		It("does not increment HTTPErrorCount for successful responses", func() {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer server.Close()
+
+			labels := prometheus.Labels{
+				apiLabel:       "test",
+				methodLabel:    http.MethodGet,
+				operationLabel: "get_",
+				codeLabel:      "200",
+			}
+			before := testutil.ToFloat64(HTTPErrorCount.With(labels))
+
+			client := NewInstrumentedHTTPClient("test")
+
+			response, err := client.Get(server.URL)
+			Expect(err).NotTo(HaveOccurred())
+			defer response.Body.Close()
+
+			after := testutil.ToFloat64(HTTPErrorCount.With(labels))
+			Expect(after - before).To(Equal(float64(0)))
+		})
+	})
 })
+
+func histogramSampleCount(observer prometheus.Observer) uint64 {
+	metric, ok := observer.(prometheus.Metric)
+	Expect(ok).To(BeTrue())
+
+	dtoMetric := &dto.Metric{}
+	Expect(metric.Write(dtoMetric)).To(Succeed())
+
+	return dtoMetric.GetHistogram().GetSampleCount()
+}