diff --git a/integration-tests/metrics_storage_test.go b/integration-tests/metrics_storage_test.go new file mode 100644 index 000000000..1de18defc --- /dev/null +++ b/integration-tests/metrics_storage_test.go @@ -0,0 +1,230 @@ +package tests + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/upsun/cli/pkg/mockapi" +) + +func mountpointMetrics(diskUsed, diskLimit, inodesUsed, inodesLimit float64) map[string]any { + return map[string]any{ + "disk_used": map[string]any{"avg": diskUsed}, + "disk_limit": map[string]any{"max": diskLimit}, + "inodes_used": map[string]any{"avg": inodesUsed}, + "inodes_limit": map[string]any{"max": inodesLimit}, + } +} + +// filterServices applies the services[] query filter, like the API. +func filterServices(req *http.Request, services map[string]any) map[string]any { + var names []string + for k, v := range req.URL.Query() { + if strings.HasPrefix(k, "services[") { + names = append(names, v...) + } + } + if len(names) == 0 { + return services + } + filtered := make(map[string]any) + for _, n := range names { + if svc, ok := services[n]; ok { + filtered[n] = svc + } + } + return filtered +} + +func setupMetricsTest(t *testing.T, withStorage, withStorageMount bool) (f *cmdFactory, projectID string) { + authServer := mockapi.NewAuthServer(t) + t.Cleanup(authServer.Close) + + apiHandler := mockapi.NewHandler(t) + apiServer := httptest.NewServer(apiHandler) + t.Cleanup(apiServer.Close) + + projectID = mockapi.ProjectID() + apiHandler.SetProjects([]*mockapi.Project{{ + ID: projectID, + Links: mockapi.MakeHALLinks("self=/projects/"+projectID, + "environments=/projects/"+projectID+"/environments"), + DefaultBranch: "main", + }}) + + main := makeEnv(projectID, "main", "production", "active", nil) + obsPath := "/projects/" + projectID + "/environments/main/observability" + app := mockapi.App{Name: "app", Type: "php:8.4", Size: "AUTO"} + if withStorageMount { + app.Mounts = map[string]mockapi.Mount{"/files": {Source: "storage", SourcePath: "files"}} + } + main.SetCurrentDeployment(&mockapi.Deployment{ + WebApps: map[string]mockapi.App{ + "app": app, + }, + Services: map[string]mockapi.App{ + "db": {Name: "db", Type: "mariadb:11.4", Size: "AUTO"}, + }, + Workers: map[string]mockapi.Worker{}, + Routes: map[string]any{}, + Links: mockapi.MakeHALLinks("self=/projects/" + projectID + "/environments/main/deployment/current"), + }) + apiHandler.SetEnvironments([]*mockapi.Environment{main}) + + apiHandler.Get(obsPath+"/", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "_links": mockapi.MakeHALLinks("resources_overview=" + apiServer.URL + obsPath + "/resources/overview"), + }) + }) + apiHandler.Get(obsPath+"/resources/overview", func(w http.ResponseWriter, req *http.Request) { + limits := map[string]any{ + "cpu_used": map[string]any{"avg": 0.1}, + "cpu_limit": map[string]any{"max": 1.0}, + "memory_used": map[string]any{"avg": 256.0}, + "memory_limit": map[string]any{"max": 1024.0}, + } + withMounts := func(mounts map[string]any) map[string]any { + m := map[string]any{"mountpoints": mounts} + for k, v := range limits { + m[k] = v + } + return m + } + appMounts := map[string]any{ + "/mnt": mountpointMetrics(100, 1000, 10, 1000), + "/tmp": mountpointMetrics(500, 1000, 40, 1000), + } + dbMounts := map[string]any{ + "/mnt": mountpointMetrics(250, 1000, 5, 1000), + "/tmp": mountpointMetrics(10, 1000, 1, 1000), + } + if withStorage { + appMounts["storage"] = mountpointMetrics(920, 1000, 30, 100) + // Storage with a partial set of inode metrics. + dbMounts["storage"] = map[string]any{ + "disk_used": map[string]any{"avg": 500}, + "disk_limit": map[string]any{"max": 1000}, + "inodes_limit": map[string]any{"max": 100}, + } + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "_grain": 60, + "_from": 1790189400, + "_to": 1790190000, + "data": []any{map[string]any{ + "timestamp": 1790190000, + "services": filterServices(req, map[string]any{ + "app": withMounts(appMounts), + "db": withMounts(dbMounts), + }), + }}, + }) + }) + + return newCommandFactory(t, apiServer.URL, authServer.URL), projectID +} + +func TestMetricsStorage(t *testing.T) { + cases := []struct { + name string + withStorage bool + withStorageMount bool + env []string + args []string + want string + notWant string + }{ + { + name: "all storage columns", + withStorage: true, + args: []string{"metrics:all", "-1", "--no-header", "--format", "plain", + "-c", "service,disk_percent,storage_percent,storage_inodes_percent"}, + want: "app\t10.0%\t92.0%\t30.0%\ndb\t25.0%\t50.0%\t\n", + }, + { + name: "all table shows storage when present", + withStorage: true, + args: []string{"metrics:all", "-1"}, + want: "Storage %", + }, + { + name: "all table hides storage when absent", + args: []string{"metrics:all", "-1"}, + notWant: "Storage", + }, + { + name: "all table shows storage when the deployment has storage mounts", + withStorageMount: true, + args: []string{"metrics:all", "-1"}, + want: "Storage %", + }, + { + name: "all table ignores storage mounts of filtered-out apps", + withStorageMount: true, + args: []string{"metrics:all", "-1", "-s", "db"}, + notWant: "Storage", + }, + { + name: "all csv always includes storage", + args: []string{"metrics:all", "-1", "--format", "csv"}, + want: "/tmp inodes %,Storage %,Storage inodes %\n", + }, + { + name: "disk-usage storage columns", + withStorage: true, + args: []string{"disk", "-1", "--no-header", "-B", "--format", "plain", + "-c", "service,storage_used,storage_limit,storage_percent,storage_ipercent"}, + want: "app\t920\t1000\t92.0%\t30.0%\ndb\t500\t1000\t50.0%\t\n", + }, + { + name: "disk-usage table shows storage when present", + withStorage: true, + args: []string{"disk", "-1"}, + want: "Storage used", + }, + { + name: "disk-usage csv always includes storage", + args: []string{"disk", "-1", "--format", "csv"}, + want: "/tmp %,Storage used,Storage limit,Storage %,Storage inodes %\n", + }, + { + name: "disabled", + withStorage: true, + env: []string{"TEST_CLI_API_METRICS_STORAGE=0"}, + args: []string{"metrics:all", "-1", "--format", "csv"}, + notWant: "Storage", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + f, projectID := setupMetricsTest(t, c.withStorage, c.withStorageMount) + f.extraEnv = c.env + args := append([]string{}, c.args...) + args = append(args, "-p", projectID, "-e", "main") + stdout, stderr, err := f.RunCombinedOutput(args...) + require.NoError(t, err, "stderr: %s", stderr) + assert.NotContains(t, stderr, "Warning") + if c.want != "" { + assert.Contains(t, stdout, c.want) + } + if c.notWant != "" { + assert.NotContains(t, stdout, c.notWant) + } + }) + } +} + +func TestMetricsStorageDisabledColumn(t *testing.T) { + f, projectID := setupMetricsTest(t, true, false) + f.extraEnv = []string{"TEST_CLI_API_METRICS_STORAGE=0"} + + _, stderr, err := f.RunCombinedOutput("disk", "-1", "-c", "storage_used", "-p", projectID, "-e", "main") + assert.Error(t, err) + assert.Contains(t, stderr, "Column not found: storage_used") +} diff --git a/legacy/config-defaults.yaml b/legacy/config-defaults.yaml index 256665a74..c032c0094 100644 --- a/legacy/config-defaults.yaml +++ b/legacy/config-defaults.yaml @@ -268,6 +268,9 @@ api: # Whether the Metrics API is enabled. metrics: false + # Whether to show network storage metrics (the "storage" mountpoint) in metrics commands. + metrics_storage: true + # Whether the Flexible Resources API (AKA sizing/scaling) is enabled. sizing: false diff --git a/legacy/src/Command/Metrics/AllMetricsCommand.php b/legacy/src/Command/Metrics/AllMetricsCommand.php index b418cafa8..a8308c60d 100644 --- a/legacy/src/Command/Metrics/AllMetricsCommand.php +++ b/legacy/src/Command/Metrics/AllMetricsCommand.php @@ -52,6 +52,14 @@ class AllMetricsCommand extends MetricsCommandBase 'tmp_inodes_used' => '/tmp inodes used', 'tmp_inodes_limit' => '/tmp inodes limit', 'tmp_inodes_percent' => '/tmp inodes %', + + 'storage_used' => 'Storage used', + 'storage_limit' => 'Storage limit', + 'storage_percent' => 'Storage %', + + 'storage_inodes_used' => 'Storage inodes used', + 'storage_inodes_limit' => 'Storage inodes limit', + 'storage_inodes_percent' => 'Storage inodes %', ]; /** @var string[] */ @@ -106,7 +114,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $bytes = $input->getOption('bytes'); - $rows = $this->buildRows($values, [ + $fields = [ 'cpu_used' => new Field( Format::Rounded2p, new SourceField(MetricKind::CpuUsed, Aggregation::Avg), @@ -203,7 +211,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int new SourceField(MetricKind::InodesLimit, Aggregation::Max, '/tmp') ), ), - ], $environment); + ]; + if ($this->storageMetricsEnabled()) { + $fields += $this->storageFields($bytes, 'storage_inodes_'); + } + $rows = $this->buildRows($values, $fields, $environment); + [$header, $defaultColumns] = $this->storageColumns(self::TABLE_HEADER, $this->defaultColumns, $values, ['storage_percent', 'storage_inodes_percent'], $environment); if (!$this->table->formatIsMachineReadable()) { $formatter = $this->propertyFormatter; @@ -215,7 +228,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int )); } - $this->table->render($rows, self::TABLE_HEADER, $this->defaultColumns); + $this->table->render($rows, $header, $defaultColumns); if (!$this->table->formatIsMachineReadable()) { $this->explainHighMemoryServices(); diff --git a/legacy/src/Command/Metrics/DiskUsageCommand.php b/legacy/src/Command/Metrics/DiskUsageCommand.php index 5142abe59..bd1433766 100644 --- a/legacy/src/Command/Metrics/DiskUsageCommand.php +++ b/legacy/src/Command/Metrics/DiskUsageCommand.php @@ -39,6 +39,12 @@ class DiskUsageCommand extends MetricsCommandBase 'tmp_iused' => '/tmp inodes used', 'tmp_ilimit' => '/tmp inodes limit', 'tmp_ipercent' => '/tmp inodes %', + 'storage_used' => 'Storage used', + 'storage_limit' => 'Storage limit', + 'storage_percent' => 'Storage %', + 'storage_iused' => 'Storage inodes used', + 'storage_ilimit' => 'Storage inodes limit', + 'storage_ipercent' => 'Storage inodes %', ]; /** @var string[] */ private array $defaultColumns = ['timestamp', 'service', 'used', 'limit', 'percent', 'ipercent', 'tmp_percent']; @@ -76,7 +82,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $bytes = $input->getOption('bytes'); - $rows = $this->buildRows($values, [ + $fields = [ 'used' => new Field( $bytes ? Format::Rounded : Format::Disk, new SourceField(MetricKind::DiskUsed, Aggregation::Avg, '/mnt'), @@ -140,7 +146,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int new SourceField(MetricKind::InodesLimit, Aggregation::Max, '/tmp') ), ), - ], $environment); + ]; + if ($this->storageMetricsEnabled()) { + $fields += $this->storageFields($bytes, 'storage_i'); + } + $rows = $this->buildRows($values, $fields, $environment); + [$header, $defaultColumns] = $this->storageColumns(self::TABLE_HEADER, $this->defaultColumns, $values, ['storage_used', 'storage_limit', 'storage_percent', 'storage_ipercent'], $environment); if (!$this->table->formatIsMachineReadable()) { $formatter = $this->propertyFormatter; @@ -153,7 +164,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int )); } - $this->table->render($rows, self::TABLE_HEADER, $this->defaultColumns); + $this->table->render($rows, $header, $defaultColumns); return 0; } diff --git a/legacy/src/Command/Metrics/MetricsCommandBase.php b/legacy/src/Command/Metrics/MetricsCommandBase.php index cac95c41c..9838114e1 100644 --- a/legacy/src/Command/Metrics/MetricsCommandBase.php +++ b/legacy/src/Command/Metrics/MetricsCommandBase.php @@ -4,7 +4,10 @@ namespace Platformsh\Cli\Command\Metrics; +use Platformsh\Cli\Model\Metrics\Aggregation; use Platformsh\Cli\Model\Metrics\Field; +use Platformsh\Cli\Model\Metrics\Format; +use Platformsh\Cli\Model\Metrics\MetricKind; use Platformsh\Cli\Model\Metrics\SourceField; use Platformsh\Cli\Model\Metrics\SourceFieldPercentage; use Platformsh\Cli\Selector\Selector; @@ -42,6 +45,9 @@ abstract class MetricsCommandBase extends CommandBase public const MIN_RANGE = 300; // 5 minutes public const DEFAULT_RANGE = 600; + // The mountpoint key of the network storage volume (used by "storage" mounts). + public const STORAGE_MOUNTPOINT = 'storage'; + // Data points that started within this many seconds of now may still be missing services. private const LATEST_SETTLE_TIME = 120; @@ -282,6 +288,110 @@ private function getServices(InputInterface $input, Environment $environment): a return $selectedServiceNames; } + protected function storageMetricsEnabled(): bool + { + return $this->config->getBool('api.metrics_storage'); + } + + /** + * Returns fields for the storage volume, with inode fields keyed by $inodesPrefix. + * + * @return array + */ + protected function storageFields(bool $bytes, string $inodesPrefix): array + { + $m = self::STORAGE_MOUNTPOINT; + + return [ + 'storage_used' => new Field( + $bytes ? Format::Rounded : Format::Disk, + new SourceField(MetricKind::DiskUsed, Aggregation::Avg, $m), + ), + 'storage_limit' => new Field( + $bytes ? Format::Rounded : Format::Disk, + new SourceField(MetricKind::DiskLimit, Aggregation::Max, $m), + ), + 'storage_percent' => new Field( + Format::Percent, + new SourceFieldPercentage( + new SourceField(MetricKind::DiskUsed, Aggregation::Avg, $m), + new SourceField(MetricKind::DiskLimit, Aggregation::Max, $m) + ), + ), + $inodesPrefix . 'used' => new Field( + Format::Rounded, + new SourceField(MetricKind::InodesUsed, Aggregation::Avg, $m), + ), + $inodesPrefix . 'limit' => new Field( + Format::Rounded, + new SourceField(MetricKind::InodesLimit, Aggregation::Max, $m), + ), + $inodesPrefix . 'percent' => new Field( + Format::Percent, + new SourceFieldPercentage( + new SourceField(MetricKind::InodesUsed, Aggregation::Avg, $m), + new SourceField(MetricKind::InodesLimit, Aggregation::Max, $m) + ), + ), + ]; + } + + /** + * Adjusts the table header and default columns for storage metrics. + * + * Storage columns are removed if storage metrics are disabled. Otherwise, + * the $storageColumns are shown by default in machine-readable formats + * (for stable output), or in tables if the environment uses storage. + * + * @param array $header + * @param string[] $defaultColumns + * @param array $values + * @param string[] $storageColumns + * @return array{array, string[]} + */ + protected function storageColumns(array $header, array $defaultColumns, array $values, array $storageColumns, Environment $environment): array + { + if (!$this->storageMetricsEnabled()) { + return [array_filter($header, fn($key): bool => !str_starts_with($key, 'storage_'), ARRAY_FILTER_USE_KEY), $defaultColumns]; + } + if ($this->table->formatIsMachineReadable() || $this->usesStorage($values, $environment)) { + return [$header, array_merge($defaultColumns, $storageColumns)]; + } + + return [$header, $defaultColumns]; + } + + /** + * Checks if any of the returned services reports storage or has storage mounts. + * + * @param array $values + */ + private function usesStorage(array $values, Environment $environment): bool + { + $serviceNames = []; + foreach ($values['data'] as $point) { + foreach ($point['services'] ?? [] as $name => $service) { + if (isset($service['mountpoints'][self::STORAGE_MOUNTPOINT])) { + return true; + } + $serviceNames[$name] = true; + } + } + $deployment = $this->api->getCurrentDeployment($environment); + foreach (array_merge($deployment->webapps, $deployment->workers) as $name => $app) { + if (!isset($serviceNames[$name])) { + continue; + } + foreach ($app->getProperty('mounts', false) ?: [] as $mount) { + if (($mount['source'] ?? null) === 'storage') { + return true; + } + } + } + + return false; + } + protected function getChooseEnvFilter(): ?callable { return null; @@ -445,7 +555,7 @@ private function getValueFromSource(array $point, SourceField|SourceFieldPercent $value = $this->extractValue($point, $fieldDefinition->value); $limit = $this->extractValue($point, $fieldDefinition->limit); - return $limit > 0 ? $value / $limit * 100 : null; + return $value !== null && $limit > 0 ? $value / $limit * 100 : null; } return $this->extractValue($point, $fieldDefinition); @@ -462,6 +572,10 @@ private function extractValue(array $point, SourceField $sourceField): ?float if (!isset($point['mountpoints'][$sourceField->mountpoint])) { return null; } + // The storage volume may not report every metric. + if ($sourceField->mountpoint === self::STORAGE_MOUNTPOINT) { + return $point['mountpoints'][$sourceField->mountpoint][$sourceField->source->value][$sourceField->aggregation->value] ?? null; + } if (!isset($point['mountpoints'][$sourceField->mountpoint][$sourceField->source->value])) { throw new \RuntimeException(\sprintf('Source "%s" not found in the mountpoint "%s".', $sourceField->source->value, $sourceField->mountpoint)); }