feat(doctor): surface the machine beneath the nodes (backend#2221)

internal/cli/testdata/golden/zz-all-strings.golden

@@ -76,6 +76,7 @@ screen. %s/%d are runtime placeholders.
 "%s · starting up, not ready yet — run %s"
 "%s ×%d"
 "%s — %s"
+"%s — Kubernetes believes %.2f× the memory this machine has, because the k3d node containers are uncapped and each reports the whole VM"
 "%s, … and %d more"
 "%s/%s"
 "%s: %v"
@@ -138,6 +139,7 @@ screen. %s/%d are runtime placeholders.
 "Cancelled — nothing was removed."
 "Cancelled — the name didn't match. Nothing was removed."
 "Chart uninstall reported: %v"
+"Check Docker is running: docker info"
 "Check on it later with:  kubectl logs -f -n %s job/%s"
 "Check your data"
 "Check your network / HTTP(S)_PROXY, then run `%s doctor` again."
@@ -189,10 +191,12 @@ screen. %s/%d are runtime placeholders.
 "Diagnose auth / cluster problems with: tracebloc doctor"
 "Do you want to change the allocation? Run `%s resources set` (guided walkthrough on a terminal)."
 "Do you want to ingest training or test data?"
+"Docker VM %s (%d cpu) → %d node%s claiming %s → %s unrequested"
 "Docker and related tools — remove them yourself if you no longer need them"
 "Dry run — nothing was changed"
 "Dry-run complete — your data and secure environment check out; nothing was created."
 "Dry-run — nothing was deleted."
+"Each node container reports the whole VM, so %d of them double-count it. Run a single-node environment, or cap the nodes (k3d --servers-memory/--agents-memory). Until then a job that fits a node may still OOM the VM."
 "Each training run already uses up to %s — nothing to change."
 "Each training run may now use up to %s."
 "Email it to support@tracebloc.io."
@@ -485,6 +489,9 @@ screen. %s/%d are runtime placeholders.
 "decoding submit response (got body %q): %w"
 "deleting stage Pod %s/%s: %w"
 "destination"
+"docker info reported cores=%d mem=%d"
+"docker info returned %q, want two fields"
+"docker info: %w"
 "dropping %s.%s: %w%s"
 "e.g. %s or %s"
 "e.g. 17 for COCO pose"
@@ -508,6 +515,8 @@ screen. %s/%d are runtime placeholders.
 "generating idempotency key: %w"
 "generating staging-dir suffix: %w"
 "github releases: HTTP %d"
+"host %s →"
+"host memory unsupported on %s"
 "how many CPU cores a single training run may use"
 "how much memory a single training run may use, in GiB"
 "http://%s.%s.svc.cluster.local:%d"
@@ -579,9 +588,11 @@ screen. %s/%d are runtime placeholders.
 "no .txt files found in %q. Text categories expect <dir>/%s/*.txt."
 "no .xml annotation files found in %q. object_detection expects <dir>/annotations/*.xml (Pascal VOC)."
 "no CLI-supported tasks for %s data yet"
+"no MemTotal in /proc/meminfo"
 "no PersistentVolumeClaim named %q found in namespace %q. The chart's _helpers.tpl pins this name; if your install renamed it out-of-band, the CLI doesn't yet support that (read-name-from-jobs-manager is a v0.2 follow-up). Verify with: kubectl get pvc -n %s"
 "no Ready node can fit a training job (needs %s)"
 "no Ready node on this machine to size a training run against"
+"no Ready node to measure"
 "no Running Pod backing service %s/%s (found %d Pod(s); check `kubectl get pods -n %s -l %s`)"
 "no Running pod with name containing %q in namespace %q"
 "no Secret of type kubernetes.io/service-account-token bound to ServiceAccount %s found in namespace %s"
@@ -596,7 +607,9 @@ screen. %s/%d are runtime placeholders.
 "no tracebloc client found"
 "no usable image files in %q — found %s, but the ingestor accepts only .jpg, .jpeg, or .png. Convert the images and re-run."
 "no usable ingestor token. TokenRequest failed: %v. Fallback to static secret also failed: %w. Remediation: either grant your user the `create` verb on `serviceaccounts/token` (RBAC), or have an admin create a long-lived Secret of type kubernetes.io/service-account-token that references the %s ServiceAccount in namespace %s."
+"nodes report no memory capacity to compare against the VM"
 "none detected"
+"not a local k3d cluster — the host/VM chain applies only to a cluster running on this machine's Docker VM"
 "not signed in — run `tracebloc login` first"
 "outcome:   early exit before the cluster was probed"
 "outcome:   early exit — no roll-up verdict (granular checks below)"
@@ -617,6 +630,7 @@ screen. %s/%d are runtime placeholders.
 "push.StagedPrefix: unsafe table name %q — caller must ValidateTableName before constructing a PVC path"
 "pvc path"
 "querying datasets: %w%s"
+"read /proc/meminfo: %w"
 "reading %q: %w"
 "reading %s header: %w"
 "reading %s/: %w"
@@ -700,6 +714,7 @@ screen. %s/%d are runtime placeholders.
 "submit response missing job_name (got body %q)"
 "submit response missing namespace (got body %q)"
 "synthesized spec failed schema validation; check the flag values above"
+"sysctl hw.memsize: %w"
 "table %q already exists in this secure environment. Re-ingesting the same table doesn't merge or replace — the run would fail after uploading everything. Re-run with --overwrite to replace it, or pick a different --name. (`tracebloc data delete %s` also removes it.)"
 "table name is %d characters; the max is %d (matches both the MySQL identifier limit and the Kubernetes label-value limit, which the stage Pod's tracebloc.io/table label is bound by). Use a shorter name."
 "tabular = a CSV table; image = labels.csv + images/; text = labels.csv + texts/"
@@ -711,6 +726,7 @@ screen. %s/%d are runtime placeholders.
 "text category %q has no primary_subdir in the vendored layout contract — the Go registry has drifted from layout.v1.json; re-run scripts/sync-schema.sh"
 "that doesn't fit. This machine has %s · %s, and tracebloc keeps about %s and %s for itself, so one run can use at most %d cores and %d GiB. Try --cores %d --memory %d."
 "the client running in this namespace is anchored to a different cluster (%s) than --kubeconfig/--context points at (%s) — check you're targeting the right cluster"
+"the cluster API is at %s, not this machine — the host/VM chain describes the Docker VM the CLI runs on, so it cannot speak for a remote k3d cluster"
 "the cluster API server at %s isn't answering — is the cluster running?"
 "the duration/time column name"
 "the label column %q has %d distinct value(s) — a classification dataset needs at least 2 classes. The cluster rejects this after the upload; check the labels and re-run."
@@ -745,6 +761,9 @@ screen. %s/%d are runtime placeholders.
 "unavailable"
 "unknown backend environment %q — valid values are dev, stg, prod (default). Check --env / $CLIENT_ENV"
 "unknown command %q for %q"
+"unparseable MemTotal %q: %w"
+"unparseable NCPU %q: %w"
+"unparseable hw.memsize: %w"
 "upgrade didn't complete (%w). You can run the installer directly:\n    %s"
 "values:"
 "verifying removal of %s: %w"

internal/doctor/doctor.go

@@ -1,3 +1,3 @@
 // Package doctor implements the checks behind `tracebloc cluster doctor`:
 // a read-only, best-effort health sweep of a running tracebloc client
 // cluster. Each check reports ✔/⚠/✖ plus a one-line remedy, so a customer
@@ -137,6 +137,13 @@
 	// HTTPProbe reports whether a URL is reachable from where the CLI runs.
 	// nil => httpProbe (proxy-aware, short timeout). Injected in tests.
 	HTTPProbe func(ctx context.Context, url string) error
+
+	// VMProbe reports the container runtime's VM size, and HostProbe the
+	// physical machine's. nil => dockerVMProbe / hostProbe. Both injected in
+	// tests so the four-level chain is exercisable without Docker
+	// (backend#2221).
+	VMProbe   VMProbe
+	HostProbe HostProbe
 }
 
 // Run executes every check in display order and returns their results. It
@@ -147,6 +154,12 @@
 	if opts.HTTPProbe == nil {
 		opts.HTTPProbe = httpProbe
 	}
+	if opts.VMProbe == nil {
+		opts.VMProbe = dockerVMProbe
+	}
+	if opts.HostProbe == nil {
+		opts.HostProbe = hostProbe
+	}
 	ns := opts.Namespace
 
 	// Discovered once: the first API call. Its error is the reachability signal
@@ -168,6 +181,7 @@
 			unknownCheck("Restart history"),
 			unknownCheck("Dataset volume (PVC)"),
 			unknownCheck("Node capacity"),
+			unknownCheck("Machine capacity"),
 			unknownCheck("Image pull secret"),
 			unknownCheck("Proxy configuration"),
 			checkBackendEgress(ctx, nil, opts.HTTPProbe),
@@ -184,6 +198,7 @@
 		checkRestartHistory(ctx, cs, ns),
 		checkPVC(ctx, cs, ns),
 		checkNodeFit(ctx, cs, jmEnv),
+		checkMachineChain(ctx, cs, opts.ServerURL, opts.VMProbe, opts.HostProbe),
 		checkImagePull(ctx, cs, ns, release),
 		checkProxy(jmEnv),
 		checkBackendEgress(ctx, jmEnv, opts.HTTPProbe),

internal/doctor/doctor_test.go

@@ -547,8 +547,12 @@ func TestRun_HealthyCluster(t *testing.T) {
 		HTTPProbe: func(context.Context, string) error { return nil },
 	})
 
-	if len(results) != 9 {
-		t.Fatalf("want 9 checks, got %d", len(results))
+	// 10 since backend#2221 added "Machine capacity". These nodes are not
+	// k3d-named, so that check reports StatusUnknown — which the rollup ignores,
+	// so the healthy verdict below is unaffected. That is the intended
+	// behaviour on a non-local cluster, not an accident of the fixture.
+	if len(results) != 10 {
+		t.Fatalf("want 10 checks, got %d", len(results))
 	}
 	if w := worstStatus(results); w != StatusOK {
 		for _, r := range results {