From ee7a83224ef1b16a516e6019558175b3a4bffe5c Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Tue, 22 Sep 2026 17:49:28 -0700 Subject: [PATCH] Remove the substrate version label from the install The install labeled every node with ate.dev/substrate-version, named the atelet DaemonSet atelet- and pinned it to nodes carrying its version. This let two substrate versions run side by side on disjoint sets of nodes, which the clone-WorkerPool upgrade runbook relied on. The GA upgrade does not split nodes by version. atelet rolls in place and worker pools roll as ordinary Deployment updates. The label breaks that flow: deploying a new atelet creates a second DaemonSet pinned to a version no node carries, so it schedules nowhere. --- README.md | 14 -- cmd/ate-setup/differences.md | 8 +- cmd/ate-setup/internal/cmd/root.go | 4 +- .../internal/demos/demotest/demotest.go | 3 - cmd/ate-setup/internal/demos/render.go | 8 +- cmd/ate-setup/internal/images/prebuilt.go | 9 +- cmd/ate-setup/internal/kube/resources.go | 15 -- cmd/ate-setup/internal/steps/csi.go | 9 +- cmd/ate-setup/internal/steps/delete.go | 11 +- cmd/ate-setup/internal/steps/deploy.go | 38 +---- cmd/ate-setup/internal/steps/env.go | 5 - cmd/ate-setup/internal/steps/overlay.go | 13 +- cmd/ate-setup/internal/steps/overlay_test.go | 7 +- cmd/ate-setup/internal/steps/publish.go | 10 +- cmd/ate-setup/internal/steps/version.go | 137 ++---------------- cmd/ate-setup/internal/steps/version_test.go | 66 ++------- cmd/objectstore-plugin/main_test.go | 2 +- demos/counter/counter-microvm.yaml.tmpl | 2 - demos/counter/counter.yaml.tmpl | 2 - docs/api-guide.md | 23 --- hack/microvm-assets/README.md | 2 +- hack/setup-csi-hostpath-kind.sh | 3 +- hack/setup-csi-nfs-kind.sh | 3 +- internal/versionlabel/cmd/main.go | 40 ----- internal/versionlabel/versionlabel.go | 99 ------------- internal/versionlabel/versionlabel_test.go | 103 ------------- manifests/ate-install/atelet.yaml | 16 +- .../kind/atelet/kustomization.yaml | 2 +- tools/setup-gcp/README.md | 10 -- .../dashboards/ate-e2e-latency-dashboard.json | 2 +- .../dashboards/ate-grpc-dashboard.json | 6 +- .../dashboards/ate-snapshot-dashboard.json | 10 +- 32 files changed, 65 insertions(+), 617 deletions(-) delete mode 100644 internal/versionlabel/cmd/main.go delete mode 100644 internal/versionlabel/versionlabel.go delete mode 100644 internal/versionlabel/versionlabel_test.go diff --git a/README.md b/README.md index aa85a02f00..3f6d4b021b 100644 --- a/README.md +++ b/README.md @@ -142,15 +142,6 @@ curl -X POST \ -i http://localhost:8000/ ``` -Worker capacity is versioned: the dataplane (the atelet DaemonSet and the -worker pods) schedules only on nodes that carry the -`ate.dev/substrate-version` label, and the install stamps it on every node -that exists when it runs. A node added later hosts no workers until you label -it with the installed version -(`kubectl label node ate.dev/substrate-version=`). -`kubectl get ds -n ate-system -l app=atelet -L ate.dev/substrate-version` -prints the installed version, off the atelet DaemonSet the install created. - ### GKE Quickstart (Development) 1. Create and configure your environment file: @@ -194,11 +185,6 @@ prints the installed version, off the atelet DaemonSet the install created. grant each atespace the namespaces it may read; see [Enable it](docs/egress-credential-injection.md#enable-it). - Nodes that GKE adds later (autoscaling, auto-repair, node upgrades) are - born with the node pool's labels, so the pool needs - `ate.dev/substrate-version` too; see - [Node version labels](tools/setup-gcp/README.md). - 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs. ```bash ./hack/install-ate.sh --deploy-demo-counter diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index 6e92de84d5..832ca52fea 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -127,13 +127,7 @@ unpinned reference is rejected by admission and every demo fails to deploy. Pinning also gives a pre-built install the property a `ko` install had for free, that a tag moving afterwards cannot change what is running. -That tag also becomes the substrate version, which names the atelet DaemonSet -and sets the node label partitioning nodes across coexisting versions. It has -to: that label must describe the atelet actually running, which came from the -image, not the checkout `git describe` happens to be sitting on. `VERSION` -overrides it, as it always has. - -The rewrite is textual, one token at a time, matching `SubstituteVersion` above. +The rewrite is textual, one token at a time. A reference is replaced wherever it appears, so the code needs no list of which fields hold images — `workerImage:` in a CRD spec is rewritten like any other — and the applied manifest differs from the checked-in YAML only in the image diff --git a/cmd/ate-setup/internal/cmd/root.go b/cmd/ate-setup/internal/cmd/root.go index 2b1fd8995f..33c23026e2 100644 --- a/cmd/ate-setup/internal/cmd/root.go +++ b/cmd/ate-setup/internal/cmd/root.go @@ -164,9 +164,7 @@ func recordRun(cmd *cobra.Command, runErr error) { // failed before it had an environment to ask. var version string if env != nil { - if v, _, err := env.SubstrateVersion(); err == nil { - version = v - } + version = env.SubstrateVersion() } var path string diff --git a/cmd/ate-setup/internal/demos/demotest/demotest.go b/cmd/ate-setup/internal/demos/demotest/demotest.go index f320794be0..66395ed54e 100644 --- a/cmd/ate-setup/internal/demos/demotest/demotest.go +++ b/cmd/ate-setup/internal/demos/demotest/demotest.go @@ -32,9 +32,6 @@ import ( // configuration, so they can be exercised without a kubeconfig. func Env(t *testing.T) *steps.Env { t.Helper() - // Pin VERSION so rendering does not depend on whatever tags (e.g. - // archive/*) happen to be in the local git checkout. - t.Setenv("VERSION", "v0.0.0-test") root, err := config.RepoRoot() if err != nil { t.Fatalf("RepoRoot: %v", err) diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go index 970ecab412..d5061fd43e 100644 --- a/cmd/ate-setup/internal/demos/render.go +++ b/cmd/ate-setup/internal/demos/render.go @@ -33,16 +33,10 @@ var ExternalVolumePlaceholders = []string{ "EXTERNAL_VOLUMES", } -// Render expands a demo template with the configured bucket name and the -// build version (pool templates pin worker pods to version-labeled nodes). +// Render expands a demo template with the configured bucket name. func Render(e *steps.Env, relPath string, extraValues map[string]string, drop []string) ([]byte, error) { - version, _, err := e.SubstrateVersion() - if err != nil { - return nil, err - } values := map[string]string{ bucketNamePlaceholder: e.Cfg.BucketName, - "SUBSTRATE_VERSION": version, } for k, v := range extraValues { values[k] = v diff --git a/cmd/ate-setup/internal/images/prebuilt.go b/cmd/ate-setup/internal/images/prebuilt.go index 5ffdb39b45..9f3a5403bb 100644 --- a/cmd/ate-setup/internal/images/prebuilt.go +++ b/cmd/ate-setup/internal/images/prebuilt.go @@ -123,11 +123,10 @@ func (p *Prebuilt) Pin(ctx context.Context, image string) (string, error) { // rewrite replaces every ko:// reference with the image it maps to. // -// Substitution is textual, the same way SubstituteVersion fills -// ${SUBSTRATE_VERSION}. References appear in CRD fields as well as pod specs (a -// WorkerPool's spec.workerImage, an ActorTemplate's image), so a schema walk -// would need to know every such field, and rewriting the bytes leaves -// everything else -- the multi-kilobyte Envoy configuration blocks in +// Substitution is textual. References appear in CRD fields as well as pod +// specs (a WorkerPool's spec.workerImage, an ActorTemplate's image), so a +// schema walk would need to know every such field, and rewriting the bytes +// leaves everything else -- the multi-kilobyte Envoy configuration blocks in // particular -- exactly as committed. func (p *Prebuilt) rewrite(ctx context.Context, manifest []byte) ([]byte, error) { // Collect the failures rather than stopping at the first. Diagnosing them diff --git a/cmd/ate-setup/internal/kube/resources.go b/cmd/ate-setup/internal/kube/resources.go index 8319b6349e..d959a16baf 100644 --- a/cmd/ate-setup/internal/kube/resources.go +++ b/cmd/ate-setup/internal/kube/resources.go @@ -155,21 +155,6 @@ func (c *Client) PatchDeployment(ctx context.Context, namespace, name string, pa return nil } -// DaemonSetNames lists the DaemonSets in a namespace matching a label -// selector. The atelet DaemonSet name carries a substrate version suffix, so -// the installed versions can only be found by label. -func (c *Client) DaemonSetNames(ctx context.Context, namespace, selector string) ([]string, error) { - list, err := c.Typed.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) - if err != nil { - return nil, fmt.Errorf("while listing daemonsets in %s matching %q: %w", namespace, selector, err) - } - names := make([]string, 0, len(list.Items)) - for _, ds := range list.Items { - names = append(names, ds.Name) - } - return names, nil -} - // SetServiceAccountAnnotation adds or removes one annotation on a // ServiceAccount; an empty value removes it. A missing ServiceAccount is not // an error, matching the `|| true` the shell installer removed one under. diff --git a/cmd/ate-setup/internal/steps/csi.go b/cmd/ate-setup/internal/steps/csi.go index 4dcf0914a8..129bdd4b5b 100644 --- a/cmd/ate-setup/internal/steps/csi.go +++ b/cmd/ate-setup/internal/steps/csi.go @@ -20,6 +20,7 @@ import ( "os" "os/exec" "strings" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -290,8 +291,8 @@ spec: // The image cache directories under ateomHostDir were just wiped, so // atelet has to recreate them. - log.Infof("Restarting the atelet DaemonSets (if present)...") - return e.RestartAteletDaemonSets(ctx) + log.Infof("Restarting the atelet DaemonSet (if present)...") + return e.Kube.RolloutRestart(ctx, e.Namespace(), "atelet", time.Now()) } func (e *Env) setupCSINFS(ctx context.Context) error { @@ -422,8 +423,8 @@ spec: return err } - log.Infof("Restarting the atelet DaemonSets (if present)...") - return e.RestartAteletDaemonSets(ctx) + log.Infof("Restarting the atelet DaemonSet (if present)...") + return e.Kube.RolloutRestart(ctx, e.Namespace(), "atelet", time.Now()) } // checkNFSDSupport verifies the host kernel can serve NFS. The in-cluster NFS diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 74568bf8b4..84b4aeecb4 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -64,12 +64,6 @@ func (e *Env) DeleteAteSystem(ctx context.Context) error { return err } - // atelet DaemonSet names carry a version suffix. - if err := e.Kube.Typed.AppsV1().DaemonSets(e.Namespace()).DeleteCollection(ctx, - metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "app=atelet"}); err != nil { - return fmt.Errorf("while deleting atelet daemonsets: %w", err) - } - for _, path := range [][]string{ {"components", "agentgateway", "configmap.yaml"}, {"postgres", "postgres.yaml"}, @@ -94,10 +88,7 @@ func (e *Env) DeleteAteSystem(ctx context.Context) error { if err := e.Kube.WaitDeleted(ctx, namespace, "", NamespacePodCert, e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil { return err } - if err := e.DeleteTrustBundles(ctx); err != nil { - return err - } - return e.UnlabelNodesSubstrateVersion(ctx) + return e.DeleteTrustBundles(ctx) } // releaseSandboxConfigs removes the WorkerPool protection finalizer from diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 77a9f98a2e..b72a90281c 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -64,11 +64,8 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.RequireCanonicalNamespace("deploy ate-system"); err != nil { return err } - // Fail fast on an unusable build version or credential provider selection - // before touching the cluster. - if _, _, err := e.SubstrateVersion(); err != nil { - return err - } + // Fail fast on an unusable credential provider selection before touching + // the cluster. provider, err := e.Cfg.CredentialProvider() if err != nil { return err @@ -85,12 +82,6 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - // Before the bundle: the atelet DaemonSet applied below and the demo - // WorkerPools' version-pinned pods schedule only to version-labeled nodes. - if err := e.LabelNodesSubstrateVersion(ctx); err != nil { - return err - } - // DeployCRDs, not EnsureCRDs: an existence check would skip upgrades, // stranding stale CRD schemas and RBAC (role.yaml has no other apply // path). @@ -139,12 +130,6 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err != nil { return err } - // The atelet DaemonSet in the bundle is version-keyed; fill its - // placeholders after the render (kustomize and ko pass them through). - manifests, err = e.SubstituteVersion(manifests) - if err != nil { - return err - } if err := e.applyWithAPIServerEnvHash(ctx, manifests); err != nil { return err } @@ -172,10 +157,6 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - ateletName, err := e.AteletDaemonSetName() - if err != nil { - return err - } log.Step("Waiting for ATE system components to be ready...") type rollout struct{ kind, name string } var waits []rollout @@ -184,7 +165,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { rollout{kube.KindDeployment, "ate-controller"}, rollout{kube.KindDeployment, "atenet-router"}, rollout{kube.KindDeployment, "atenet-egress"}, - rollout{kube.KindDaemonSet, ateletName}, + rollout{kube.KindDaemonSet, "atelet"}, ) if provider.Kubernetes() { waits = append(waits, rollout{kube.KindDeployment, k8sCredentialProviderDeployment}) @@ -354,9 +335,6 @@ func (e *Env) DeployAtelet(ctx context.Context) error { if err := e.EnsureAteSystemNamespace(ctx); err != nil { return err } - if err := e.LabelNodesSubstrateVersion(ctx); err != nil { - return err - } if err := e.applyOtelConfig(ctx); err != nil { return err } @@ -375,18 +353,10 @@ func (e *Env) DeployAtelet(ctx context.Context) error { if err != nil { return err } - manifest, err = e.SubstituteVersion(manifest) - if err != nil { - return err - } if err := e.Kube.ApplyBytes(ctx, manifest); err != nil { return err } - ateletName, err := e.AteletDaemonSetName() - if err != nil { - return err - } - return e.Kube.RolloutStatus(ctx, kube.KindDaemonSet, e.Namespace(), ateletName, e.Cfg.RolloutTimeout) + return e.Kube.RolloutStatus(ctx, kube.KindDaemonSet, e.Namespace(), "atelet", e.Cfg.RolloutTimeout) } // DeployAtenet redeploys the atenet dataplane: router and egress. diff --git a/cmd/ate-setup/internal/steps/env.go b/cmd/ate-setup/internal/steps/env.go index 1e22e5b889..1591e77fd8 100644 --- a/cmd/ate-setup/internal/steps/env.go +++ b/cmd/ate-setup/internal/steps/env.go @@ -81,11 +81,6 @@ type Env struct { // work on a machine with no container registry credentials configured, and // the build-from-source resolver fails at construction without them. resolver imageResolver - - // substrateVersion caches the derived build version and its object-name - // suffix; see SubstrateVersion. - substrateVersion string - substrateVersionSuffix string } // Namespace is the namespace the control plane is installed into. It is diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 9f884d5ee6..7e5081fc53 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -529,16 +529,5 @@ func (e *Env) applyOtelEndpointOverride(ctx context.Context) error { return err } } - // atelet DaemonSet names carry a version suffix; restart whichever - // versions are installed. - daemonSets, err := e.Kube.DaemonSetNames(ctx, e.Namespace(), "app=atelet") - if err != nil { - return err - } - for _, name := range daemonSets { - if err := e.Kube.RolloutRestart(ctx, e.Namespace(), name, now); err != nil { - return err - } - } - return nil + return e.Kube.RolloutRestart(ctx, e.Namespace(), "atelet", now) } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index b213fbfa96..57881c6ae8 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -772,13 +772,10 @@ func TestApplyOtelEndpointOverride(t *testing.T) { }) t.Run("patches and restarts the consumers", func(t *testing.T) { - // The atelet DaemonSet name carries a substrate version suffix, so it - // can only be found by label. atelet := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: NamespaceAteSystem, - Name: "atelet-v1-2-3", - Labels: map[string]string{"app": "atelet"}, + Name: "atelet", }, } e := &Env{ @@ -799,7 +796,7 @@ func TestApplyOtelEndpointOverride(t *testing.T) { if !restartedAt(t, e, "deployment", "ate-api-server") { t.Error("ate-api-server was not restarted") } - if !restartedAt(t, e, "daemonset", "atelet-v1-2-3") { + if !restartedAt(t, e, "daemonset", "atelet") { t.Error("the atelet DaemonSet was not restarted") } }) diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index e43f207a6f..d25ed6c493 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -34,10 +34,7 @@ var workerImages = []string{"ateom-gvisor", "ateom-microvm"} // output. A WorkerPool moves to this build by pointing its workerImage at // the ref. func (e *Env) PublishWorkerImages(ctx context.Context, w io.Writer) error { - version, _, err := e.SubstrateVersion() - if err != nil { - return err - } + version := e.SubstrateVersion() log.Stepf("publish_worker_images (%s)", version) runner, err := e.koRunner() if err != nil { @@ -71,10 +68,7 @@ func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer) error { if e.Cfg.KODockerRepo == "" { return fmt.Errorf("publishing release images needs a registry to push to; set KO_DOCKER_REPO or --ko-docker-repo") } - tag, _, err := e.SubstrateVersion() - if err != nil { - return err - } + tag := e.SubstrateVersion() log.Stepf("publish_release_images (%s)", tag) runner, err := e.koRunner() if err != nil { diff --git a/cmd/ate-setup/internal/steps/version.go b/cmd/ate-setup/internal/steps/version.go index 8ac5ac72b7..5fb94f21d9 100644 --- a/cmd/ate-setup/internal/steps/version.go +++ b/cmd/ate-setup/internal/steps/version.go @@ -15,139 +15,28 @@ package steps import ( - "context" - "fmt" "os" - "regexp" "strings" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/ko" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/internal/versionlabel" ) -// SubstrateVersion returns the build version and the object-name suffix for atelet. -// The version is the same one ko stamps into the binaries. +// SubstrateVersion returns the build version, the same one ko stamps into the +// binaries. // // With --image-repo nothing is built, so `git describe` would report the -// checkout rather than the images being installed. The image tag is the version -// in that case. It has to be: the version names the atelet DaemonSet and sets -// the node label that partitions nodes across coexisting versions, so it must -// describe the atelet that is actually running, which came from the image. -// VERSION still wins over both, so a tag that is not a valid label value can be -// worked around the same way. -func (e *Env) SubstrateVersion() (version, suffix string, err error) { - if e.substrateVersion != "" { - return e.substrateVersion, e.substrateVersionSuffix, nil - } - var raw string +// checkout rather than the images being installed. The image tag is the +// version in that case. VERSION still wins over both. +func (e *Env) SubstrateVersion() string { if e.Cfg.Images.IsPrebuilt() { - raw = os.Getenv("VERSION") - if raw == "" { - // The tag may carry its digest (v1@sha256:...); the version - // is the tag alone. - raw, _, _ = strings.Cut(e.Cfg.Images.Tag, "@") - } - } else { - // ko.BuildVersion consults VERSION itself before `git describe`. - raw = ko.BuildVersion(e.Cfg.Root) - } - if v := versionlabel.Value(raw); v != raw { - return "", "", fmt.Errorf("build version %q is not a valid label value (it would sanitize to %q); pin a label-safe one with the VERSION env var", raw, v) - } - e.substrateVersion = raw - e.substrateVersionSuffix = versionlabel.NameSuffix(raw) - return e.substrateVersion, e.substrateVersionSuffix, nil -} - -func (e *Env) AteletDaemonSetName() (string, error) { - _, suffix, err := e.SubstrateVersion() - if err != nil { - return "", err - } - return "atelet-" + suffix, nil -} - -// unquotedVersionScalar matches a whole-scalar ${SUBSTRATE_VERSION} value -// kustomize re-emitted unquoted. It is re-quoted before substitution so an -// all-digit version lands as a YAML string, not an integer. -var unquotedVersionScalar = regexp.MustCompile(`(?m)^(\s*[^:\n]+): \$\{SUBSTRATE_VERSION\}$`) - -// SubstituteVersion fills the ${SUBSTRATE_VERSION} and -// ${SUBSTRATE_VERSION_SUFFIX} placeholders in a rendered manifest. -func (e *Env) SubstituteVersion(manifest []byte) ([]byte, error) { - version, suffix, err := e.SubstrateVersion() - if err != nil { - return nil, err - } - s := unquotedVersionScalar.ReplaceAllString(string(manifest), `$1: "$${SUBSTRATE_VERSION}"`) - s = strings.ReplaceAll(s, "${SUBSTRATE_VERSION}", version) - s = strings.ReplaceAll(s, "${SUBSTRATE_VERSION_SUFFIX}", suffix) - return []byte(s), nil -} - -// RestartAteletDaemonSets pod-restarts every atelet DaemonSet by label. -func (e *Env) RestartAteletDaemonSets(ctx context.Context) error { - list, err := e.Kube.Typed.AppsV1().DaemonSets(e.Namespace()).List(ctx, metav1.ListOptions{ - LabelSelector: "app=atelet", - }) - if err != nil { - return fmt.Errorf("while listing atelet daemonsets: %w", err) - } - for _, ds := range list.Items { - if err := e.Kube.RolloutRestart(ctx, e.Namespace(), ds.Name, time.Now()); err != nil { - return err - } - } - return nil -} - -// LabelNodesSubstrateVersion stamps ate.dev/substrate-version on every node -// that does not carry it yet. Nodes that already carry the label keep their -// value: during a rolling upgrade the operator owns the per-node value, and -// an install must not yank nodes across versions behind its back. -func (e *Env) LabelNodesSubstrateVersion(ctx context.Context) error { - version, _, err := e.SubstrateVersion() - if err != nil { - return err - } - log.Stepf("label_nodes_substrate_version (%s)", version) - nodes, err := e.Kube.Typed.CoreV1().Nodes().List(ctx, metav1.ListOptions{ - LabelSelector: "!" + versionlabel.Key, - }) - if err != nil { - return fmt.Errorf("while listing unlabeled nodes: %w", err) - } - patch := fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, versionlabel.Key, version) - for _, node := range nodes.Items { - if _, err := e.Kube.Typed.CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { - return fmt.Errorf("while labeling node %s: %w", node.Name, err) - } - } - return nil -} - -// UnlabelNodesSubstrateVersion clears ate.dev/substrate-version from every -// node that carries it. Uninstall removes the label with the system: the -// install won't relabels a labeled node, so a leftover label would pin the -// next install to the old version world. -func (e *Env) UnlabelNodesSubstrateVersion(ctx context.Context) error { - log.Step("unlabel_nodes_substrate_version") - nodes, err := e.Kube.Typed.CoreV1().Nodes().List(ctx, metav1.ListOptions{ - LabelSelector: versionlabel.Key, - }) - if err != nil { - return fmt.Errorf("while listing labeled nodes: %w", err) - } - patch := fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, versionlabel.Key) - for _, node := range nodes.Items { - if _, err := e.Kube.Typed.CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { - return fmt.Errorf("while unlabeling node %s: %w", node.Name, err) + if v := os.Getenv("VERSION"); v != "" { + return v } + // The tag may carry its digest (v1@sha256:...); the version + // is the tag alone. + v, _, _ := strings.Cut(e.Cfg.Images.Tag, "@") + return v } - return nil + // ko.BuildVersion consults VERSION itself before `git describe`. + return ko.BuildVersion(e.Cfg.Root) } diff --git a/cmd/ate-setup/internal/steps/version_test.go b/cmd/ate-setup/internal/steps/version_test.go index d0772c1095..c1674e7eab 100644 --- a/cmd/ate-setup/internal/steps/version_test.go +++ b/cmd/ate-setup/internal/steps/version_test.go @@ -21,72 +21,36 @@ import ( "github.com/agent-substrate/substrate/cmd/ate-setup/internal/images" ) -// TestSubstrateVersionPrebuilt covers the version a prebuilt install stamps. +// TestSubstrateVersionPrebuilt covers the version a prebuilt install reports. // Root points at a directory that is not a checkout, so `git describe` has // nothing to say and only the image tag can supply it. func TestSubstrateVersionPrebuilt(t *testing.T) { t.Setenv("VERSION", "") cases := []struct { - name string - source images.Source - want string - wantSuffix string + name string + source images.Source + want string }{{ - name: "tag is the version", - source: images.Source{Repo: "example.com/substrate", Tag: "v0.0.0-503-4b3423c0"}, - want: "v0.0.0-503-4b3423c0", - wantSuffix: "v0-0-0-503-4b3423c0", + name: "tag is the version", + source: images.Source{Repo: "example.com/substrate", Tag: "v0.0.0-503-4b3423c0"}, + want: "v0.0.0-503-4b3423c0", }, { - name: "a tag carrying its digest is the version without it", - source: images.Source{Repo: "example.com/substrate", Tag: "v0.0.0@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, - want: "v0.0.0", - wantSuffix: "v0-0-0", + name: "a tag carrying its digest is the version without it", + source: images.Source{Repo: "example.com/substrate", Tag: "v0.0.0@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, + want: "v0.0.0", }, { - name: "built from source falls back to git", - source: images.Source{}, - want: "dev", - wantSuffix: "dev", + name: "built from source falls back to git", + source: images.Source{}, + want: "dev", }} for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { e := &Env{Cfg: &config.Config{Root: t.TempDir(), Images: tc.source}} - got, suffix, err := e.SubstrateVersion() - if err != nil { - t.Fatalf("SubstrateVersion: %v", err) - } - if got != tc.want || suffix != tc.wantSuffix { - t.Fatalf("SubstrateVersion = %q, %q; want %q, %q", got, suffix, tc.want, tc.wantSuffix) + if got := e.SubstrateVersion(); got != tc.want { + t.Fatalf("SubstrateVersion = %q, want %q", got, tc.want) } }) } } - -func TestSubstituteVersion(t *testing.T) { - e := &Env{substrateVersion: "v1.2.3", substrateVersionSuffix: "v1-2-3"} - in := "" + - "metadata:\n" + - " name: atelet-${SUBSTRATE_VERSION_SUFFIX}\n" + - " labels:\n" + - " ate.dev/substrate-version: ${SUBSTRATE_VERSION}\n" + - "nodeSelector:\n" + - " ate.dev/substrate-version: \"${SUBSTRATE_VERSION}\"\n" - // The unquoted scalar is re-quoted (an all-digit version must land as a - // YAML string); already-quoted values and name suffixes pass through. - want := "" + - "metadata:\n" + - " name: atelet-v1-2-3\n" + - " labels:\n" + - " ate.dev/substrate-version: \"v1.2.3\"\n" + - "nodeSelector:\n" + - " ate.dev/substrate-version: \"v1.2.3\"\n" - - got, err := e.SubstituteVersion([]byte(in)) - if err != nil { - t.Fatalf("SubstituteVersion: %v", err) - } - if string(got) != want { - t.Fatalf("SubstituteVersion mismatch:\ngot:\n%s\nwant:\n%s", got, want) - } -} diff --git a/cmd/objectstore-plugin/main_test.go b/cmd/objectstore-plugin/main_test.go index 07b9cd1f9b..a939cc614e 100644 --- a/cmd/objectstore-plugin/main_test.go +++ b/cmd/objectstore-plugin/main_test.go @@ -58,7 +58,7 @@ func TestInstallStagingCapacityFitsVolume(t *testing.T) { if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { t.Fatal(err) } - if obj.Kind == "DaemonSet" && strings.HasPrefix(obj.Name, "atelet-") { + if obj.Kind == "DaemonSet" && obj.Name == "atelet" { ds = &obj } } diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index aa68bdd1f0..08e1048abb 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -45,8 +45,6 @@ spec: name: microvm workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm template: - nodeSelector: - ate.dev/substrate-version: "${SUBSTRATE_VERSION}" # Per-worker resources: these size the worker POD and advertise its # scheduling capacity (the per-actor ceiling). The micro-VM sandbox itself is # sized by the ActorTemplate's resources in counter-microvm-template.yaml.tmpl, diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index dea8cb2541..4cc0ea0516 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -44,8 +44,6 @@ spec: sandboxClasses: - name: gvisor template: - nodeSelector: - ate.dev/substrate-version: "${SUBSTRATE_VERSION}" # Per-worker resources: these size the worker POD and advertise its # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by # the ActorTemplate's resources in counter-template.yaml.tmpl, not by these diff --git a/docs/api-guide.md b/docs/api-guide.md index d87e300aa6..3bc3722649 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -37,29 +37,6 @@ syntax. metadata; they do not affect actor scheduling. Actor selectors match `WorkerPool.metadata.labels`, not `WorkerPool.spec.template.labels`. -#### Pin pools to the installed substrate version (`template.nodeSelector`) - -The installation labels every node with the substrate build version it deployed. -Set the same label as a `nodeSelector` on every pool, so its workers only run -on nodes of that version: - -```yaml -spec: - template: - nodeSelector: - ate.dev/substrate-version: "" -``` - -Read the installed version off the atelet DaemonSet: - -```bash -kubectl get ds -n ate-system -l app=atelet -L ate.dev/substrate-version -``` - -The pin is critical to make a rolling upgrade possible. An upgrade moves nodes to -the new version one at a time, deleting each node's old worker pods once it -moves. A pinned pool cannot put those pods back on a moved node, so the old -version drains away node by node. An unpinned pool breaks this constraints. #### Worker Capacity (`spec.template.resources`) Setting `resources.limits` (CPU and Memory) on a `WorkerPool` establishes each worker pod's **capacity** — the envelope its actor sandboxes share, taken from the `ateom` container's limits. The scheduler only places an actor on a worker whose remaining capacity is `>=` the actor's declared resource limits (see [Sandbox Right-Sizing](#sandbox-right-sizing-resources) on the `ActorTemplate`). diff --git a/hack/microvm-assets/README.md b/hack/microvm-assets/README.md index d4b9f26af8..3fe5b35534 100644 --- a/hack/microvm-assets/README.md +++ b/hack/microvm-assets/README.md @@ -42,7 +42,7 @@ it as a device, which is what places micro-VM workers there. 4. **Apply the demo + drive it:** ```sh - BUCKET_NAME=ate-snapshots SUBSTRATE_VERSION="$(git describe --tags --always --dirty)" envsubst < demos/counter/counter-microvm.yaml.tmpl | kubectl apply -f - # the pool pins workers to nodes labeled with this version + BUCKET_NAME=ate-snapshots envsubst < demos/counter/counter-microvm.yaml.tmpl | kubectl apply -f - ``` Create an actor from `counter-microvm`, hit the in-RAM counter to increment it, suspend (checkpoint), resume on a different worker pod, and confirm the count continues — proving the diff --git a/hack/setup-csi-hostpath-kind.sh b/hack/setup-csi-hostpath-kind.sh index 514d9ae64c..f0a0a6e95b 100755 --- a/hack/setup-csi-hostpath-kind.sh +++ b/hack/setup-csi-hostpath-kind.sh @@ -134,8 +134,7 @@ spec: EOF # 9. Restart atelet to recreate image cache directories if they were wiped. -# atelet DaemonSet names carry a build-version suffix, so select by label. echo "Restarting atelet DaemonSet (if present)..." -kubectl rollout restart daemonset -l app=atelet -n ate-system >/dev/null 2>&1 || true +kubectl rollout restart daemonset/atelet -n ate-system >/dev/null 2>&1 || true echo "CSI Hostpath setup complete!" diff --git a/hack/setup-csi-nfs-kind.sh b/hack/setup-csi-nfs-kind.sh index 68a41e011b..eeec5a61ac 100755 --- a/hack/setup-csi-nfs-kind.sh +++ b/hack/setup-csi-nfs-kind.sh @@ -119,8 +119,7 @@ spec: EOF # 9. Restart atelet to ensure it reconnects to the new CSI socket. -# atelet DaemonSet names carry a build-version suffix, so select by label. echo "Restarting atelet DaemonSet (if present)..." -kubectl rollout restart daemonset -l app=atelet -n ate-system >/dev/null 2>&1 || true +kubectl rollout restart daemonset/atelet -n ate-system >/dev/null 2>&1 || true echo "CSI NFS setup complete!" diff --git a/internal/versionlabel/cmd/main.go b/internal/versionlabel/cmd/main.go deleted file mode 100644 index afe127f070..0000000000 --- a/internal/versionlabel/cmd/main.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Command cmd prints "