From 07b36c580aaec1f45baeeb8517ecf77110931752 Mon Sep 17 00:00:00 2001 From: sabnaj Date: Fri, 30 Oct 2026 20:36:27 +0600 Subject: [PATCH 1/4] Add kubectl dba adopt milvus: adopt a running Milvus in place Adopting a Distributed Milvus 2.6 that runs from a Helm chart was a manual procedure: gather the effective config, write three Secrets and a staged Milvus, diff the rendered config, then scale each role by hand. The new command group does it: - plan RELEASE: read-only. Finds the release's workloads (by app.kubernetes.io/instance and the component label, or --workload), merges the mounted config files in Milvus' load order, finds the client Service and the MilvusVersion for the running image, and checks the WAL (Woodpecker on object storage), etcd, object storage and the root password, logging in to a running proxy through a port-forward (Milvus applies a root password only on first start). --output-dir writes the objects for review. - run RELEASE: creates the Secrets, a state ConfigMap -adopt and the Milvus with every role at 0 (orphan-on-delete, DoNotTerminate, external etcd with the current root path, autoOps off, write check off), removes a chart Service on a name KubeDB needs headless, and stops if the config the operator rendered differs from the running one on storage, etcd root path, WAL, security or ports. Then hands the roles over, overlap (default; one QueryNode at a time) or stop-start, recording each one; re-running resumes. - status RELEASE: where each role runs. - rollback RELEASE: deletes the Milvus with orphan-on-delete, moves each role back to the chart starting the chart's pods first, restores the client Service's selector, labels and ports, and with --cleanup deletes what KubeDB left and recreates the removed chart Services. --name covers a Milvus adopted by hand. Several workloads of one role become node groups (e.g. QueryNodes on two GPU pools). Tested end to end on a cluster running the upstream chart 5.0.19 (Milvus 2.6.15): hand-adoption rollback, plan, overlap run, state rollback, stop-start run killed and resumed; data bit-exact after each. Signed-off-by: sabnaj --- pkg/adopt/adopt.go | 43 +++ pkg/adopt/findings.go | 61 ++++ pkg/adopt/milvus.go | 555 +++++++++++++++++++++++++++++++++++ pkg/adopt/milvus_auth.go | 114 +++++++ pkg/adopt/milvus_cluster.go | 362 +++++++++++++++++++++++ pkg/adopt/milvus_config.go | 264 +++++++++++++++++ pkg/adopt/milvus_handover.go | 506 +++++++++++++++++++++++++++++++ pkg/adopt/milvus_plan.go | 350 ++++++++++++++++++++++ pkg/adopt/milvus_render.go | 356 ++++++++++++++++++++++ pkg/adopt/milvus_source.go | 543 ++++++++++++++++++++++++++++++++++ pkg/adopt/milvus_test.go | 507 ++++++++++++++++++++++++++++++++ pkg/cmds/adopt.go | 26 ++ pkg/cmds/root.go | 6 + 13 files changed, 3693 insertions(+) create mode 100644 pkg/adopt/adopt.go create mode 100644 pkg/adopt/findings.go create mode 100644 pkg/adopt/milvus.go create mode 100644 pkg/adopt/milvus_auth.go create mode 100644 pkg/adopt/milvus_cluster.go create mode 100644 pkg/adopt/milvus_config.go create mode 100644 pkg/adopt/milvus_handover.go create mode 100644 pkg/adopt/milvus_plan.go create mode 100644 pkg/adopt/milvus_render.go create mode 100644 pkg/adopt/milvus_source.go create mode 100644 pkg/adopt/milvus_test.go create mode 100644 pkg/cmds/adopt.go diff --git a/pkg/adopt/adopt.go b/pkg/adopt/adopt.go new file mode 100644 index 000000000..d5c04026b --- /dev/null +++ b/pkg/adopt/adopt.go @@ -0,0 +1,43 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package adopt implements `kubectl dba adopt`, which brings a database that +// already runs in Kubernetes (for example from a Helm chart) under KubeDB in +// place: the data stays where it is and the workloads change hands one role at +// a time. +package adopt + +import ( + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdAdopt returns the `adopt` command group. +func NewCmdAdopt(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "adopt", + Short: i18n.T("Bring a running database under KubeDB in place, without moving its data"), + Long: templates.LongDesc(` + Take over a database that already runs in Kubernetes, for example one + installed by a Helm chart. The data stays in its current storage, the + credentials stay the same, and clients keep their address; only the + workloads change hands, one role at a time.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand(NewCmdMilvus(f)) + return cmd +} diff --git a/pkg/adopt/findings.go b/pkg/adopt/findings.go new file mode 100644 index 000000000..aedfeb79b --- /dev/null +++ b/pkg/adopt/findings.go @@ -0,0 +1,61 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "fmt" + "io" +) + +const ( + levelError = "ERROR" + levelWarn = "WARN" + levelInfo = "OK" +) + +type finding struct { + Level string + Msg string +} + +// findings are the results of the pre-flight checks. An ERROR blocks the +// adoption; a WARN needs a look before the handover. +type findings []finding + +func (f *findings) errorf(format string, a ...any) { + *f = append(*f, finding{levelError, fmt.Sprintf(format, a...)}) +} + +func (f *findings) warnf(format string, a ...any) { + *f = append(*f, finding{levelWarn, fmt.Sprintf(format, a...)}) +} + +func (f *findings) infof(format string, a ...any) { + *f = append(*f, finding{levelInfo, fmt.Sprintf(format, a...)}) +} + +func (f findings) hasErrors() bool { + for _, x := range f { + if x.Level == levelError { + return true + } + } + return false +} + +func (f findings) print(w io.Writer) { + for _, x := range f { + _, _ = fmt.Fprintf(w, " [%-5s] %s\n", x.Level, x.Msg) + } +} diff --git a/pkg/adopt/milvus.go b/pkg/adopt/milvus.go new file mode 100644 index 000000000..8e6e5adb8 --- /dev/null +++ b/pkg/adopt/milvus.go @@ -0,0 +1,555 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdMilvus returns `adopt milvus`. +func NewCmdMilvus(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "milvus", + Short: i18n.T("Adopt a running Distributed Milvus 2.6 in place"), + Long: templates.LongDesc(` + Adopt a Distributed Milvus 2.6 that runs from a Helm chart (or any set of + Deployments and StatefulSets) in place. The metadata stays in its etcd, the + segments, indexes and write-ahead log stay in its bucket, the root + credentials and the client Service stay the same. + + * plan: check the cluster and show what an adoption creates (read-only) + * run: create the Milvus and hand the roles over, one at a time + * status: show where each role runs + * rollback: give the cluster back to the chart's workloads + + RELEASE names the running cluster: by default its workloads are those with + app.kubernetes.io/instance=RELEASE, and the "component" label names each + workload's role, as in the upstream Milvus Helm chart.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand(newCmdMilvusPlan(f), newCmdMilvusRun(f), newCmdMilvusStatus(f), newCmdMilvusRollback(f)) + return cmd +} + +func addSourceFlags(cmd *cobra.Command, o *sourceOptions) { + cmd.Flags().StringVar(&o.Selector, "selector", "", "Label selector of the cluster's workloads (default app.kubernetes.io/instance=RELEASE)") + cmd.Flags().StringVar(&o.RoleLabel, "role-label", "component", "Pod label that names a workload's Milvus role") + cmd.Flags().StringArrayVar(&o.Workloads, "workload", nil, "Map a workload to a role, as role[/group]=Kind/name (repeat for each workload; replaces --selector)") +} + +func addPlanFlags(cmd *cobra.Command, o *planOptions) { + addSourceFlags(cmd, &o.sourceOptions) + cmd.Flags().StringVar(&o.Name, "name", "", "Name of the Milvus object (default: the name of the Service clients use, which KubeDB then takes over)") + cmd.Flags().StringVar(&o.Version, "version", "", "MilvusVersion to use (default: the one matching the running image)") + cmd.Flags().StringVar(&o.AuthSecret, "auth-secret", "", "Existing basic-auth Secret with the current root user and password") + cmd.Flags().StringVar(&o.RootPasswordFile, "root-password-file", "", "File holding the current root password (default: common.security.defaultRootPassword of the running config)") + cmd.Flags().StringVar(&o.EtcdAuthSecret, "etcd-auth-secret", "", "Basic-auth Secret for an etcd that uses authentication") + cmd.Flags().StringVar(&o.ConfigFile, "config-file", "", "The effective milvus.yaml, when it is not mounted from a ConfigMap or Secret") + cmd.Flags().BoolVar(&o.RestrictedSCC, "restricted-scc", false, "Set spec.restrictedSCC (OpenShift)") + cmd.Flags().BoolVar(&o.SkipCredentialCheck, "skip-credential-check", false, "Do not log in to the running cluster to check the root credentials") +} + +func addTimingFlags(cmd *cobra.Command, t *timing) { + cmd.Flags().DurationVar(&t.Timeout, "timeout", 15*time.Minute, "Longest wait for one step (pods Ready or gone)") + cmd.Flags().DurationVar(&t.Settle, "settle", 20*time.Second, "Pause after each role") + cmd.Flags().DurationVar(&t.QueryNodeDelay, "querynode-delay", time.Minute, "Overlap: time a new QueryNode serves before an old one stops (raise it for large collections)") +} + +func releaseArg(args []string) (string, error) { + if len(args) != 1 { + return "", errors.New("give the RELEASE (the Helm release, or the instance label value of the workloads)") + } + return args[0], nil +} + +func newCmdMilvusPlan(f cmdutil.Factory) *cobra.Command { + var o planOptions + var outDir string + cmd := &cobra.Command{ + Use: "plan RELEASE", + Short: i18n.T("Check a running Milvus and show what adopting it creates"), + Long: templates.LongDesc(` + Read-only. Finds the workloads, effective configuration, image and client + Service of the running cluster, checks everything an in-place adoption + depends on (Woodpecker WAL, etcd root path, object storage, a matching + MilvusVersion, the current root password), and shows the Secrets, the + state ConfigMap and the staged Milvus that "run" would create. + + With --output-dir the objects are written there, unredacted. Edit them if + needed and apply them; "run" then uses what exists.`), + Example: templates.Examples(` + kubectl dba adopt milvus plan my-release -n milvus + kubectl dba adopt milvus plan my-release -n milvus --output-dir ./adopt`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + var err error + if o.Release, err = releaseArg(args); err != nil { + return err + } + ctx := context.Background() + c, err := newCluster(f, cmd.OutOrStdout()) + if err != nil { + return err + } + p, err := buildPlan(ctx, c, o) + if err != nil { + return err + } + out := cmd.OutOrStdout() + _, _ = fmt.Fprintf(out, "Pre-flight checks for %s in namespace %s:\n", o.Release, c.ns) + p.Findings.print(out) + if p.Findings.hasErrors() { + return errors.New("the adoption is blocked; fix the ERROR findings") + } + docs, err := p.yamlDocs(outDir == "") + if err != nil { + return err + } + if outDir != "" { + if err := os.MkdirAll(outDir, 0o700); err != nil { + return err + } + for _, name := range sortedKeys(docs) { + if err := os.WriteFile(filepath.Join(outDir, name), []byte(docs[name]), 0o600); err != nil { + return err + } + } + _, _ = fmt.Fprintf(out, "\nWrote %s to %s\n", strings.Join(sortedKeys(docs), ", "), outDir) + } else { + for _, name := range sortedKeys(docs) { + _, _ = fmt.Fprintf(out, "\n# %s\n%s", name, docs[name]) + } + } + printHandoverPlan(out, p.State, methodOverlap) + _, _ = fmt.Fprintf(out, "\nNext: kubectl dba adopt milvus run %s -n %s --yes\n", o.Release, c.ns) + return nil + }, + } + addPlanFlags(cmd, &o) + cmd.Flags().StringVar(&outDir, "output-dir", "", "Write the objects to this directory (Secrets unredacted, mode 0600)") + return cmd +} + +func printHandoverPlan(out io.Writer, st *State, method string) { + _, _ = fmt.Fprintf(out, "\nHandover (%s), in this order:\n", method) + for _, role := range handoverOrder(method) { + for _, u := range st.unitsOf(role) { + if st.isDone(u) { + continue + } + how := "start KubeDB's, wait Ready, then stop the chart's" + if method == methodStopStart { + how = "stop the chart's, then start KubeDB's" + } else if u.Role == roleQueryNode && u.Replicas > 1 { + how = "one QueryNode at a time: start one, let it serve, stop one" + } + _, _ = fmt.Fprintf(out, " %-24s %d replicas: %s\n", u.key(), u.Replicas, how) + } + } +} + +type runOptions struct { + planOptions + timing + Method string + Yes bool + SkipConfigCheck bool + ReadyTimeout time.Duration +} + +func newCmdMilvusRun(f cmdutil.Factory) *cobra.Command { + var o runOptions + cmd := &cobra.Command{ + Use: "run RELEASE", + Short: i18n.T("Adopt a running Milvus: create the Milvus object and hand the roles over"), + Long: templates.LongDesc(` + Runs the checks of "plan", creates the Secrets, the state ConfigMap + NAME-adopt (NAME is the Milvus) and the Milvus with every role at 0 replicas, and waits for + the operator. Before any pod moves, it compares the config the operator + rendered with the running one on every setting that decides where the data + is read from, and stops on a difference. + + Then it hands the roles over one at a time: + + * --method overlap (default): start KubeDB's pods of a role, wait until they + are Ready, then stop the chart's. QueryNodes move one at a time. No role + stops serving while it moves; it needs room for one extra set of pods of + a role (one extra QueryNode). + * --method stop-start: stop the chart's pods of a role, then start KubeDB's. + Each role is briefly unavailable; no extra capacity is needed. + + Progress is recorded in the state ConfigMap: if the command stops, run it + again and it continues where it was. "rollback" gives the cluster back. + + Do not remove the chart with "helm uninstall" afterwards: it would delete + the client Service that KubeDB now uses.`), + Example: templates.Examples(` + kubectl dba adopt milvus run my-release -n milvus --yes + kubectl dba adopt milvus run my-release -n milvus --method stop-start --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + var err error + if o.Release, err = releaseArg(args); err != nil { + return err + } + if o.Method != methodOverlap && o.Method != methodStopStart { + return fmt.Errorf("--method must be %s or %s", methodOverlap, methodStopStart) + } + c, err := newCluster(f, cmd.OutOrStdout()) + if err != nil { + return err + } + return runAdoption(context.Background(), c, o) + }, + } + addPlanFlags(cmd, &o.planOptions) + addTimingFlags(cmd, &o.timing) + cmd.Flags().StringVar(&o.Method, "method", methodOverlap, "Handover method: overlap or stop-start") + cmd.Flags().BoolVar(&o.Yes, "yes", false, "Proceed; without it, run only shows what it would do") + cmd.Flags().BoolVar(&o.SkipConfigCheck, "skip-config-check", false, "Do not stop when the rendered config differs from the running one") + cmd.Flags().DurationVar(&o.ReadyTimeout, "ready-timeout", 15*time.Minute, "Longest wait for the operator, and for the Milvus to be Ready at the end") + return cmd +} + +func runAdoption(ctx context.Context, c *cluster, o runOptions) error { + out := c.out + st, _, err := c.loadState(ctx, o.Release) + if err != nil { + return err + } + var p *adoptPlan + if st == nil || st.Phase == phasePlanned { + o.allowExisting = st != nil + if p, err = buildPlan(ctx, c, o.planOptions); err != nil { + return err + } + _, _ = fmt.Fprintf(out, "Pre-flight checks for %s in namespace %s:\n", o.Release, c.ns) + p.Findings.print(out) + if p.Findings.hasErrors() { + return errors.New("the adoption is blocked; fix the ERROR findings") + } + if st == nil { + st = p.State + } + } else { + _, _ = fmt.Fprintf(out, "Resuming the adoption of %s as Milvus %s (%s, %s); handed over: %s\n", + o.Release, st.Name, st.Phase, st.Method, orNone(strings.Join(st.Done, ", "))) + if st.Method != "" && st.Method != o.Method { + c.logf("continuing with --method %s (started with %s)", o.Method, st.Method) + } + } + if st.Phase == phaseAdopted { + _, _ = fmt.Fprintf(out, "Milvus %s is already adopted.\n", st.Name) + return nil + } + if !o.Yes { + printHandoverPlan(out, st, o.Method) + return errors.New("re-run with --yes to proceed") + } + + if p != nil { + if err := c.createObjects(ctx, p); err != nil { + return err + } + if err := c.removeGoverningServices(ctx, st, p.GoverningServices); err != nil { + return err + } + if _, err := c.createMilvus(ctx, p, o.ReadyTimeout); err != nil { + return err + } + if !o.SkipConfigCheck { + diffs, err := c.checkRenderedConfig(ctx, st.Name, p.Source.Config) + if err != nil { + return err + } + if len(diffs) > 0 { + _, _ = fmt.Fprintf(out, "The config the operator rendered for Milvus %s differs from the running one:\n", st.Name) + for _, d := range diffs { + _, _ = fmt.Fprintf(out, " %s\n", d) + } + return fmt.Errorf("no pod was moved; fix Secret %s or %s and run again, or pass --skip-config-check", configSecretName(st.Name), objectStorageSecretName(st.Name)) + } + c.logf("rendered config matches the running one (storage, etcd root path, WAL, security, ports)") + } + } else if _, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, st.Name, metav1.GetOptions{}); kerr.IsNotFound(err) { + return fmt.Errorf("the Milvus %s is gone while %s; run \"rollback\" to give the cluster back", st.Name, st.Phase) + } else if err != nil { + return err + } + + st.Phase, st.Method = phaseHandingOver, o.Method + if err := c.saveState(ctx, st); err != nil { + return err + } + start := time.Now() + if err := c.handOver(ctx, st, o.Method, o.timing); err != nil { + return fmt.Errorf("%w\nThe handover stopped; run the same command to continue, or \"rollback\" to give the cluster back", err) + } + if err := c.waitMilvusReady(ctx, st.Name, o.ReadyTimeout); err != nil { + return err + } + st.Phase = phaseAdopted + if err := c.saveState(ctx, st); err != nil { + return err + } + c.logf("Milvus %s is Ready; adopted in %s", st.Name, time.Since(start).Round(time.Second)) + _, _ = fmt.Fprintf(out, ` +Next: + - check the data from a client (collection count, known queries) + - the chart's workloads are kept at 0 replicas; do not run "helm uninstall" (it would delete + the client Service KubeDB now uses), and do not "helm upgrade" (it would start them again) + - to give the cluster back: kubectl dba adopt milvus rollback %s -n %s --yes +`, o.Release, c.ns) + return nil +} + +func newCmdMilvusStatus(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "status RELEASE", + Short: i18n.T("Show where each role of an adoption runs"), + Example: templates.Examples(`kubectl dba adopt milvus status my-release -n milvus`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + release, err := releaseArg(args) + if err != nil { + return err + } + ctx := context.Background() + c, err := newCluster(f, cmd.OutOrStdout()) + if err != nil { + return err + } + st, _, err := c.loadState(ctx, release) + if err != nil { + return err + } + if st == nil { + return fmt.Errorf("no adoption of %s in namespace %s", release, c.ns) + } + return c.printStatus(ctx, st) + }, + } + return cmd +} + +func (c *cluster) printStatus(ctx context.Context, st *State) error { + phase := "not found" + milvusExists := false + if db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, st.Name, metav1.GetOptions{}); err == nil { + milvusExists = true + phase, _, _ = unstructured.NestedString(db.Object, "status", "phase") + } else if !kerr.IsNotFound(err) { + return err + } + _, _ = fmt.Fprintf(c.out, "Adoption of %s: %s (method %s). Milvus %s: %s\n\n", st.Release, st.Phase, orNone(st.Method), st.Name, phase) + w := tabwriter.NewWriter(c.out, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "ROLE\tCHART WORKLOAD\tCHART\tKUBEDB\tHANDED OVER") + for _, role := range milvusRoles { + for _, u := range st.unitsOf(role) { + var kd side = petSetSide{c, st.Name, u} + if milvusExists { + kd = kubedbSide{c, st.Name, u} + } + _, _ = fmt.Fprintf(w, "%s\t%s %s\t%s\t%s\t%v\n", u.key(), u.Kind, u.Name, + c.sideSummary(ctx, chartSide{c, u}), c.sideSummary(ctx, kd), st.isDone(u)) + } + } + return w.Flush() +} + +func (c *cluster) sideSummary(ctx context.Context, s side) string { + d, err := s.desired(ctx) + if err != nil { + return "?" + } + r, _, err := c.podCounts(ctx, s.selector()) + if err != nil { + return "?" + } + return fmt.Sprintf("%d/%d Ready", r, d) +} + +type rollbackOptions struct { + sourceOptions + timing + Name string + Yes bool + Cleanup bool +} + +func newCmdMilvusRollback(f cmdutil.Factory) *cobra.Command { + var o rollbackOptions + cmd := &cobra.Command{ + Use: "rollback RELEASE", + Short: i18n.T("Give an adopted (or partly adopted) Milvus back to the chart's workloads"), + Long: templates.LongDesc(` + Deletes the Milvus object with kubedb.com/milvus-orphan-on-delete, so its + PetSets keep serving, then moves each role back to the chart's workloads, + starting the chart's pods before it stops KubeDB's. Once the chart's + proxies are Ready, the client Service gets back its original selector, + labels and ports. The data is never touched. + + With --cleanup it then deletes the PetSets (at 0 replicas), the Services + other than the client Service and the rendered config KubeDB left, and + recreates the chart Services the adoption removed. PVCs are kept. + + For a Milvus adopted by hand (no state ConfigMap), give its --name: the + chart's replicas are then taken from the PetSets, and the client Service + gets the chart proxies' selector.`), + Example: templates.Examples(` + kubectl dba adopt milvus rollback my-release -n milvus --yes --cleanup`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + var err error + if o.Release, err = releaseArg(args); err != nil { + return err + } + c, err := newCluster(f, cmd.OutOrStdout()) + if err != nil { + return err + } + return runRollback(context.Background(), c, o) + }, + } + addSourceFlags(cmd, &o.sourceOptions) + addTimingFlags(cmd, &o.timing) + cmd.Flags().StringVar(&o.Name, "name", "", "Name of the Milvus, for an adoption done by hand (no state ConfigMap)") + cmd.Flags().BoolVar(&o.Yes, "yes", false, "Proceed; without it, rollback only shows what it would do") + cmd.Flags().BoolVar(&o.Cleanup, "cleanup", false, "Delete what KubeDB left behind once the chart serves again (PVCs are kept)") + return cmd +} + +func runRollback(ctx context.Context, c *cluster, o rollbackOptions) error { + st, cm, err := c.loadState(ctx, o.Release) + if err != nil { + return err + } + if st == nil { + if o.Name == "" { + return fmt.Errorf("no adoption of %s in namespace %s; for a Milvus adopted by hand, give its --name", o.Release, c.ns) + } + if st, err = c.stateFromManualAdoption(ctx, o); err != nil { + return err + } + } + out := c.out + _, _ = fmt.Fprintf(out, "Rollback of Milvus %s to the workloads of %s:\n", st.Name, o.Release) + for _, role := range rollbackOrder { + for _, u := range st.unitsOf(role) { + _, _ = fmt.Fprintf(out, " %-24s -> %s %s, %d replicas\n", u.key(), u.Kind, u.Name, u.Replicas) + } + } + if st.Service != nil { + _, _ = fmt.Fprintf(out, " Service %s -> selector %v\n", st.Service.Name, st.Service.Selector) + } + if !o.Yes { + return errors.New("re-run with --yes to proceed") + } + start := time.Now() + if err := c.releaseMilvus(ctx, st.Name, o.Timeout); err != nil { + return err + } + if err := c.rollBack(ctx, st, o.timing); err != nil { + if cm != nil { + _ = c.saveState(ctx, st) + } + return fmt.Errorf("%w\nThe rollback stopped; run the same command to continue", err) + } + if o.Cleanup { + if err := c.cleanupKubeDBObjects(ctx, st); err != nil { + return err + } + } + if cm != nil { + if err := c.kube.CoreV1().ConfigMaps(c.ns).Delete(ctx, cm.Name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + } + c.logf("the chart's workloads serve %s again; rolled back in %s", o.Release, time.Since(start).Round(time.Second)) + if !o.Cleanup { + _, _ = fmt.Fprintf(out, "KubeDB's PetSets (0 replicas), Services and rendered config are left in place; re-run with --cleanup to delete them.\n") + } + _, _ = fmt.Fprintf(out, "The Secrets with label %s=%s (credentials and the carried config) are kept.\n", LabelRelease, o.Release) + return nil +} + +// stateFromManualAdoption rebuilds the state of a Milvus adopted by hand: the +// chart workloads from the release, their replicas from the KubeDB PetSets. +func (c *cluster) stateFromManualAdoption(ctx context.Context, o rollbackOptions) (*State, error) { + src, err := discover(ctx, c.kube, c.ns, o.sourceOptions) + if err != nil { + return nil, err + } + if len(src.Units) == 0 { + return nil, fmt.Errorf("found no workloads of %s", o.Release) + } + st := &State{Release: o.Release, Name: o.Name, Phase: phaseHandingOver, Units: src.Units} + for i := range st.Units { + u := &st.Units[i] + n, err := petSetSide{c, o.Name, *u}.desired(ctx) + if err != nil { + return nil, err + } + if chart, err := (chartSide{c, *u}).desired(ctx); err == nil && chart > n { + n = chart + } + if n == 0 { + return nil, fmt.Errorf("cannot tell how many replicas %s %s should run: it and PetSet %s are both at 0", u.Kind, u.Name, petSetName(o.Name, *u)) + } + u.Replicas = n + } + svc, err := c.kube.CoreV1().Services(c.ns).Get(ctx, o.Name, metav1.GetOptions{}) + if err == nil { + proxy := src.firstUnit(roleProxy) + st.Service = fallbackServiceRecord(*svc, proxy, src.Templates[proxy.key()]) + } else if !kerr.IsNotFound(err) { + return nil, err + } + return st, nil +} + +func orNone(s string) string { + if s == "" { + return "none" + } + return s +} + +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/pkg/adopt/milvus_auth.go b/pkg/adopt/milvus_auth.go new file mode 100644 index 000000000..23f6af7a6 --- /dev/null +++ b/pkg/adopt/milvus_auth.go @@ -0,0 +1,114 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "kmodules.xyz/client-go/tools/portforward" +) + +// Milvus REST error codes for a failed login. +var milvusAuthErrorCodes = map[int]bool{1800: true, 2: true} + +type credentialResult int + +const ( + credentialsAccepted credentialResult = iota + credentialsRejected + credentialsUnverified +) + +// checkCredentials logs in to a running proxy of the release with the given +// root credentials, through a port-forward to its RESTful API. The API server's +// service proxy cannot be used: it does not pass the Authorization header on. +func (c *cluster) checkCredentials(ctx context.Context, proxy Unit, restPort int, user, password string) (credentialResult, string) { + pods, err := c.kube.CoreV1().Pods(c.ns).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(proxy.Selector).String()}) + if err != nil { + return credentialsUnverified, err.Error() + } + pod := "" + for _, p := range pods.Items { + if p.DeletionTimestamp == nil && podReady(p) { + pod = p.Name + break + } + } + if pod == "" { + return credentialsUnverified, "no Ready proxy pod to log in to" + } + tunnel := portforward.NewTunnel(portforward.TunnelOptions{ + Client: c.kube.CoreV1().RESTClient(), + Config: c.cfg, + Resource: "pods", + Name: pod, + Namespace: c.ns, + Remote: restPort, + }) + if err := tunnel.ForwardPort(); err != nil { + return credentialsUnverified, fmt.Sprintf("port-forward to %s: %v", pod, err) + } + defer tunnel.Close() + + url := "http://127.0.0.1:" + strconv.Itoa(tunnel.Local) + "/v2/vectordb/collections/list" + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, url, bytes.NewBufferString("{}")) + if err != nil { + return credentialsUnverified, err.Error() + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+user+":"+password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return credentialsUnverified, fmt.Sprintf("RESTful API of %s on port %d: %v", pod, restPort, err) + } + defer resp.Body.Close() // nolint:errcheck + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + var r struct { + Code int `json:"code"` + Message string `json:"message"` + } + if err := json.Unmarshal(body, &r); err != nil { + return credentialsUnverified, fmt.Sprintf("RESTful API of %s answered HTTP %d: %.200s", pod, resp.StatusCode, body) + } + switch { + case r.Code == 0: + return credentialsAccepted, pod + case milvusAuthErrorCodes[r.Code]: + return credentialsRejected, r.Message + } + return credentialsUnverified, fmt.Sprintf("RESTful API of %s answered code %d: %s", pod, r.Code, r.Message) +} + +// restPort is where the running proxies serve the RESTful API: the gRPC port +// unless proxy.http.port moves it. +func restPort(cfg map[string]any) int { + if p, err := strconv.Atoi(lookupString(cfg, "proxy.http.port")); err == nil && p > 0 { + return p + } + if p, err := strconv.Atoi(lookupString(cfg, "proxy.port")); err == nil && p > 0 { + return p + } + return 19530 +} diff --git a/pkg/adopt/milvus_cluster.go b/pkg/adopt/milvus_cluster.go new file mode 100644 index 000000000..b21810bc5 --- /dev/null +++ b/pkg/adopt/milvus_cluster.go @@ -0,0 +1,362 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "encoding/json" + "fmt" + "io" + "time" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +// cluster bundles the clients and the namespace one command works in. +type cluster struct { + kube kubernetes.Interface + dyn dynamic.Interface + cfg *rest.Config + ns string + out io.Writer + // poll is the interval of every wait. + poll time.Duration +} + +func newCluster(f cmdutil.Factory, out io.Writer) (*cluster, error) { + ns, _, err := f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil, err + } + cfg, err := f.ToRESTConfig() + if err != nil { + return nil, err + } + kc, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + return &cluster{kube: kc, dyn: dyn, cfg: cfg, ns: ns, out: out, poll: 3 * time.Second}, nil +} + +func (c *cluster) logf(format string, a ...any) { + _, _ = fmt.Fprintf(c.out, "%s %s\n", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...)) +} + +// side is one place a Unit's pods run: the chart workload, the KubeDB Milvus +// role, or (during a rollback, after the Milvus is gone) the orphaned PetSet. +type side interface { + String() string + desired(ctx context.Context) (int32, error) + scale(ctx context.Context, n int32) error + selector() map[string]string +} + +// podCounts returns the Ready pods (not terminating) and all pods a selector matches. +func (c *cluster) podCounts(ctx context.Context, sel map[string]string) (ready, total int32, err error) { + pods, err := c.kube.CoreV1().Pods(c.ns).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(sel).String()}) + if err != nil { + return 0, 0, err + } + for _, p := range pods.Items { + total++ + if p.DeletionTimestamp == nil && podReady(p) { + ready++ + } + } + return ready, total, nil +} + +func podReady(p core.Pod) bool { + for _, c := range p.Status.Conditions { + if c.Type == core.PodReady { + return c.Status == core.ConditionTrue + } + } + return false +} + +// waitReady waits until a side has at least n Ready pods. +func (c *cluster) waitReady(ctx context.Context, s side, n int32, timeout time.Duration) error { + return c.waitFor(ctx, timeout, fmt.Sprintf("%s: %d Ready", s, n), func() (bool, string, error) { + r, t, err := c.podCounts(ctx, s.selector()) + return r >= n, fmt.Sprintf("%d/%d Ready", r, t), err + }) +} + +// waitGone waits until a side has at most n pods, terminating ones included. +func (c *cluster) waitGone(ctx context.Context, s side, n int32, timeout time.Duration) error { + return c.waitFor(ctx, timeout, fmt.Sprintf("%s: at most %d pods", s, n), func() (bool, string, error) { + _, t, err := c.podCounts(ctx, s.selector()) + return t <= n, fmt.Sprintf("%d pods", t), err + }) +} + +func (c *cluster) waitFor(ctx context.Context, timeout time.Duration, what string, cond func() (bool, string, error)) error { + deadline := time.Now().Add(timeout) + lastReport := time.Now() + for { + ok, now, err := cond() + if err != nil { + return fmt.Errorf("waiting for %s: %w", what, err) + } + if ok { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for %s (now %s)", timeout, what, now) + } + if time.Since(lastReport) >= 30*time.Second { + c.logf(" waiting for %s (now %s)", what, now) + lastReport = time.Now() + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.poll): + } + } +} + +// chartSide is the workload the chart created. +type chartSide struct { + c *cluster + u Unit +} + +func (s chartSide) String() string { return fmt.Sprintf("%s %s", s.u.Kind, s.u.Name) } + +func (s chartSide) selector() map[string]string { return s.u.Selector } + +func (s chartSide) desired(ctx context.Context) (int32, error) { + switch s.u.Kind { + case "Deployment": + d, err := s.c.kube.AppsV1().Deployments(s.c.ns).Get(ctx, s.u.Name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + if d.Spec.Replicas == nil { + return 1, nil + } + return *d.Spec.Replicas, nil + case "StatefulSet": + st, err := s.c.kube.AppsV1().StatefulSets(s.c.ns).Get(ctx, s.u.Name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + if st.Spec.Replicas == nil { + return 1, nil + } + return *st.Spec.Replicas, nil + } + return 0, fmt.Errorf("unsupported workload kind %s", s.u.Kind) +} + +func (s chartSide) scale(ctx context.Context, n int32) error { + patch := []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, n)) + var err error + switch s.u.Kind { + case "Deployment": + _, err = s.c.kube.AppsV1().Deployments(s.c.ns).Patch(ctx, s.u.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + case "StatefulSet": + _, err = s.c.kube.AppsV1().StatefulSets(s.c.ns).Patch(ctx, s.u.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + default: + err = fmt.Errorf("unsupported workload kind %s", s.u.Kind) + } + if err != nil { + return fmt.Errorf("failed to scale %s to %d: %w", s, n, err) + } + return nil +} + +// kubedbSide is the role (or node group) of the KubeDB Milvus. +type kubedbSide struct { + c *cluster + name string + u Unit +} + +func (s kubedbSide) String() string { + if s.u.Group != "" { + return fmt.Sprintf("Milvus %s %s group %s", s.name, s.u.Role, s.u.Group) + } + return fmt.Sprintf("Milvus %s %s", s.name, s.u.Role) +} + +func (s kubedbSide) selector() map[string]string { return kubedbSelector(s.name, s.u) } + +// replicasPath returns the field path of the Unit's replicas in the Milvus. +func (s kubedbSide) replicasPath(db *unstructured.Unstructured) ([]string, int, error) { + base := []string{"spec", "topology", "distributed", s.u.Role} + if s.u.Group == "" { + return append(base, "replicas"), -1, nil + } + groups, _, _ := unstructured.NestedSlice(db.Object, append(base, "groups")...) + for i, g := range groups { + if m, ok := g.(map[string]any); ok && m["name"] == s.u.Group { + return nil, i, nil + } + } + return nil, -1, fmt.Errorf("no %s group %q in Milvus %s", s.u.Role, s.u.Group, s.name) +} + +func (s kubedbSide) desired(ctx context.Context) (int32, error) { + db, err := s.c.dyn.Resource(MilvusGVR).Namespace(s.c.ns).Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + path, idx, err := s.replicasPath(db) + if err != nil { + return 0, err + } + if idx < 0 { + n, _, _ := unstructured.NestedInt64(db.Object, path...) + return int32(n), nil + } + groups, _, _ := unstructured.NestedSlice(db.Object, "spec", "topology", "distributed", s.u.Role, "groups") + n, _, _ := unstructured.NestedInt64(groups[idx].(map[string]any), "replicas") + return int32(n), nil +} + +func (s kubedbSide) scale(ctx context.Context, n int32) error { + db, err := s.c.dyn.Resource(MilvusGVR).Namespace(s.c.ns).Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + return err + } + _, idx, err := s.replicasPath(db) + if err != nil { + return err + } + var patch []byte + pt := types.MergePatchType + if idx < 0 { + patch = []byte(fmt.Sprintf(`{"spec":{"topology":{"distributed":{%q:{"replicas":%d}}}}}`, s.u.Role, n)) + } else { + pt = types.JSONPatchType + patch = []byte(fmt.Sprintf(`[{"op":"add","path":"/spec/topology/distributed/%s/groups/%d/replicas","value":%d}]`, s.u.Role, idx, n)) + } + if _, err := s.c.dyn.Resource(MilvusGVR).Namespace(s.c.ns).Patch(ctx, s.name, pt, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to set %s to %d replicas: %w", s, n, err) + } + return nil +} + +// petSetSide is a KubeDB PetSet after its Milvus was deleted with the +// orphan-on-delete annotation: the pods keep running under the PetSet controller. +type petSetSide struct { + c *cluster + name string + u Unit +} + +func (s petSetSide) String() string { return "PetSet " + petSetName(s.name, s.u) } + +func (s petSetSide) selector() map[string]string { return kubedbSelector(s.name, s.u) } + +func (s petSetSide) desired(ctx context.Context) (int32, error) { + ps, err := s.c.dyn.Resource(PetSetGVR).Namespace(s.c.ns).Get(ctx, petSetName(s.name, s.u), metav1.GetOptions{}) + if kerr.IsNotFound(err) { + return 0, nil + } + if err != nil { + return 0, err + } + n, found, _ := unstructured.NestedInt64(ps.Object, "spec", "replicas") + if !found { + return 1, nil + } + return int32(n), nil +} + +func (s petSetSide) scale(ctx context.Context, n int32) error { + patch := []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, n)) + _, err := s.c.dyn.Resource(PetSetGVR).Namespace(s.c.ns).Patch(ctx, petSetName(s.name, s.u), types.MergePatchType, patch, metav1.PatchOptions{}) + if kerr.IsNotFound(err) && n == 0 { + return nil + } + if err != nil { + return fmt.Errorf("failed to scale %s to %d: %w", s, n, err) + } + return nil +} + +// loadState finds the adoption of a release in the namespace. +func (c *cluster) loadState(ctx context.Context, release string) (*State, *core.ConfigMap, error) { + cms, err := c.kube.CoreV1().ConfigMaps(c.ns).List(ctx, metav1.ListOptions{LabelSelector: LabelRelease + "=" + release}) + if err != nil { + return nil, nil, err + } + for i := range cms.Items { + cm := &cms.Items[i] + raw, ok := cm.Data[stateKey] + if !ok { + continue + } + st := &State{} + if err := json.Unmarshal([]byte(raw), st); err != nil { + return nil, nil, fmt.Errorf("ConfigMap %s holds an unreadable adoption state: %w", cm.Name, err) + } + return st, cm, nil + } + return nil, nil, nil +} + +func (c *cluster) saveState(ctx context.Context, st *State) error { + raw, err := json.Marshal(st) + if err != nil { + return err + } + cm, err := c.kube.CoreV1().ConfigMaps(c.ns).Get(ctx, stateConfigMapName(st.Name), metav1.GetOptions{}) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[stateKey] = string(raw) + _, err = c.kube.CoreV1().ConfigMaps(c.ns).Update(ctx, cm, metav1.UpdateOptions{}) + return err +} + +// renderedConfig returns the milvus.yaml the operator rendered for a Milvus: +// the Secret it owns that carries that key. +func (c *cluster) renderedConfig(ctx context.Context, db *unstructured.Unstructured) (string, error) { + secs, err := c.kube.CoreV1().Secrets(c.ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return "", err + } + for _, s := range secs.Items { + for _, o := range s.OwnerReferences { + if o.UID == db.GetUID() { + if v, ok := s.Data["milvus.yaml"]; ok { + return string(v), nil + } + } + } + } + return "", nil +} diff --git a/pkg/adopt/milvus_config.go b/pkg/adopt/milvus_config.go new file mode 100644 index 000000000..5e7d41d79 --- /dev/null +++ b/pkg/adopt/milvus_config.go @@ -0,0 +1,264 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "fmt" + "sort" + "strings" + + "sigs.k8s.io/yaml" +) + +// Milvus reads its configuration files from /milvus/configs in this order, each +// overriding the one before; a chart mounts some of them from a ConfigMap. +var milvusConfigFiles = []string{"milvus.yaml", "default.yaml", "user.yaml"} + +const defaultEtcdRootPath = "by-dev" + +// The minio.* keys the KubeDB operator reads from spec.objectStorage.configSecret +// and writes back into the rendered milvus.yaml. +var objectStorageKeys = []string{ + "address", "port", "useSSL", "bucketName", "rootPath", "useIAM", "cloudProvider", + "region", "iamEndpoint", "useVirtualHost", "accessKeyID", "secretAccessKey", +} + +// mergeConfigFiles merges the files of one Milvus in load order. Unknown file +// names are returned so the caller can report them. +func mergeConfigFiles(files map[string]string) (map[string]any, []string, error) { + out := map[string]any{} + for _, name := range milvusConfigFiles { + raw, ok := files[name] + if !ok { + continue + } + var doc map[string]any + if err := yaml.Unmarshal([]byte(raw), &doc); err != nil { + return nil, nil, fmt.Errorf("failed to parse %s: %w", name, err) + } + out = deepMerge(out, doc) + } + var unknown []string + for name := range files { + if !contains(milvusConfigFiles, name) { + unknown = append(unknown, name) + } + } + sort.Strings(unknown) + return out, unknown, nil +} + +func deepMerge(dst, src map[string]any) map[string]any { + for k, v := range src { + if sm, ok := v.(map[string]any); ok { + if dm, ok := dst[k].(map[string]any); ok { + dst[k] = deepMerge(dm, sm) + continue + } + } + dst[k] = v + } + return dst +} + +// lookup returns the value at a dotted path, e.g. "minio.bucketName". +func lookup(cfg map[string]any, path string) (any, bool) { + var cur any = cfg + for _, p := range strings.Split(path, ".") { + m, ok := cur.(map[string]any) + if !ok { + return nil, false + } + if cur, ok = m[p]; !ok { + return nil, false + } + } + return cur, true +} + +func lookupString(cfg map[string]any, path string) string { + v, ok := lookup(cfg, path) + if !ok || v == nil { + return "" + } + return scalarString(v) +} + +func lookupBool(cfg map[string]any, path string) bool { + return strings.EqualFold(lookupString(cfg, path), "true") +} + +// scalarString renders a YAML scalar the way it would be written back: numbers +// without exponent, booleans as true/false. +func scalarString(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + case float64: + if t == float64(int64(t)) { + return fmt.Sprintf("%d", int64(t)) + } + return fmt.Sprintf("%v", t) + default: + return fmt.Sprintf("%v", t) + } +} + +// etcdSettings is what the running Milvus uses to reach its metadata. +type etcdSettings struct { + Endpoints []string + RootPath string + AuthEnabled bool + TLSEnabled bool +} + +func etcdFromConfig(cfg map[string]any) etcdSettings { + s := etcdSettings{RootPath: lookupString(cfg, "etcd.rootPath")} + if s.RootPath == "" { + s.RootPath = defaultEtcdRootPath + } + s.AuthEnabled = lookupBool(cfg, "etcd.auth.enabled") + s.TLSEnabled = lookupBool(cfg, "etcd.ssl.enabled") + scheme := "http://" + if s.TLSEnabled { + scheme = "https://" + } + var eps []string + switch v, _ := lookup(cfg, "etcd.endpoints"); t := v.(type) { + case []any: + for _, e := range t { + eps = append(eps, scalarString(e)) + } + case string: + eps = strings.Split(t, ",") + } + for _, e := range eps { + if e = strings.TrimSpace(e); e == "" { + continue + } + if !strings.Contains(e, "://") { + e = scheme + e + } + s.Endpoints = append(s.Endpoints, e) + } + return s +} + +// walType returns the message queue the config selects ("" when it does not say). +func walType(cfg map[string]any) string { + if t := lookupString(cfg, "mq.type"); t != "" && t != "default" { + return t + } + return lookupString(cfg, "messageQueue") +} + +// objectStorageData returns the Secret data for spec.objectStorage.configSecret: +// every minio.* key the running config sets. +func objectStorageData(cfg map[string]any) map[string]string { + out := map[string]string{} + for _, k := range objectStorageKeys { + if v, ok := lookup(cfg, "minio."+k); ok && v != nil && scalarString(v) != "" { + out[k] = scalarString(v) + } + } + return out +} + +// carriedConfig is the milvus.yaml handed to KubeDB through +// spec.configuration.secretName. The etcd section is dropped (the Milvus object +// sets the endpoints and root path); everything else is kept as it runs today. +// Milvus serves its RESTful API on the gRPC port unless proxy.http.port is set, +// while KubeDB defaults it to 8080, so an unset port is pinned to "". +func carriedConfig(cfg map[string]any) (string, error) { + out := deepCopy(cfg) + delete(out, "etcd") + if v, ok := lookup(out, "proxy.http.port"); !ok || v == nil { + proxy, _ := out["proxy"].(map[string]any) + if proxy == nil { + proxy = map[string]any{} + out["proxy"] = proxy + } + http, _ := proxy["http"].(map[string]any) + if http == nil { + http = map[string]any{} + proxy["http"] = http + } + http["port"] = "" + } + b, err := yaml.Marshal(out) + return string(b), err +} + +func deepCopy(in map[string]any) map[string]any { + out := make(map[string]any, len(in)) + for k, v := range in { + if m, ok := v.(map[string]any); ok { + out[k] = deepCopy(m) + continue + } + out[k] = v + } + return out +} + +// criticalConfigPaths are the settings that decide where the data is read from +// and how clients log in. After the Milvus is created, the config the operator +// renders must agree with the running one on each of them. +var criticalConfigPaths = []string{ + "minio.address", "minio.port", "minio.bucketName", "minio.rootPath", "minio.useSSL", + "minio.useIAM", "minio.cloudProvider", "minio.region", "minio.accessKeyID", "minio.secretAccessKey", + "mq.type", "woodpecker.storage.type", "woodpecker.storage.rootPath", + "common.security.authorizationEnabled", "proxy.port", "proxy.http.port", +} + +// configMismatches compares the running config with the one the operator +// rendered. A path the running config does not set is compared only for the +// etcd root path, whose Milvus and KubeDB defaults differ. +func configMismatches(running, rendered map[string]any) []string { + var out []string + want := etcdFromConfig(running).RootPath + if got := lookupString(rendered, "etcd.rootPath"); got != want { + out = append(out, fmt.Sprintf("etcd.rootPath: running %q, rendered %q", want, got)) + } + for _, p := range criticalConfigPaths { + rv, ok := lookup(running, p) + if !ok || rv == nil { + if p == "proxy.http.port" { + rv = "" // carriedConfig pins it + } else { + continue + } + } + a, b := scalarString(rv), lookupString(rendered, p) + if a != b { + if strings.Contains(strings.ToLower(p), "secret") || strings.HasSuffix(p, "accessKeyID") { + out = append(out, p+": differs") + continue + } + out = append(out, fmt.Sprintf("%s: running %q, rendered %q", p, a, b)) + } + } + return out +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} diff --git a/pkg/adopt/milvus_handover.go b/pkg/adopt/milvus_handover.go new file mode 100644 index 000000000..41573c39f --- /dev/null +++ b/pkg/adopt/milvus_handover.go @@ -0,0 +1,506 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "fmt" + "time" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" +) + +// Handover methods. +const ( + // methodOverlap starts the KubeDB pods of a role before it stops the chart's, + // so the role never stops serving. It needs room for one extra set of pods + // (one extra QueryNode at a time). + methodOverlap = "overlap" + // methodStopStart stops the chart's pods of a role before it starts KubeDB's. + // Each role is briefly unavailable; no extra capacity is needed. + methodStopStart = "stop-start" +) + +// handoverOrder is the order the roles change hands in. With the overlap +// method the proxies go first, so clients reach the KubeDB pods early; a +// stop-start handover moves the coordinators first and the proxies last. +func handoverOrder(method string) []string { + if method == methodStopStart { + return []string{roleMixCoord, roleDataNode, roleQueryNode, roleStreamingNode, roleProxy} + } + return []string{roleProxy, roleMixCoord, roleDataNode, roleStreamingNode, roleQueryNode} +} + +// rollbackOrder returns the roles to the chart, the proxies last so the +// client Service moves once every other role serves from the chart again. +var rollbackOrder = []string{roleQueryNode, roleStreamingNode, roleDataNode, roleMixCoord, roleProxy} + +type timing struct { + Timeout time.Duration // per wait + Settle time.Duration // after each role + QueryNodeDelay time.Duration // after each new QueryNode, before an old one stops +} + +// step is one replica change on the way from one side to the other: grow +// `to` to toN, then shrink `from` to fromN. +type step struct { + toN, fromN int32 +} + +// swapSteps returns the steps that move n replicas from one side to the +// other. QueryNodes go one at a time, so the cluster never holds more than one +// extra QueryNode and every old one stops only after a new one serves. +func swapSteps(n int32, oneByOne bool) []step { + if !oneByOne || n <= 1 { + return []step{{toN: n, fromN: 0}} + } + var out []step + for i := int32(1); i <= n; i++ { + out = append(out, step{toN: i, fromN: n - i}) + } + return out +} + +// overlapSwap moves a Unit from one side to the other without a gap: each +// step grows the target first, waits for it to be Ready, then shrinks the +// source. It is safe to repeat: the target never shrinks and the source never +// grows. +func (c *cluster) overlapSwap(ctx context.Context, from, to side, n int32, oneByOne bool, t timing, beforeShrink func() error) error { + steps := swapSteps(n, oneByOne) + for i, s := range steps { + cur, err := to.desired(ctx) + if err != nil { + return err + } + if cur < s.toN { + c.logf("%s -> %d replicas", to, s.toN) + if err := to.scale(ctx, s.toN); err != nil { + return err + } + } + if err := c.waitReady(ctx, to, s.toN, t.Timeout); err != nil { + return err + } + if oneByOne && t.QueryNodeDelay > 0 && cur < s.toN { + c.logf(" %s serves %d; waiting %s before stopping one of %s", to, s.toN, t.QueryNodeDelay, from) + sleep(ctx, t.QueryNodeDelay) + } + if i == len(steps)-1 && beforeShrink != nil { + if err := beforeShrink(); err != nil { + return err + } + } + cur, err = from.desired(ctx) + if err != nil { + return err + } + if cur > s.fromN { + c.logf("%s -> %d replicas", from, s.fromN) + if err := from.scale(ctx, s.fromN); err != nil { + return err + } + } + if err := c.waitGone(ctx, from, s.fromN, t.Timeout); err != nil { + return err + } + } + return nil +} + +// stopStartSwap stops the source, waits until its pods are gone, then starts +// the target. +func (c *cluster) stopStartSwap(ctx context.Context, from, to side, n int32, t timing) error { + if cur, err := from.desired(ctx); err != nil { + return err + } else if cur > 0 { + c.logf("%s -> 0 replicas", from) + if err := from.scale(ctx, 0); err != nil { + return err + } + } + if err := c.waitGone(ctx, from, 0, t.Timeout); err != nil { + return err + } + if cur, err := to.desired(ctx); err != nil { + return err + } else if cur < n { + c.logf("%s -> %d replicas", to, n) + if err := to.scale(ctx, n); err != nil { + return err + } + } + return c.waitReady(ctx, to, n, t.Timeout) +} + +// handOver moves every Unit of the adoption to KubeDB, recording each one. +func (c *cluster) handOver(ctx context.Context, st *State, method string, t timing) error { + for _, role := range handoverOrder(method) { + for _, u := range st.unitsOf(role) { + if st.isDone(u) { + continue + } + start := time.Now() + from, to := chartSide{c, u}, kubedbSide{c, st.Name, u} + c.logf("== %s (%s): %d replicas, %s", u.key(), u.Name, u.Replicas, method) + var err error + if method == methodStopStart { + err = c.stopStartSwap(ctx, from, to, u.Replicas, t) + } else { + err = c.overlapSwap(ctx, from, to, u.Replicas, u.Role == roleQueryNode, t, nil) + } + if err != nil { + return fmt.Errorf("%s: %w", u.key(), err) + } + st.markDone(u) + if err := c.saveState(ctx, st); err != nil { + return err + } + c.logf("== %s handed over in %s", u.key(), time.Since(start).Round(time.Second)) + sleep(ctx, t.Settle) + } + } + return nil +} + +// rollBack returns every Unit to the chart workloads. The Milvus must be gone +// (deleted with the orphan-on-delete annotation), so its PetSets are scaled +// directly and the operator no longer moves the client Service. +func (c *cluster) rollBack(ctx context.Context, st *State, t timing) error { + for _, role := range rollbackOrder { + for _, u := range st.unitsOf(role) { + from, to := petSetSide{c, st.Name, u}, chartSide{c, u} + cur, err := from.desired(ctx) + if err != nil { + return err + } + _, pods, err := c.podCounts(ctx, from.selector()) + if err != nil { + return err + } + var restore func() error + if u.Role == roleProxy && st.Service != nil { + restore = func() error { return c.restoreClientService(ctx, st) } + } + if cur == 0 && pods == 0 { + // Nothing of this Unit runs on KubeDB (it was never handed over, or a + // stop-start handover stopped between the two sides). + if n, err := to.desired(ctx); err != nil { + return err + } else if n < u.Replicas { + c.logf("%s -> %d replicas", to, u.Replicas) + if err := to.scale(ctx, u.Replicas); err != nil { + return err + } + } + if err := c.waitReady(ctx, to, u.Replicas, t.Timeout); err != nil { + return err + } + if restore != nil { + if err := restore(); err != nil { + return err + } + } + st.unmarkDone(u) + continue + } + start := time.Now() + c.logf("== %s (%s): back to the chart, %d replicas", u.key(), u.Name, u.Replicas) + if err := c.overlapSwap(ctx, from, to, u.Replicas, u.Role == roleQueryNode, t, restore); err != nil { + return fmt.Errorf("%s: %w", u.key(), err) + } + st.unmarkDone(u) + c.logf("== %s back on the chart in %s", u.key(), time.Since(start).Round(time.Second)) + sleep(ctx, t.Settle) + } + } + return nil +} + +// restoreClientService points the client Service back at the chart's proxies, +// with the labels and ports it had before the adoption. +func (c *cluster) restoreClientService(ctx context.Context, st *State) error { + rec := st.Service + svc, err := c.kube.CoreV1().Services(c.ns).Get(ctx, rec.Name, metav1.GetOptions{}) + if err != nil { + return err + } + svc.Spec.Selector = rec.Selector + if rec.Labels != nil { + svc.Labels = rec.Labels + } + if len(rec.Ports) > 0 { + svc.Spec.Ports = rec.Ports + } + if _, err := c.kube.CoreV1().Services(c.ns).Update(ctx, svc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to point Service %s back at the chart proxies: %w", rec.Name, err) + } + c.logf("Service %s points at the chart proxies again", rec.Name) + return nil +} + +// releaseMilvus deletes the Milvus with the orphan-on-delete annotation, so +// its PetSets, Services and pods stay as they are. +func (c *cluster) releaseMilvus(ctx context.Context, name string, timeout time.Duration) error { + ri := c.dyn.Resource(MilvusGVR).Namespace(c.ns) + if _, err := ri.Get(ctx, name, metav1.GetOptions{}); kerr.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:"true"}},"spec":{"deletionPolicy":"Halt"}}`, annOrphanOnDelete) + if _, err := ri.Patch(ctx, name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to prepare Milvus %s for release: %w", name, err) + } + c.logf("deleting Milvus %s with %s; its pods keep running", name, annOrphanOnDelete) + if err := ri.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + return c.waitFor(ctx, timeout, "Milvus "+name+" to be released", func() (bool, string, error) { + _, err := ri.Get(ctx, name, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + return true, "", nil + } + return false, "still present", err + }) +} + +// createObjects creates what a plan holds, keeping any object that already +// exists (for example one edited from "plan --output-dir"). +func (c *cluster) createObjects(ctx context.Context, p *adoptPlan) error { + for _, s := range p.Objects.Secrets { + _, err := c.kube.CoreV1().Secrets(c.ns).Create(ctx, s, metav1.CreateOptions{}) + switch { + case kerr.IsAlreadyExists(err): + c.logf("Secret %s exists; using it", s.Name) + case err != nil: + return fmt.Errorf("failed to create Secret %s: %w", s.Name, err) + default: + c.logf("created Secret %s", s.Name) + } + } + _, err := c.kube.CoreV1().ConfigMaps(c.ns).Create(ctx, p.Objects.State, metav1.CreateOptions{}) + switch { + case kerr.IsAlreadyExists(err): + case err != nil: + return fmt.Errorf("failed to create ConfigMap %s: %w", p.Objects.State.Name, err) + default: + c.logf("created ConfigMap %s (adoption state)", p.Objects.State.Name) + } + return nil +} + +// removeGoverningServices deletes the chart's non-headless Services on names +// KubeDB needs headless, recording them so a rollback can recreate them. +func (c *cluster) removeGoverningServices(ctx context.Context, st *State, svcs []core.Service) error { + for _, svc := range svcs { + st.RemovedServices = append(st.RemovedServices, recordService(svc)) + } + if len(svcs) > 0 { + if err := c.saveState(ctx, st); err != nil { + return err + } + } + for _, svc := range svcs { + err := c.kube.CoreV1().Services(c.ns).Delete(ctx, svc.Name, metav1.DeleteOptions{}) + if err != nil && !kerr.IsNotFound(err) { + return fmt.Errorf("failed to delete Service %s: %w", svc.Name, err) + } + c.logf("deleted the chart's Service %s (KubeDB recreates it headless)", svc.Name) + } + return nil +} + +func recordService(svc core.Service) ServiceRecord { + return ServiceRecord{ + Name: svc.Name, Selector: svc.Spec.Selector, Labels: svc.Labels, + Annotations: svc.Annotations, Ports: svc.Spec.Ports, Type: svc.Spec.Type, + } +} + +// createMilvus creates the staged Milvus and waits until the operator has +// rendered its config and created a PetSet for every Unit. +func (c *cluster) createMilvus(ctx context.Context, p *adoptPlan, timeout time.Duration) (*unstructured.Unstructured, error) { + ri := c.dyn.Resource(MilvusGVR).Namespace(c.ns) + db, err := ri.Create(ctx, p.Objects.Milvus, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + db, err = ri.Get(ctx, p.State.Name, metav1.GetOptions{}) + } else if err == nil { + c.logf("created Milvus %s, every role at 0 replicas", p.State.Name) + } + if err != nil { + return nil, fmt.Errorf("failed to create Milvus %s: %w", p.State.Name, err) + } + return db, c.waitOperator(ctx, p.State, timeout) +} + +func (c *cluster) waitOperator(ctx context.Context, st *State, timeout time.Duration) error { + return c.waitFor(ctx, timeout, "the operator to create the PetSets and render the config of Milvus "+st.Name, func() (bool, string, error) { + db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, st.Name, metav1.GetOptions{}) + if err != nil { + return false, "", err + } + cfg, err := c.renderedConfig(ctx, db) + if err != nil { + return false, "", err + } + if cfg == "" { + return false, "no rendered config yet; see kubectl describe milvus " + st.Name, nil + } + for _, u := range st.Units { + if _, err := c.dyn.Resource(PetSetGVR).Namespace(c.ns).Get(ctx, petSetName(st.Name, u), metav1.GetOptions{}); kerr.IsNotFound(err) { + return false, "PetSet " + petSetName(st.Name, u) + " not created yet", nil + } else if err != nil { + return false, "", err + } + } + return true, "", nil + }) +} + +// checkRenderedConfig compares the config the operator rendered with the one +// the cluster runs, on every setting that decides where the data is read from. +func (c *cluster) checkRenderedConfig(ctx context.Context, name string, running map[string]any) ([]string, error) { + db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + raw, err := c.renderedConfig(ctx, db) + if err != nil { + return nil, err + } + var rendered map[string]any + if err := yaml.Unmarshal([]byte(raw), &rendered); err != nil { + return nil, fmt.Errorf("failed to parse the rendered config: %w", err) + } + return configMismatches(running, rendered), nil +} + +// waitMilvusReady waits for the Milvus to report Ready. +func (c *cluster) waitMilvusReady(ctx context.Context, name string, timeout time.Duration) error { + return c.waitFor(ctx, timeout, "Milvus "+name+" to be Ready", func() (bool, string, error) { + db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, "", err + } + phase, _, _ := unstructured.NestedString(db.Object, "status", "phase") + return phase == "Ready", "phase " + phase, nil + }) +} + +// cleanupKubeDBObjects deletes what the released Milvus left behind (its +// PetSets, Services other than the client Service, rendered config and RBAC), +// then recreates the chart Services the adoption removed. PVCs are kept. +func (c *cluster) cleanupKubeDBObjects(ctx context.Context, st *State) error { + sel := metav1.ListOptions{LabelSelector: labelInstance + "=" + st.Name + "," + labelManagedBy + "=kubedb.com"} + pss, err := c.dyn.Resource(PetSetGVR).Namespace(c.ns).List(ctx, sel) + if err != nil { + return err + } + for _, ps := range pss.Items { + if n, _, _ := unstructured.NestedInt64(ps.Object, "spec", "replicas"); n > 0 { + return fmt.Errorf("PetSet %s still runs %d replicas; not deleting", ps.GetName(), n) + } + if err := c.dyn.Resource(PetSetGVR).Namespace(c.ns).Delete(ctx, ps.GetName(), metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted PetSet %s", ps.GetName()) + } + svcs, err := c.kube.CoreV1().Services(c.ns).List(ctx, sel) + if err != nil { + return err + } + for _, svc := range svcs.Items { + if st.Service != nil && svc.Name == st.Service.Name { + continue + } + if err := c.kube.CoreV1().Services(c.ns).Delete(ctx, svc.Name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted Service %s", svc.Name) + } + secs, err := c.kube.CoreV1().Secrets(c.ns).List(ctx, sel) + if err != nil { + return err + } + for _, s := range secs.Items { + if _, ok := s.Data["milvus.yaml"]; !ok { + continue // credentials stay + } + if err := c.kube.CoreV1().Secrets(c.ns).Delete(ctx, s.Name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted Secret %s (rendered config)", s.Name) + } + for _, rec := range st.RemovedServices { + svc := &core.Service{ + ObjectMeta: metav1.ObjectMeta{Name: rec.Name, Namespace: c.ns, Labels: rec.Labels, Annotations: rec.Annotations}, + Spec: core.ServiceSpec{Type: rec.Type, Selector: rec.Selector, Ports: withoutNodePorts(rec.Ports)}, + } + if _, err := c.kube.CoreV1().Services(c.ns).Create(ctx, svc, metav1.CreateOptions{}); err != nil && !kerr.IsAlreadyExists(err) { + return fmt.Errorf("failed to recreate the chart's Service %s: %w", rec.Name, err) + } + c.logf("recreated the chart's Service %s", rec.Name) + } + return nil +} + +func withoutNodePorts(in []core.ServicePort) []core.ServicePort { + out := make([]core.ServicePort, len(in)) + for i, p := range in { + p.NodePort = 0 + out[i] = p + } + return out +} + +// fallbackServiceRecord rebuilds the client Service record for a Milvus that +// was adopted by hand: the chart proxies' selector, and the Service's current +// ports with the port names only KubeDB pods define replaced by numbers. +func fallbackServiceRecord(svc core.Service, proxy Unit, t core.PodTemplateSpec) *ServiceRecord { + rec := &ServiceRecord{Name: svc.Name, Selector: proxy.Selector} + named := map[string]bool{} + numbers := map[int32]bool{} + if c := milvusContainer(t); c != nil { + for _, p := range c.Ports { + named[p.Name] = true + numbers[p.ContainerPort] = true + } + } + for _, p := range svc.Spec.Ports { + if p.TargetPort.Type == intstr.String && !named[p.TargetPort.StrVal] { + if !numbers[p.Port] { + continue // a port only the KubeDB pods serve + } + p.TargetPort = intstr.FromInt32(p.Port) + } + rec.Ports = append(rec.Ports, p) + } + return rec +} + +func sleep(ctx context.Context, d time.Duration) { + if d <= 0 { + return + } + select { + case <-ctx.Done(): + case <-time.After(d): + } +} diff --git a/pkg/adopt/milvus_plan.go b/pkg/adopt/milvus_plan.go new file mode 100644 index 000000000..b88febab0 --- /dev/null +++ b/pkg/adopt/milvus_plan.go @@ -0,0 +1,350 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "fmt" + "os" + "strings" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/yaml" +) + +// planOptions are the inputs of an adoption that discovery cannot find. +type planOptions struct { + sourceOptions + Name string + Version string + AuthSecret string + RootPasswordFile string + EtcdAuthSecret string + ConfigFile string + RestrictedSCC bool + SkipCredentialCheck bool + // allowExisting accepts a Milvus of that name: run resumes an adoption + // whose Milvus it already created. + allowExisting bool +} + +// adoptPlan is a checked adoption: the findings and, when nothing blocks it, +// the objects to create. +type adoptPlan struct { + Findings findings + Source *Source + State *State + Objects *adoptObjects + // GoverningServices are the chart's Services on names KubeDB needs headless; + // run deletes them before it creates the Milvus. + GoverningServices []core.Service +} + +// governingRoles have a headless governing Service named -. +var governingRoles = []string{roleMixCoord, roleDataNode, roleQueryNode, roleStreamingNode} + +func buildPlan(ctx context.Context, c *cluster, o planOptions) (*adoptPlan, error) { + src, err := discover(ctx, c.kube, c.ns, o.sourceOptions) + if err != nil { + return nil, err + } + p := &adoptPlan{Source: src, Findings: src.Findings} + fs := &p.Findings + if o.ConfigFile != "" { + raw, err := os.ReadFile(o.ConfigFile) + if err != nil { + return nil, err + } + cfg, _, err := mergeConfigFiles(map[string]string{"user.yaml": string(raw)}) + if err != nil { + return nil, err + } + src.Config = cfg + src.ConfigFrom = []string{o.ConfigFile} + p.Findings = withoutConfigError(p.Findings) + } + if fs.hasErrors() { + return p, nil + } + for _, u := range src.Units { + fs.infof("%-13s %s %s, %d replicas%s", u.Role, u.Kind, u.Name, u.Replicas, groupNote(u)) + } + fs.infof("config read from %s", strings.Join(src.ConfigFrom, "; ")) + cfg := src.Config + + // The Milvus is named after the client Service, so KubeDB takes it over. + name := o.Name + switch { + case name == "" && src.Service != nil: + name = src.Service.Name + case name == "": + fs.errorf("no Service selects the proxy pods; name the Milvus with --name") + return p, nil + case src.Service != nil && name != src.Service.Name: + fs.warnf("clients use Service %s; the Milvus %s gets a new Service, and clients must move to it", src.Service.Name, name) + } + if src.Service != nil && src.Service.Name == name { + fs.infof("client Service %s (%s) is taken over: same name, IP and ports", name, src.Service.Spec.ClusterIP) + } + if _, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}); err == nil { + if !o.allowExisting { + fs.errorf("a Milvus %s already exists", name) + } + } else if !kerr.IsNotFound(err) { + return nil, err + } + + // Write-ahead log: KubeDB runs Woodpecker on object storage only. + switch wt := walType(cfg); wt { + case "woodpecker": + fs.infof("write-ahead log: woodpecker") + case "": + if c.streamingNodeLogsMention(ctx, src, "woodpecker") { + fs.infof("write-ahead log: woodpecker (from the StreamingNode logs; the config does not set mq.type)") + } else { + fs.errorf("cannot confirm the write-ahead log is Woodpecker: the config sets no mq.type and the StreamingNode logs do not mention it") + } + default: + fs.errorf("write-ahead log is %s; KubeDB runs Woodpecker only, and changing the WAL of a running cluster is a migration", wt) + } + if t := lookupString(cfg, "woodpecker.storage.type"); t != "" && t != "minio" { + fs.errorf("woodpecker.storage.type is %s; KubeDB keeps the write-ahead log on the object storage (minio)", t) + } + + // Metadata. + etcd := etcdFromConfig(cfg) + if len(etcd.Endpoints) == 0 { + fs.errorf("the config sets no etcd.endpoints") + } else { + fs.infof("etcd %s, root path %q (kept)", strings.Join(etcd.Endpoints, ","), etcd.RootPath) + } + if etcd.TLSEnabled { + fs.errorf("the etcd uses TLS; this command does not carry etcd client certificates yet, see the adoption guide") + } + if etcd.AuthEnabled { + if o.EtcdAuthSecret == "" { + fs.errorf("the etcd uses authentication; give its root user in a basic-auth Secret with --etcd-auth-secret") + } else if err := c.basicAuthSecretExists(ctx, o.EtcdAuthSecret); err != nil { + fs.errorf("--etcd-auth-secret: %v", err) + } + } + + // Object storage. + osData := objectStorageData(cfg) + if osData["bucketName"] == "" || osData["address"] == "" { + fs.errorf("the config does not set minio.address and minio.bucketName") + } else { + fs.infof("object storage %s, bucket %s, root path %q (kept)", osData["address"], osData["bucketName"], osData["rootPath"]) + } + if osData["accessKeyID"] == "" && !strings.EqualFold(osData["useIAM"], "true") { + fs.warnf("the config has no minio.accessKeyID and minio.useIAM is not true; if the credentials come from environment variables, add them to Secret %s", objectStorageSecretName(name)) + } + + // Version. + version := o.Version + if version == "" { + list, err := c.dyn.Resource(MilvusVersionGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list MilvusVersions: %w", err) + } + if version, err = pickVersion(src.Image, list.Items); err != nil { + fs.errorf("%v", err) + } + } + if version != "" { + fs.infof("image %s -> MilvusVersion %s", src.Image, version) + } + + // Credentials. + in := renderInput{ + Namespace: c.ns, Name: name, Version: version, Release: o.Release, Source: src, + EtcdAuthSecret: o.EtcdAuthSecret, RestrictedSCC: o.RestrictedSCC, + } + if !lookupBool(cfg, "common.security.authorizationEnabled") { + in.DisableAuth = true + fs.infof("authentication is off; the Milvus gets spec.disableSecurity: true") + } else if err := c.resolveRootCredentials(ctx, o, src, &in, fs); err != nil { + return nil, err + } + + // Services KubeDB needs headless. + for _, role := range governingRoles { + svc, err := c.kube.CoreV1().Services(c.ns).Get(ctx, name+"-"+role, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + continue + } + if err != nil { + return nil, err + } + if svc.Spec.ClusterIP == core.ClusterIPNone { + continue + } + if belongsToRelease(*svc, src) { + p.GoverningServices = append(p.GoverningServices, *svc) + fs.infof("Service %s is not headless; run deletes it (KubeDB recreates it headless; Milvus does not use it)", svc.Name) + } else { + fs.errorf("Service %s exists, is not headless and does not belong to the release; KubeDB needs that name for a headless Service", svc.Name) + } + } + + if fs.hasErrors() { + return p, nil + } + p.State = newState(in) + if p.Objects, err = render(in, p.State); err != nil { + return nil, err + } + return p, nil +} + +func groupNote(u Unit) string { + if u.Group == "" { + return "" + } + return ", node group " + u.Group +} + +func withoutConfigError(fs findings) findings { + var out findings + for _, f := range fs { + if f.Level == levelError && strings.Contains(f.Msg, "--config-file") { + continue + } + out = append(out, f) + } + return out +} + +// belongsToRelease reports whether a Service selects pods of the release, or +// carries the release's instance label. +func belongsToRelease(svc core.Service, src *Source) bool { + for _, u := range src.Units { + if len(svc.Spec.Selector) > 0 && labels.SelectorFromSet(svc.Spec.Selector).Matches(labels.Set(src.Templates[u.key()].Labels)) { + return true + } + } + for _, u := range src.Units { + if v := u.Selector[labelInstance]; v != "" && svc.Labels[labelInstance] == v { + return true + } + } + return false +} + +func (c *cluster) basicAuthSecretExists(ctx context.Context, name string) error { + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + if len(s.Data[core.BasicAuthUsernameKey]) == 0 || len(s.Data[core.BasicAuthPasswordKey]) == 0 { + return fmt.Errorf("the Secret %s needs the keys %s and %s", name, core.BasicAuthUsernameKey, core.BasicAuthPasswordKey) + } + return nil +} + +// resolveRootCredentials picks the root credentials (an existing Secret, a +// password file, or the chart's defaultRootPassword) and logs in with them. +// Milvus applies a root password only on its first start, so a password that +// does not log in today must not be handed to KubeDB. +func (c *cluster) resolveRootCredentials(ctx context.Context, o planOptions, src *Source, in *renderInput, fs *findings) error { + user, password, from := "root", "", "" + switch { + case o.AuthSecret != "": + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, o.AuthSecret, metav1.GetOptions{}) + if err != nil { + fs.errorf("--auth-secret: %v", err) + return nil + } + user, password = string(s.Data[core.BasicAuthUsernameKey]), string(s.Data[core.BasicAuthPasswordKey]) + if user == "" || password == "" { + fs.errorf("Secret %s needs the keys %s and %s", o.AuthSecret, core.BasicAuthUsernameKey, core.BasicAuthPasswordKey) + return nil + } + in.AuthSecret, from = o.AuthSecret, "Secret "+o.AuthSecret + case o.RootPasswordFile != "": + raw, err := os.ReadFile(o.RootPasswordFile) + if err != nil { + return err + } + password, from = strings.TrimRight(string(raw), "\r\n"), o.RootPasswordFile + default: + password = lookupString(src.Config, "common.security.defaultRootPassword") + if password == "" { + fs.errorf("authentication is on and the config has no defaultRootPassword; give the current root password with --auth-secret or --root-password-file") + return nil + } + from = "common.security.defaultRootPassword" + } + in.RootUser, in.RootPassword = user, password + if o.SkipCredentialCheck { + fs.warnf("root credentials from %s are not checked (--skip-credential-check)", from) + return nil + } + res, detail := c.checkCredentials(ctx, src.firstUnit(roleProxy), restPort(src.Config), user, password) + switch res { + case credentialsAccepted: + fs.infof("root credentials from %s log in to %s", from, detail) + case credentialsRejected: + fs.errorf("root credentials from %s are rejected by the running cluster (%s); Milvus applies a root password only on its first start, so give the current one with --auth-secret or --root-password-file", from, detail) + default: + fs.warnf("root credentials from %s could not be checked: %s", from, detail) + } + return nil +} + +// streamingNodeLogsMention reports whether a StreamingNode's recent log mentions s. +func (c *cluster) streamingNodeLogsMention(ctx context.Context, src *Source, s string) bool { + u := src.firstUnit(roleStreamingNode) + pods, err := c.kube.CoreV1().Pods(c.ns).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(u.Selector).String()}) + if err != nil || len(pods.Items) == 0 { + return false + } + limit := int64(4 << 20) + raw, err := c.kube.CoreV1().Pods(c.ns).GetLogs(pods.Items[0].Name, &core.PodLogOptions{LimitBytes: &limit}).DoRaw(ctx) + return err == nil && strings.Contains(strings.ToLower(string(raw)), s) +} + +// yamlDocs renders the plan's objects as YAML, one document set per file. +func (p *adoptPlan) yamlDocs(redact bool) (map[string]string, error) { + out := map[string]string{} + var secrets []string + for _, s := range p.Objects.Secrets { + cp := s.DeepCopy() + if redact { + for k := range cp.StringData { + cp.StringData[k] = "" + } + } + b, err := yaml.Marshal(cp) + if err != nil { + return nil, err + } + secrets = append(secrets, string(b)) + } + out["00-secrets.yaml"] = strings.Join(secrets, "---\n") + b, err := yaml.Marshal(p.Objects.State) + if err != nil { + return nil, err + } + out["01-adopt-state.yaml"] = string(b) + b, err = yaml.Marshal(p.Objects.Milvus.Object) + if err != nil { + return nil, err + } + out["02-milvus.yaml"] = string(b) + return out, nil +} diff --git a/pkg/adopt/milvus_render.go b/pkg/adopt/milvus_render.go new file mode 100644 index 000000000..b005a2653 --- /dev/null +++ b/pkg/adopt/milvus_render.go @@ -0,0 +1,356 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + MilvusGVR = schema.GroupVersionResource{Group: "kubedb.com", Version: "v1alpha2", Resource: "milvuses"} + MilvusVersionGVR = schema.GroupVersionResource{Group: "catalog.kubedb.com", Version: "v1alpha1", Resource: "milvusversions"} + PetSetGVR = schema.GroupVersionResource{Group: "apps.k8s.appscode.com", Version: "v1", Resource: "petsets"} +) + +const ( + // LabelRelease marks the objects this command creates for one adoption. + LabelRelease = "adopt.kubedb.com/release" + stateKey = "state.json" + + annOrphanOnDelete = "kubedb.com/milvus-orphan-on-delete" + labelRole = "kubedb.com/role" + labelNodeGroup = "kubedb.com/node-group" + labelInstance = "app.kubernetes.io/instance" + labelManagedBy = "app.kubernetes.io/managed-by" + + milvusContainerName = "milvus" +) + +func stateConfigMapName(name string) string { return name + "-adopt" } +func authSecretName(name string) string { return name + "-auth" } +func objectStorageSecretName(name string) string { return name + "-object-storage" } +func configSecretName(name string) string { return name + "-adopt-config" } + +// petSetName is the KubeDB PetSet that replaces a Unit. +func petSetName(name string, u Unit) string { + n := name + "-" + u.Role + if u.Group != "" { + n += "-" + u.Group + } + return n +} + +// kubedbSelector selects the KubeDB pods of a Unit. +func kubedbSelector(name string, u Unit) map[string]string { + sel := map[string]string{labelInstance: name, labelRole: u.Role} + if u.Group != "" { + sel[labelNodeGroup] = u.Group + } + return sel +} + +// adoptObjects are the objects an adoption creates, in creation order. +type adoptObjects struct { + Secrets []*core.Secret + State *core.ConfigMap + Milvus *unstructured.Unstructured +} + +// renderInput is everything the objects are built from. +type renderInput struct { + Namespace string + Name string + Version string + Release string + Source *Source + AuthSecret string // existing Secret to use; "" with RootUser set creates one + RootUser string + RootPassword string + DisableAuth bool + EtcdAuthSecret string + RestrictedSCC bool +} + +func newState(in renderInput) *State { + st := &State{Release: in.Release, Name: in.Name, Phase: phasePlanned, Units: in.Source.Units} + if svc := in.Source.Service; svc != nil && svc.Name == in.Name { + rec := recordService(*svc) + st.Service = &rec + } + return st +} + +func render(in renderInput, st *State) (*adoptObjects, error) { + cfg := in.Source.Config + objs := &adoptObjects{} + lbl := map[string]string{LabelRelease: in.Release} + + authName := in.AuthSecret + if !in.DisableAuth && authName == "" { + authName = authSecretName(in.Name) + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: authName, Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeBasicAuth, + StringData: map[string]string{core.BasicAuthUsernameKey: in.RootUser, core.BasicAuthPasswordKey: in.RootPassword}, + }) + } + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: objectStorageSecretName(in.Name), Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeOpaque, + StringData: objectStorageData(cfg), + }) + carried, err := carriedConfig(cfg) + if err != nil { + return nil, err + } + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: configSecretName(in.Name), Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeOpaque, + StringData: map[string]string{"milvus.yaml": carried}, + }) + + raw, err := json.Marshal(st) + if err != nil { + return nil, err + } + objs.State = &core.ConfigMap{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"}, + ObjectMeta: metav1.ObjectMeta{Name: stateConfigMapName(in.Name), Namespace: in.Namespace, Labels: lbl}, + Data: map[string]string{stateKey: string(raw)}, + } + + etcd := etcdFromConfig(cfg) + meta := map[string]any{ + "externallyManaged": true, + "endpoints": toAnySlice(etcd.Endpoints), + "rootPath": etcd.RootPath, + } + if in.EtcdAuthSecret != "" { + meta["authSecret"] = map[string]any{"name": in.EtcdAuthSecret} + } + spec := map[string]any{ + "version": in.Version, + "deletionPolicy": "DoNotTerminate", + "autoOps": map[string]any{"disabled": true}, + "healthChecker": map[string]any{"disableWriteCheck": true}, + "metaStorage": meta, + "objectStorage": map[string]any{"configSecret": map[string]any{"name": objectStorageSecretName(in.Name)}}, + "configuration": map[string]any{"secretName": configSecretName(in.Name)}, + "topology": map[string]any{ + "mode": "Distributed", + "distributed": distributedSpec(in.Source), + }, + } + if in.DisableAuth { + spec["disableSecurity"] = true + } else { + spec["authSecret"] = map[string]any{"name": authName} + } + if in.RestrictedSCC { + spec["restrictedSCC"] = true + } + objs.Milvus = &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kubedb.com/v1alpha2", + "kind": "Milvus", + "metadata": map[string]any{ + "name": in.Name, + "namespace": in.Namespace, + "labels": map[string]any{LabelRelease: in.Release}, + "annotations": map[string]any{annOrphanOnDelete: "true"}, + }, + "spec": spec, + }} + return objs, nil +} + +// distributedSpec stages every role at 0 replicas, with the chart's resources, +// node selector, tolerations and GPU count. A role with several workloads +// becomes node groups. +func distributedSpec(src *Source) map[string]any { + out := map[string]any{} + for _, role := range milvusRoles { + var units []Unit + for _, u := range src.Units { + if u.Role == role { + units = append(units, u) + } + } + if len(units) == 0 { + continue + } + if len(units) == 1 && units[0].Group == "" { + node := nodeSpec(src.Templates[units[0].key()]) + node["replicas"] = int64(0) + out[role] = node + continue + } + groups := make([]any, 0, len(units)) + for _, u := range units { + g := nodeSpec(src.Templates[u.key()]) + g["name"] = u.Group + g["replicas"] = int64(0) + groups = append(groups, g) + } + out[role] = map[string]any{"groups": groups} + } + return out +} + +// Resources a KubeDB Milvus container takes from the chart as they are. GPUs +// go to the gpu field; SR-IOV and RDMA devices come from the network field. +func carriedResource(name core.ResourceName) bool { + n := string(name) + return name == core.ResourceCPU || name == core.ResourceMemory || name == core.ResourceEphemeralStorage || + strings.HasPrefix(n, core.ResourceHugePagesPrefix) +} + +func gpuResource(name core.ResourceName) bool { + return strings.HasSuffix(string(name), "/gpu") +} + +func nodeSpec(t core.PodTemplateSpec) map[string]any { + node := map[string]any{} + podSpec := map[string]any{} + c := milvusContainer(t) + if c != nil { + res := map[string]any{} + var gpuName core.ResourceName + var gpuCount int64 + for _, part := range []struct { + key string + list core.ResourceList + }{{"requests", c.Resources.Requests}, {"limits", c.Resources.Limits}} { + m := map[string]any{} + for _, n := range sortedResourceNames(part.list) { + q := part.list[n] + switch { + case carriedResource(n): + m[string(n)] = q.String() + case gpuResource(n): + gpuName, gpuCount = n, q.Value() + } + } + if len(m) > 0 { + res[part.key] = m + } + } + if len(res) > 0 { + podSpec["containers"] = []any{map[string]any{"name": milvusContainerName, "resources": res}} + } + if gpuCount > 0 { + gpu := map[string]any{"count": gpuCount} + if gpuName != "nvidia.com/gpu" { + gpu["resourceName"] = string(gpuName) + } + node["gpu"] = gpu + } + } + if len(t.Spec.NodeSelector) > 0 { + ns := map[string]any{} + for k, v := range t.Spec.NodeSelector { + ns[k] = v + } + podSpec["nodeSelector"] = ns + } + if len(t.Spec.Tolerations) > 0 { + tols := make([]any, 0, len(t.Spec.Tolerations)) + for _, tol := range t.Spec.Tolerations { + b, _ := json.Marshal(tol) + var m map[string]any + _ = json.Unmarshal(b, &m) + tols = append(tols, m) + } + podSpec["tolerations"] = tols + } + if len(podSpec) > 0 { + node["podTemplate"] = map[string]any{"spec": podSpec} + } + return node +} + +func sortedResourceNames(l core.ResourceList) []core.ResourceName { + out := make([]core.ResourceName, 0, len(l)) + for n := range l { + out = append(out, n) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +func toAnySlice(in []string) []any { + out := make([]any, 0, len(in)) + for _, s := range in { + out = append(out, s) + } + return out +} + +// imageVersion splits a Milvus image into its version and whether it is a GPU +// build: "milvusdb/milvus:v2.6.15-gpu" -> ("2.6.15", true). +func imageVersion(image string) (string, bool, error) { + if strings.Contains(image, "@") { + return "", false, fmt.Errorf("image %s is pinned by digest; pass the version with --version", image) + } + i := strings.LastIndex(image, ":") + if i < 0 || strings.Contains(image[i:], "/") { + return "", false, fmt.Errorf("image %s has no tag; pass the version with --version", image) + } + tag := strings.TrimPrefix(image[i+1:], "v") + gpu := strings.Contains(tag, "-gpu") + ver, _, _ := strings.Cut(tag, "-") + return ver, gpu, nil +} + +// pickVersion returns the MilvusVersion for an image: same version, same GPU +// support, not deprecated. +func pickVersion(image string, versions []unstructured.Unstructured) (string, error) { + ver, gpu, err := imageVersion(image) + if err != nil { + return "", err + } + var names []string + for _, v := range versions { + sv, _, _ := unstructured.NestedString(v.Object, "spec", "version") + g, _, _ := unstructured.NestedBool(v.Object, "spec", "db", "gpu", "supported") + dep, _, _ := unstructured.NestedBool(v.Object, "spec", "deprecated") + if sv == ver && g == gpu && !dep { + names = append(names, v.GetName()) + } + } + if len(names) == 0 { + build := "CPU" + if gpu { + build = "GPU" + } + return "", fmt.Errorf("no MilvusVersion for %s (%s build of %s); install the KubeDB catalog that has it, or pass --version", image, build, ver) + } + sort.Strings(names) + for _, n := range names { // prefer the canonical name + if n == ver || n == ver+"-gpu" { + return n, nil + } + } + return names[0], nil +} diff --git a/pkg/adopt/milvus_source.go b/pkg/adopt/milvus_source.go new file mode 100644 index 000000000..f38c86ca9 --- /dev/null +++ b/pkg/adopt/milvus_source.go @@ -0,0 +1,543 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "fmt" + "path" + "regexp" + "sort" + "strings" + + apps "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" +) + +// Distributed Milvus 2.6 roles, in the order the CR lists them. +const ( + roleProxy = "proxy" + roleMixCoord = "mixcoord" + roleDataNode = "datanode" + roleQueryNode = "querynode" + roleStreamingNode = "streamingnode" +) + +var milvusRoles = []string{roleProxy, roleMixCoord, roleDataNode, roleQueryNode, roleStreamingNode} + +// Components of Milvus before 2.6 (separate coordinators, IndexNode) and the +// Standalone mode. A cluster with any of them is not a 2.6 Distributed cluster. +var unsupportedComponents = []string{"rootcoord", "querycoord", "datacoord", "indexcoord", "indexnode", "standalone"} + +// Unit is one workload of the running cluster and the KubeDB PetSet that +// replaces it: a role, or one node group of a role when the role runs as +// several workloads (for example QueryNodes on two GPU pools). +type Unit struct { + Role string `json:"role"` + Group string `json:"group,omitempty"` + Kind string `json:"kind"` + Name string `json:"name"` + Replicas int32 `json:"replicas"` + Selector map[string]string `json:"selector"` +} + +func (u Unit) key() string { + if u.Group == "" { + return u.Role + } + return u.Role + "/" + u.Group +} + +// ServiceRecord is the client Service as the chart left it, so a rollback can +// give it back unchanged. +type ServiceRecord struct { + Name string `json:"name"` + Type core.ServiceType `json:"type,omitempty"` + Selector map[string]string `json:"selector"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Ports []core.ServicePort `json:"ports"` +} + +// Phases of an adoption, recorded in the state ConfigMap. +const ( + phasePlanned = "Planned" + phaseHandingOver = "HandingOver" + phaseAdopted = "Adopted" + phaseRolledBack = "RolledBack" +) + +// State is the record of one adoption, kept in the ConfigMap -adopt so +// that run can resume and rollback knows what to restore. +type State struct { + Release string `json:"release"` + Name string `json:"name"` + Method string `json:"method,omitempty"` + Phase string `json:"phase"` + Units []Unit `json:"units"` + Service *ServiceRecord `json:"service,omitempty"` + // RemovedServices are chart Services deleted because KubeDB needs their + // names for headless Services; a rollback recreates them. + RemovedServices []ServiceRecord `json:"removedServices,omitempty"` + Done []string `json:"done,omitempty"` +} + +func (s *State) unitsOf(role string) []Unit { + var out []Unit + for _, u := range s.Units { + if u.Role == role { + out = append(out, u) + } + } + return out +} + +func (s *State) isDone(u Unit) bool { return contains(s.Done, u.key()) } + +func (s *State) markDone(u Unit) { + if !s.isDone(u) { + s.Done = append(s.Done, u.key()) + } +} + +func (s *State) unmarkDone(u Unit) { + out := s.Done[:0] + for _, k := range s.Done { + if k != u.key() { + out = append(out, k) + } + } + s.Done = out +} + +// Source is what discovery found about the running cluster. +type Source struct { + Units []Unit + Templates map[string]core.PodTemplateSpec // by Unit.key() + Config map[string]any + ConfigFrom []string + Image string + Service *core.Service + Findings findings +} + +type workloadRef struct { + Kind string + Name string + Labels map[string]string + Replicas int32 + Selector map[string]string + Template core.PodTemplateSpec +} + +func refFromDeployment(d apps.Deployment) workloadRef { + r := workloadRef{Kind: "Deployment", Name: d.Name, Labels: d.Labels, Replicas: 1, Template: d.Spec.Template} + if d.Spec.Replicas != nil { + r.Replicas = *d.Spec.Replicas + } + if d.Spec.Selector != nil { + r.Selector = d.Spec.Selector.MatchLabels + } + return r +} + +func refFromStatefulSet(s apps.StatefulSet) workloadRef { + r := workloadRef{Kind: "StatefulSet", Name: s.Name, Labels: s.Labels, Replicas: 1, Template: s.Spec.Template} + if s.Spec.Replicas != nil { + r.Replicas = *s.Spec.Replicas + } + if s.Spec.Selector != nil { + r.Selector = s.Spec.Selector.MatchLabels + } + return r +} + +// sourceOptions select the running cluster. +type sourceOptions struct { + Release string + Selector string // label selector of the release's workloads + RoleLabel string // label that names a workload's role + Workloads []string // explicit role[/group]=Kind/name, instead of the selector +} + +func (o sourceOptions) selector() string { + if o.Selector != "" { + return o.Selector + } + return "app.kubernetes.io/instance=" + o.Release +} + +// assignUnits turns the release's workloads into Units. Workloads of one role +// become node groups named after the workload. +func assignUnits(refs []workloadRef, o sourceOptions) ([]Unit, map[string]core.PodTemplateSpec, findings, error) { + var fs findings + explicit := map[string]string{} // Kind/name -> role[/group] + for _, w := range o.Workloads { + k, v, ok := strings.Cut(w, "=") + if !ok { + return nil, nil, nil, fmt.Errorf("--workload %q: want role[/group]=Kind/name", w) + } + explicit[strings.ToLower(v)] = k + } + byRole := map[string][]workloadRef{} + groupOf := map[string]string{} + for _, r := range refs { + var role, group string + if len(explicit) > 0 { + rg, ok := explicit[strings.ToLower(r.Kind+"/"+r.Name)] + if !ok { + continue + } + role, group, _ = strings.Cut(rg, "/") + } else { + role = r.Template.Labels[o.RoleLabel] + if role == "" { + role = r.Labels[o.RoleLabel] + } + } + switch { + case role == "": + continue // etcd, MinIO and other workloads of the release + case contains(unsupportedComponents, role): + fs.errorf("%s %s is a %q component: only a Distributed Milvus 2.6 (MixCoord, Proxy, DataNode, QueryNode, StreamingNode) can be adopted", r.Kind, r.Name, role) + continue + case !contains(milvusRoles, role): + fs.infof("%s %s has %s=%s, not a Milvus role; ignored", r.Kind, r.Name, o.RoleLabel, role) + continue + } + byRole[role] = append(byRole[role], r) + if group != "" { + groupOf[r.Kind+"/"+r.Name] = group + } + } + if len(explicit) > 0 { + found := 0 + for _, rs := range byRole { + found += len(rs) + } + if found != len(explicit) { + return nil, nil, nil, fmt.Errorf("found %d of the %d workloads given with --workload", found, len(explicit)) + } + } + + var units []Unit + templates := map[string]core.PodTemplateSpec{} + for _, role := range milvusRoles { + rs := byRole[role] + if len(rs) == 0 { + fs.errorf("no workload found for the %s role (selector %q, role label %q)", role, o.selector(), o.RoleLabel) + continue + } + sort.Slice(rs, func(i, j int) bool { return rs[i].Name < rs[j].Name }) + used := map[string]bool{} + for _, r := range rs { + u := Unit{Role: role, Kind: r.Kind, Name: r.Name, Replicas: r.Replicas, Selector: r.Selector} + if len(rs) > 1 { + g := groupOf[r.Kind+"/"+r.Name] + if g == "" { + g = groupName(r.Name, role) + } + if used[g] { + return nil, nil, nil, fmt.Errorf("workloads of role %s map to the same group %q; name them with --workload", role, g) + } + used[g] = true + u.Group = g + } + if len(u.Selector) == 0 { + return nil, nil, nil, fmt.Errorf("%s %s has no matchLabels selector", r.Kind, r.Name) + } + if u.Replicas == 0 { + fs.errorf("%s %s runs 0 replicas: nothing to hand over. If an adoption already moved it, run \"status\"", r.Kind, r.Name) + } + units = append(units, u) + templates[u.key()] = r.Template + } + } + return units, templates, fs, nil +} + +var dnsUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// groupName derives a node group name from a workload name, e.g. +// "milvus-querynode-h200" -> "h200". +func groupName(workload, role string) string { + n := strings.ToLower(workload) + if i := strings.LastIndex(n, role+"-"); i >= 0 && i+len(role)+1 < len(n) { + n = n[i+len(role)+1:] + } + n = strings.Trim(dnsUnsafe.ReplaceAllString(n, "-"), "-") + if len(n) > 20 { + n = strings.Trim(n[len(n)-20:], "-") + } + return n +} + +// milvusContainer returns the Milvus container of a pod template. +func milvusContainer(t core.PodTemplateSpec) *core.Container { + for i := range t.Spec.Containers { + c := &t.Spec.Containers[i] + if strings.Contains(c.Image, "milvus") { + return c + } + } + if len(t.Spec.Containers) > 0 { + return &t.Spec.Containers[0] + } + return nil +} + +// configMountDir is where Milvus reads its configuration files. +const configMountDir = "/milvus/configs" + +// configSource is one configuration file mounted into the Milvus container. +type configSource struct { + File string // milvus.yaml, default.yaml, user.yaml + Kind string // ConfigMap or Secret + Name string + Key string // "" for every key of a directory mount + IsDir bool + Volume string +} + +// configSources lists the configuration files a pod template mounts. +func configSources(t core.PodTemplateSpec) []configSource { + c := milvusContainer(t) + if c == nil { + return nil + } + vols := map[string]core.Volume{} + for _, v := range t.Spec.Volumes { + vols[v.Name] = v + } + var out []configSource + for _, m := range c.VolumeMounts { + v, ok := vols[m.Name] + if !ok { + continue + } + src := configSource{Volume: m.Name} + switch { + case v.ConfigMap != nil: + src.Kind, src.Name = "ConfigMap", v.ConfigMap.Name + case v.Secret != nil: + src.Kind, src.Name = "Secret", v.Secret.SecretName + default: + continue + } + mp := strings.TrimSuffix(m.MountPath, "/") + switch { + case mp == configMountDir: + src.IsDir = true + case path.Dir(mp) == configMountDir: + src.File = path.Base(mp) + src.Key = m.SubPath + if src.Key == "" { + continue // a directory mounted over a file name; not a config file + } + default: + continue + } + out = append(out, src) + } + return out +} + +// discover reads the release's workloads, configuration, image and client +// Service. Problems that block an adoption are returned as error findings. +func discover(ctx context.Context, kc kubernetes.Interface, ns string, o sourceOptions) (*Source, error) { + sel := o.selector() + if _, err := labels.Parse(sel); err != nil { + return nil, fmt.Errorf("invalid selector %q: %w", sel, err) + } + lo := metav1.ListOptions{LabelSelector: sel} + if len(o.Workloads) > 0 { + lo = metav1.ListOptions{} + } + var refs []workloadRef + deps, err := kc.AppsV1().Deployments(ns).List(ctx, lo) + if err != nil { + return nil, fmt.Errorf("failed to list Deployments: %w", err) + } + for _, d := range deps.Items { + refs = append(refs, refFromDeployment(d)) + } + sts, err := kc.AppsV1().StatefulSets(ns).List(ctx, lo) + if err != nil { + return nil, fmt.Errorf("failed to list StatefulSets: %w", err) + } + for _, s := range sts.Items { + refs = append(refs, refFromStatefulSet(s)) + } + units, templates, fs, err := assignUnits(refs, o) + if err != nil { + return nil, err + } + src := &Source{Units: units, Templates: templates, Findings: fs} + if src.Findings.hasErrors() { + return src, nil + } + + proxy := templates[src.firstUnit(roleProxy).key()] + if c := milvusContainer(proxy); c != nil { + src.Image = c.Image + } + if err := src.readConfig(ctx, kc, ns, proxy); err != nil { + return nil, err + } + if err := src.findClientService(ctx, kc, ns, proxy); err != nil { + return nil, err + } + src.checkTemplates() + return src, nil +} + +func (s *Source) firstUnit(role string) Unit { + for _, u := range s.Units { + if u.Role == role { + return u + } + } + return Unit{} +} + +func (s *Source) readConfig(ctx context.Context, kc kubernetes.Interface, ns string, t core.PodTemplateSpec) error { + files := map[string]string{} + for _, src := range configSources(t) { + var data map[string]string + switch src.Kind { + case "ConfigMap": + cm, err := kc.CoreV1().ConfigMaps(ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read the Milvus config ConfigMap %s: %w", src.Name, err) + } + data = cm.Data + case "Secret": + sec, err := kc.CoreV1().Secrets(ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read the Milvus config Secret %s: %w", src.Name, err) + } + data = map[string]string{} + for k, v := range sec.Data { + data[k] = string(v) + } + } + if src.IsDir { + for k, v := range data { + if strings.HasSuffix(k, ".yaml") { + files[k] = v + s.ConfigFrom = append(s.ConfigFrom, fmt.Sprintf("%s %s key %s", src.Kind, src.Name, k)) + } + } + continue + } + v, ok := data[src.Key] + if !ok { + return fmt.Errorf("%s %s has no key %q mounted as %s", src.Kind, src.Name, src.Key, src.File) + } + files[src.File] = v + s.ConfigFrom = append(s.ConfigFrom, fmt.Sprintf("%s %s key %s as %s", src.Kind, src.Name, src.Key, src.File)) + } + if len(files) == 0 { + s.Findings.errorf("found no configuration file mounted in %s; pass the effective milvus.yaml with --config-file", configMountDir) + s.Config = map[string]any{} + return nil + } + cfg, unknown, err := mergeConfigFiles(files) + if err != nil { + return err + } + for _, f := range unknown { + s.Findings.warnf("%s/%s is mounted but Milvus does not read it; not carried over", configMountDir, f) + } + s.Config = cfg + return nil +} + +// findClientService finds the Service whose selector picks the proxy pods. +func (s *Source) findClientService(ctx context.Context, kc kubernetes.Interface, ns string, proxy core.PodTemplateSpec) error { + svcs, err := kc.CoreV1().Services(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list Services: %w", err) + } + var matches []core.Service + for _, svc := range svcs.Items { + if len(svc.Spec.Selector) == 0 || svc.Spec.ClusterIP == core.ClusterIPNone { + continue + } + if labels.SelectorFromSet(svc.Spec.Selector).Matches(labels.Set(proxy.Labels)) { + matches = append(matches, svc) + } + } + sort.Slice(matches, func(i, j int) bool { + pi, pj := hasPort(matches[i], 19530), hasPort(matches[j], 19530) + if pi != pj { + return pi + } + return matches[i].Name < matches[j].Name + }) + switch len(matches) { + case 0: + s.Findings.warnf("no Service selects the proxy pods; clients will need the new Service ") + case 1: + default: + var names []string + for _, m := range matches[1:] { + names = append(names, m.Name) + } + s.Findings.warnf("Services %s also select the proxy pods; they keep pointing at the chart proxies and stop working once those are gone", + strings.Join(names, ", ")) + } + if len(matches) > 0 { + s.Service = &matches[0] + } + return nil +} + +func hasPort(svc core.Service, port int32) bool { + for _, p := range svc.Spec.Ports { + if p.Port == port { + return true + } + } + return false +} + +// checkTemplates reports pod settings that the Milvus object does not carry over +// by itself. +func (s *Source) checkTemplates() { + for _, u := range s.Units { + t := s.Templates[u.key()] + id := u.Kind + " " + u.Name + if t.Spec.Affinity != nil { + s.Findings.warnf("%s sets affinity, which is not carried over; use a PlacementPolicy (spec.topology.distributed.%s.podTemplate.spec.podPlacementPolicy)", id, u.Role) + } + if n := t.Annotations["k8s.v1.cni.cncf.io/networks"]; n != "" { + s.Findings.warnf("%s attaches secondary networks (%s); set spec.topology.distributed.%s.network before handing it over (see the GPU and SR-IOV guide)", id, n, u.Role) + } + for _, ic := range t.Spec.InitContainers { + s.Findings.warnf("%s has init container %q, which is not carried over", id, ic.Name) + } + if c := milvusContainer(t); c != nil { + for _, e := range c.Env { + s.Findings.warnf("%s sets environment variable %s; Milvus may read it as configuration, and it is not carried over", id, e.Name) + } + if c.Image != s.Image { + s.Findings.errorf("%s runs image %s, the proxy runs %s; adopt a cluster that runs one version", id, c.Image, s.Image) + } + } + } +} diff --git a/pkg/adopt/milvus_test.go b/pkg/adopt/milvus_test.go new file mode 100644 index 000000000..5393f3a65 --- /dev/null +++ b/pkg/adopt/milvus_test.go @@ -0,0 +1,507 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "reflect" + "strings" + "testing" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" +) + +// The ConfigMap the upstream Milvus Helm chart (5.0.x) renders, trimmed. +const chartDefaultYAML = ` +etcd: + endpoints: + - mv-etcd-0.mv-etcd-headless.ns.svc.cluster.local:2379 + - mv-etcd-1.mv-etcd-headless.ns.svc.cluster.local:2379 +minio: + address: minio.storage.svc + port: 9000 + accessKeyID: ak + secretAccessKey: sk + useSSL: false + bucketName: milvus-data + rootPath: prod + useIAM: false + cloudProvider: aws + iamEndpoint: + region: +mq: + type: woodpecker +messageQueue: woodpecker +proxy: + port: 19530 +woodpecker: + storage: + type: minio + rootPath: /woodpecker/data +` + +const chartUserYAML = ` +common: + security: + authorizationEnabled: true + defaultRootPassword: S3cret +minio: + rootPath: prod-override +` + +func chartConfig(t *testing.T) map[string]any { + t.Helper() + cfg, unknown, err := mergeConfigFiles(map[string]string{"default.yaml": chartDefaultYAML, "user.yaml": chartUserYAML}) + if err != nil { + t.Fatal(err) + } + if len(unknown) != 0 { + t.Fatalf("unexpected unknown files %v", unknown) + } + return cfg +} + +func TestMergeConfigFilesUserOverridesDefault(t *testing.T) { + cfg := chartConfig(t) + if got := lookupString(cfg, "minio.rootPath"); got != "prod-override" { + t.Fatalf("user.yaml must override default.yaml: rootPath = %q", got) + } + if got := lookupString(cfg, "minio.bucketName"); got != "milvus-data" { + t.Fatalf("keys only in default.yaml are kept: bucketName = %q", got) + } + _, unknown, err := mergeConfigFiles(map[string]string{"user.yaml": "a: 1", "extra.yaml": "b: 2"}) + if err != nil || !reflect.DeepEqual(unknown, []string{"extra.yaml"}) { + t.Fatalf("unknown files = %v, %v", unknown, err) + } +} + +func TestEtcdFromConfig(t *testing.T) { + s := etcdFromConfig(chartConfig(t)) + want := []string{ + "http://mv-etcd-0.mv-etcd-headless.ns.svc.cluster.local:2379", + "http://mv-etcd-1.mv-etcd-headless.ns.svc.cluster.local:2379", + } + if !reflect.DeepEqual(s.Endpoints, want) { + t.Fatalf("endpoints = %v", s.Endpoints) + } + if s.RootPath != "by-dev" { + t.Fatalf("an unset etcd.rootPath is the Milvus default by-dev, got %q", s.RootPath) + } + s = etcdFromConfig(map[string]any{"etcd": map[string]any{ + "endpoints": "a:2379, b:2379", "rootPath": "custom", + "ssl": map[string]any{"enabled": true}, "auth": map[string]any{"enabled": "true"}, + }}) + if !reflect.DeepEqual(s.Endpoints, []string{"https://a:2379", "https://b:2379"}) || s.RootPath != "custom" || !s.AuthEnabled || !s.TLSEnabled { + t.Fatalf("got %+v", s) + } +} + +func TestObjectStorageData(t *testing.T) { + got := objectStorageData(chartConfig(t)) + want := map[string]string{ + "address": "minio.storage.svc", "port": "9000", "accessKeyID": "ak", "secretAccessKey": "sk", + "useSSL": "false", "bucketName": "milvus-data", "rootPath": "prod-override", "useIAM": "false", "cloudProvider": "aws", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v", got) + } +} + +func TestCarriedConfig(t *testing.T) { + raw, err := carriedConfig(chartConfig(t)) + if err != nil { + t.Fatal(err) + } + var cfg map[string]any + if err := yaml.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatal(err) + } + if _, ok := cfg["etcd"]; ok { + t.Fatal("the etcd section must be dropped: the Milvus object sets it") + } + if v, ok := lookup(cfg, "proxy.http.port"); !ok || v != "" { + t.Fatalf("an unset proxy.http.port must be pinned to \"\" to keep REST on the gRPC port, got %v %v", v, ok) + } + if lookupString(cfg, "woodpecker.storage.rootPath") != "/woodpecker/data" { + t.Fatal("the woodpecker section must be kept as it runs") + } + raw, _ = carriedConfig(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": nil}}}) + _ = yaml.Unmarshal([]byte(raw), &cfg) + if v, ok := lookup(cfg, "proxy.http.port"); !ok || v != "" { + t.Fatalf("a null proxy.http.port must be pinned to \"\" as well, got %v %v", v, ok) + } + raw, _ = carriedConfig(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": float64(8080)}}}) + if !strings.Contains(raw, "port: 8080") { + t.Fatalf("a set proxy.http.port is kept: %s", raw) + } +} + +func TestWalType(t *testing.T) { + for _, tc := range []struct { + cfg map[string]any + want string + }{ + {map[string]any{"mq": map[string]any{"type": "woodpecker"}}, "woodpecker"}, + {map[string]any{"mq": map[string]any{"type": "default"}, "messageQueue": "pulsar"}, "pulsar"}, + {map[string]any{"messageQueue": "kafka"}, "kafka"}, + {map[string]any{}, ""}, + } { + if got := walType(tc.cfg); got != tc.want { + t.Errorf("walType(%v) = %q, want %q", tc.cfg, got, tc.want) + } + } +} + +func TestConfigMismatches(t *testing.T) { + running := chartConfig(t) + carried, _ := carriedConfig(running) + var rendered map[string]any + _ = yaml.Unmarshal([]byte(carried), &rendered) + rendered["etcd"] = map[string]any{"rootPath": "by-dev"} + rendered["minio"].(map[string]any)["port"] = float64(9000) // the operator writes the port as a number + if d := configMismatches(running, rendered); len(d) != 0 { + t.Fatalf("identical settings reported as different: %v", d) + } + + rendered["etcd"] = map[string]any{"rootPath": "mv-etcd"} + rendered["minio"].(map[string]any)["rootPath"] = "files" + rendered["minio"].(map[string]any)["secretAccessKey"] = "other" + d := configMismatches(running, rendered) + if len(d) != 3 { + t.Fatalf("want 3 differences, got %v", d) + } + for _, s := range d { + if strings.Contains(s, "other") || strings.Contains(s, `"sk"`) { + t.Fatalf("a credential leaked into the report: %s", s) + } + } +} + +func ref(name, role string, replicas int32) workloadRef { + sel := map[string]string{"app.kubernetes.io/instance": "rel", "component": role} + t := core.PodTemplateSpec{Spec: core.PodSpec{Containers: []core.Container{{Name: role, Image: "milvusdb/milvus:v2.6.15"}}}} + t.Labels = sel + return workloadRef{Kind: "Deployment", Name: name, Replicas: replicas, Selector: sel, Template: t} +} + +func allRoles() []workloadRef { + return []workloadRef{ + ref("rel-milvus-proxy", "proxy", 3), + ref("rel-milvus-mixcoord", "mixcoord", 2), + ref("rel-milvus-datanode", "datanode", 2), + ref("rel-milvus-querynode", "querynode", 6), + ref("rel-milvus-streamingnode", "streamingnode", 2), + {Kind: "StatefulSet", Name: "rel-etcd", Replicas: 3, Selector: map[string]string{"app": "etcd"}}, + } +} + +func TestAssignUnits(t *testing.T) { + o := sourceOptions{Release: "rel", RoleLabel: "component"} + units, tmpls, fs, err := assignUnits(allRoles(), o) + if err != nil || fs.hasErrors() { + t.Fatalf("err %v, findings %v", err, fs) + } + if len(units) != 5 || len(tmpls) != 5 { + t.Fatalf("want the 5 roles and not etcd, got %v", units) + } + for _, u := range units { + if u.Group != "" { + t.Fatalf("one workload per role needs no group: %+v", u) + } + } + + // Two QueryNode pools become node groups named after the workloads. + refs := allRoles() + refs[3] = ref("rel-milvus-querynode-l40s", "querynode", 4) + refs = append(refs, ref("rel-milvus-querynode-h200", "querynode", 2)) + units, _, fs, err = assignUnits(refs, o) + if err != nil || fs.hasErrors() { + t.Fatalf("err %v, findings %v", err, fs) + } + var groups []string + for _, u := range units { + if u.Role == roleQueryNode { + groups = append(groups, u.Group) + } + } + if !reflect.DeepEqual(groups, []string{"h200", "l40s"}) { + t.Fatalf("groups = %v", groups) + } + + // A pre-2.6 component blocks the adoption; a missing role does too. + refs = append(allRoles()[1:], ref("rel-milvus-rootcoord", "rootcoord", 1)) + _, _, fs, _ = assignUnits(refs, o) + if !fs.hasErrors() || !strings.Contains(findingText(fs), "rootcoord") || !strings.Contains(findingText(fs), "proxy role") { + t.Fatalf("findings %v", fs) + } + + // A role at 0 replicas has nothing to hand over. + refs = allRoles() + refs[0].Replicas = 0 + _, _, fs, _ = assignUnits(refs, o) + if !fs.hasErrors() { + t.Fatal("a workload at 0 replicas must be reported") + } + + // Explicit mapping, for charts without a role label. + refs = allRoles() + for i := range refs { + delete(refs[i].Template.Labels, "component") + } + o.Workloads = []string{ + "proxy=Deployment/rel-milvus-proxy", "mixcoord=Deployment/rel-milvus-mixcoord", "datanode=Deployment/rel-milvus-datanode", + "querynode/gpu=Deployment/rel-milvus-querynode", "streamingnode=Deployment/rel-milvus-streamingnode", + } + units, _, fs, err = assignUnits(refs, o) + if err != nil || fs.hasErrors() || len(units) != 5 { + t.Fatalf("explicit mapping: units %v, findings %v, err %v", units, fs, err) + } +} + +func findingText(fs findings) string { + var b strings.Builder + for _, f := range fs { + b.WriteString(f.Msg + "\n") + } + return b.String() +} + +func TestGroupName(t *testing.T) { + for in, want := range map[string]string{ + "milvus-querynode-h200": "h200", + "milvus-querynode-L40S_gpu": "l40s-gpu", + "qn-pool-a": "qn-pool-a", + } { + if got := groupName(in, roleQueryNode); got != want { + t.Errorf("groupName(%q) = %q, want %q", in, got, want) + } + } +} + +func TestConfigSources(t *testing.T) { + tpl := core.PodTemplateSpec{Spec: core.PodSpec{ + Containers: []core.Container{{Name: "proxy", Image: "milvusdb/milvus:v2.6.15", VolumeMounts: []core.VolumeMount{ + {Name: "milvus-config", MountPath: "/milvus/configs/default.yaml", SubPath: "default.yaml"}, + {Name: "milvus-config", MountPath: "/milvus/configs/user.yaml", SubPath: "user.yaml"}, + {Name: "tools", MountPath: "/milvus/tools"}, + }}}, + Volumes: []core.Volume{ + {Name: "milvus-config", VolumeSource: core.VolumeSource{ConfigMap: &core.ConfigMapVolumeSource{LocalObjectReference: core.LocalObjectReference{Name: "rel-milvus"}}}}, + {Name: "tools", VolumeSource: core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{}}}, + }, + }} + got := configSources(tpl) + if len(got) != 2 || got[0].File != "default.yaml" || got[1].Key != "user.yaml" || got[0].Name != "rel-milvus" { + t.Fatalf("got %+v", got) + } +} + +func TestImageVersionAndPickVersion(t *testing.T) { + v, gpu, err := imageVersion("milvusdb/milvus:v2.6.15-gpu") + if err != nil || v != "2.6.15" || !gpu { + t.Fatalf("got %s %v %v", v, gpu, err) + } + if _, _, err := imageVersion("registry:5000/milvus"); err == nil { + t.Fatal("an image without a tag must fail") + } + mk := func(name, ver string, gpu, deprecated bool) unstructured.Unstructured { + u := unstructured.Unstructured{Object: map[string]any{"spec": map[string]any{ + "version": ver, "deprecated": deprecated, "db": map[string]any{"gpu": map[string]any{"supported": gpu}}, + }}} + u.SetName(name) + return u + } + versions := []unstructured.Unstructured{ + mk("2.6.15", "2.6.15", false, false), mk("2.6.15-gpu", "2.6.15", true, false), + mk("2.6.11", "2.6.11", false, false), mk("2.6.9-old", "2.6.9", false, true), + } + for image, want := range map[string]string{ + "milvusdb/milvus:v2.6.15": "2.6.15", + "milvusdb/milvus:v2.6.15-gpu": "2.6.15-gpu", + } { + if got, err := pickVersion(image, versions); err != nil || got != want { + t.Errorf("pickVersion(%s) = %s, %v; want %s", image, got, err, want) + } + } + for _, image := range []string{"milvusdb/milvus:v2.6.9", "milvusdb/milvus:v2.6.11-gpu"} { + if _, err := pickVersion(image, versions); err == nil { + t.Errorf("pickVersion(%s): a deprecated or missing build must fail", image) + } + } +} + +func TestRenderStagedMilvus(t *testing.T) { + refs := allRoles() + qn := &refs[3].Template.Spec.Containers[0] + qn.Resources = core.ResourceRequirements{ + Requests: core.ResourceList{core.ResourceCPU: resource.MustParse("8"), core.ResourceMemory: resource.MustParse("144Gi")}, + Limits: core.ResourceList{ + core.ResourceMemory: resource.MustParse("144Gi"), "nvidia.com/gpu": resource.MustParse("1"), + "intel.com/sriov_rdma": resource.MustParse("1"), + }, + } + refs[3].Template.Spec.NodeSelector = map[string]string{"pool": "gpu"} + refs[3].Template.Spec.Tolerations = []core.Toleration{{Key: "nvidia.com/gpu", Operator: core.TolerationOpExists, Effect: core.TaintEffectNoSchedule}} + units, tmpls, _, _ := assignUnits(refs, sourceOptions{Release: "rel", RoleLabel: "component"}) + svc := &core.Service{Spec: core.ServiceSpec{Selector: map[string]string{"component": "proxy"}, Ports: []core.ServicePort{{Name: "milvus", Port: 19530}}}} + svc.Name = "rel-milvus" + src := &Source{Units: units, Templates: tmpls, Config: chartConfig(t), Service: svc} + in := renderInput{ + Namespace: "ns", Name: "rel-milvus", Version: "2.6.15-gpu", Release: "rel", Source: src, + RootUser: "root", RootPassword: "S3cret", + } + st := newState(in) + if st.Service == nil || st.Service.Name != "rel-milvus" || len(st.Service.Ports) != 1 { + t.Fatalf("the client Service must be recorded for a rollback: %+v", st.Service) + } + objs, err := render(in, st) + if err != nil { + t.Fatal(err) + } + if len(objs.Secrets) != 3 || objs.Secrets[0].Name != "rel-milvus-auth" || objs.Secrets[0].StringData["password"] != "S3cret" { + t.Fatalf("secrets %v", objs.Secrets) + } + m := objs.Milvus.Object + str := func(path ...string) string { s, _, _ := unstructured.NestedString(m, path...); return s } + if str("spec", "metaStorage", "rootPath") != "by-dev" || str("spec", "authSecret", "name") != "rel-milvus-auth" || + str("spec", "deletionPolicy") != "DoNotTerminate" || str("metadata", "annotations", annOrphanOnDelete) != "true" { + t.Fatalf("Milvus %v", m) + } + if b, _, _ := unstructured.NestedBool(m, "spec", "metaStorage", "externallyManaged"); !b { + t.Fatal("the etcd must be externally managed") + } + for _, role := range milvusRoles { + n, found, _ := unstructured.NestedInt64(m, "spec", "topology", "distributed", role, "replicas") + if !found || n != 0 { + t.Fatalf("%s must be staged at 0 replicas, got %d %v", role, n, found) + } + } + q, _, _ := unstructured.NestedMap(m, "spec", "topology", "distributed", "querynode") + if c, _, _ := unstructured.NestedInt64(q, "gpu", "count"); c != 1 { + t.Fatalf("GPU count not carried: %v", q) + } + ctrs, _, _ := unstructured.NestedSlice(q, "podTemplate", "spec", "containers") + lim := ctrs[0].(map[string]any)["resources"].(map[string]any)["limits"].(map[string]any) + if lim["memory"] != "144Gi" || lim["nvidia.com/gpu"] != nil || lim["intel.com/sriov_rdma"] != nil { + t.Fatalf("limits %v: memory is carried, GPU and SR-IOV devices are not", lim) + } + if ns, _, _ := unstructured.NestedStringMap(q, "podTemplate", "spec", "nodeSelector"); ns["pool"] != "gpu" { + t.Fatalf("nodeSelector not carried: %v", q) + } + + // Without authentication there is no auth Secret. + in.DisableAuth = true + objs, _ = render(in, st) + if len(objs.Secrets) != 2 { + t.Fatalf("want no auth Secret, got %d secrets", len(objs.Secrets)) + } + if b, _, _ := unstructured.NestedBool(objs.Milvus.Object, "spec", "disableSecurity"); !b { + t.Fatal("disableSecurity must be set") + } +} + +func TestRenderNodeGroups(t *testing.T) { + refs := allRoles() + refs[3] = ref("rel-milvus-querynode-l40s", "querynode", 4) + refs = append(refs, ref("rel-milvus-querynode-h200", "querynode", 2)) + units, tmpls, _, _ := assignUnits(refs, sourceOptions{Release: "rel", RoleLabel: "component"}) + spec := distributedSpec(&Source{Units: units, Templates: tmpls}) + groups, _, _ := unstructured.NestedSlice(spec, "querynode", "groups") + if len(groups) != 2 || groups[0].(map[string]any)["name"] != "h200" || groups[0].(map[string]any)["replicas"] != int64(0) { + t.Fatalf("groups %v", groups) + } + u := units[3] + if petSetName("db", u) != "db-querynode-h200" || kubedbSelector("db", u)[labelNodeGroup] != "h200" { + t.Fatalf("group naming: %s %v", petSetName("db", u), kubedbSelector("db", u)) + } +} + +func TestSwapSteps(t *testing.T) { + if got := swapSteps(3, false); !reflect.DeepEqual(got, []step{{3, 0}}) { + t.Fatalf("whole role: %v", got) + } + if got := swapSteps(3, true); !reflect.DeepEqual(got, []step{{1, 2}, {2, 1}, {3, 0}}) { + t.Fatalf("one by one: %v", got) + } +} + +func TestHandoverOrder(t *testing.T) { + if o := handoverOrder(methodOverlap); o[0] != roleProxy || o[len(o)-1] != roleQueryNode { + t.Fatalf("overlap moves the proxies first and the QueryNodes last: %v", o) + } + if o := handoverOrder(methodStopStart); o[0] != roleMixCoord || o[len(o)-1] != roleProxy { + t.Fatalf("stop-start moves the coordinators first and the proxies last: %v", o) + } + if len(handoverOrder(methodOverlap)) != len(milvusRoles) || len(rollbackOrder) != len(milvusRoles) { + t.Fatal("every role is handed over and rolled back") + } +} + +func TestStateDone(t *testing.T) { + st := &State{} + u := Unit{Role: roleQueryNode, Group: "h200"} + st.markDone(u) + st.markDone(u) + if !st.isDone(u) || len(st.Done) != 1 || st.Done[0] != "querynode/h200" { + t.Fatalf("done %v", st.Done) + } + st.unmarkDone(u) + if st.isDone(u) { + t.Fatal("unmarkDone") + } +} + +func TestFallbackServiceRecord(t *testing.T) { + // A client Service KubeDB took over: its ports target the KubeDB container's + // port names. The chart's proxy names its gRPC port "milvus". + svc := core.Service{Spec: core.ServiceSpec{Ports: []core.ServicePort{ + {Name: "proxy", Port: 19530, TargetPort: intstr.FromString("proxy")}, + {Name: "metrics", Port: 9091, TargetPort: intstr.FromString("metrics")}, + {Name: "http", Port: 8080, TargetPort: intstr.FromString("http")}, + {Name: "admin", Port: 9999, TargetPort: intstr.FromInt32(9999)}, + }}} + svc.Name = "rel-milvus" + proxy := Unit{Role: roleProxy, Selector: map[string]string{"component": "proxy"}} + tpl := core.PodTemplateSpec{Spec: core.PodSpec{Containers: []core.Container{{ + Name: "proxy", Image: "milvusdb/milvus:v2.6.15", + Ports: []core.ContainerPort{{Name: "milvus", ContainerPort: 19530}, {Name: "metrics", ContainerPort: 9091}}, + }}}} + rec := fallbackServiceRecord(svc, proxy, tpl) + if !reflect.DeepEqual(rec.Selector, proxy.Selector) { + t.Fatalf("selector %v", rec.Selector) + } + got := make([]string, 0, len(rec.Ports)) + for _, p := range rec.Ports { + got = append(got, p.Name+"->"+p.TargetPort.String()) + } + want := []string{"proxy->19530", "metrics->metrics", "admin->9999"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ports %v, want %v (8080 only exists on KubeDB pods)", got, want) + } +} + +func TestRestPort(t *testing.T) { + if restPort(map[string]any{}) != 19530 { + t.Fatal("default") + } + if restPort(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": ""}, "port": float64(19531)}}) != 19531 { + t.Fatal("an empty http port serves REST on the gRPC port") + } + if restPort(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": float64(8080)}}}) != 8080 { + t.Fatal("a set http port") + } +} diff --git a/pkg/cmds/adopt.go b/pkg/cmds/adopt.go new file mode 100644 index 000000000..76fdd243e --- /dev/null +++ b/pkg/cmds/adopt.go @@ -0,0 +1,26 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmds + +import ( + "kubedb.dev/cli/pkg/adopt" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +// NewCmdAdopt returns the command group that adopts a running database in place. +func NewCmdAdopt(f cmdutil.Factory) *cobra.Command { + return adopt.NewCmdAdopt(f) +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 1ab74a0f1..d9c583920 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -114,6 +114,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command { NewCmdMSSQL(f), }, }, + { + Message: "Adopt a running database in place", + Commands: []*cobra.Command{ + NewCmdAdopt(f), + }, + }, { Message: "Cross Data Center DR Commands", Commands: []*cobra.Command{ From c8b3c52cf7c23a36e9a4439a665e574b0ee1661d Mon Sep 17 00:00:00 2001 From: sabnaj Date: Fri, 30 Oct 2026 23:16:58 +0600 Subject: [PATCH 2/4] adopt milvus: read the options from a file with -f For a chart without the upstream labels, plan and run need about fifteen flags (a --workload per workload, the secrets, the timing), repeated on every command. All four commands now take -f FILE, kind MilvusAdoption (apiVersion cli.kubedb.com/v1alpha1): release, namespace, name, version, source (selector, roleLabel, workloads with node groups, configFile), auth, restrictedSCC, handover (method and timing) and rollback (cleanup). - A flag given on the command line wins; the file fills only the flags that were not given, through the flag set, so the parsing is shared. - RELEASE may come from the file; an argument that differs is refused. The file's namespace is used unless -n is given. - Unknown keys are errors, and --yes is never read from the file. - plan --output-dir also writes adopt.yaml with what plan found, ready for run -f. Tested on the cluster: rollback, plan and run driven by the file alone. Signed-off-by: sabnaj --- pkg/adopt/milvus.go | 65 ++++---- pkg/adopt/milvus_cluster.go | 9 +- pkg/adopt/milvus_file.go | 290 ++++++++++++++++++++++++++++++++++++ pkg/adopt/milvus_test.go | 156 +++++++++++++++++++ 4 files changed, 493 insertions(+), 27 deletions(-) create mode 100644 pkg/adopt/milvus_file.go diff --git a/pkg/adopt/milvus.go b/pkg/adopt/milvus.go index 8e6e5adb8..3f3f4ddeb 100644 --- a/pkg/adopt/milvus.go +++ b/pkg/adopt/milvus.go @@ -85,18 +85,12 @@ func addTimingFlags(cmd *cobra.Command, t *timing) { cmd.Flags().DurationVar(&t.QueryNodeDelay, "querynode-delay", time.Minute, "Overlap: time a new QueryNode serves before an old one stops (raise it for large collections)") } -func releaseArg(args []string) (string, error) { - if len(args) != 1 { - return "", errors.New("give the RELEASE (the Helm release, or the instance label value of the workloads)") - } - return args[0], nil -} - func newCmdMilvusPlan(f cmdutil.Factory) *cobra.Command { var o planOptions + var fo fileOptions var outDir string cmd := &cobra.Command{ - Use: "plan RELEASE", + Use: "plan [RELEASE]", Short: i18n.T("Check a running Milvus and show what adopting it creates"), Long: templates.LongDesc(` Read-only. Finds the workloads, effective configuration, image and client @@ -105,19 +99,22 @@ func newCmdMilvusPlan(f cmdutil.Factory) *cobra.Command { MilvusVersion, the current root password), and shows the Secrets, the state ConfigMap and the staged Milvus that "run" would create. - With --output-dir the objects are written there, unredacted. Edit them if - needed and apply them; "run" then uses what exists.`), + With --output-dir the objects are written there, unredacted, together with + adopt.yaml: every option of this adoption as plan found it. Review it and + pass it to run, status and rollback with -f instead of repeating the flags. + Edit the objects if needed and apply them; "run" then uses what exists.`), Example: templates.Examples(` kubectl dba adopt milvus plan my-release -n milvus - kubectl dba adopt milvus plan my-release -n milvus --output-dir ./adopt`), + kubectl dba adopt milvus plan my-release -n milvus --output-dir ./adopt + kubectl dba adopt milvus plan -f ./adopt/adopt.yaml`), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { var err error - if o.Release, err = releaseArg(args); err != nil { + if o.Release, err = fo.resolve(cmd, args); err != nil { return err } ctx := context.Background() - c, err := newCluster(f, cmd.OutOrStdout()) + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) if err != nil { return err } @@ -139,6 +136,11 @@ func newCmdMilvusPlan(f cmdutil.Factory) *cobra.Command { if err := os.MkdirAll(outDir, 0o700); err != nil { return err } + af, err := adoptionFileFor(p, o, c.ns) + if err != nil { + return err + } + docs[adoptionFileName] = string(af) for _, name := range sortedKeys(docs) { if err := os.WriteFile(filepath.Join(outDir, name), []byte(docs[name]), 0o600); err != nil { return err @@ -151,11 +153,16 @@ func newCmdMilvusPlan(f cmdutil.Factory) *cobra.Command { } } printHandoverPlan(out, p.State, methodOverlap) - _, _ = fmt.Fprintf(out, "\nNext: kubectl dba adopt milvus run %s -n %s --yes\n", o.Release, c.ns) + if outDir != "" { + _, _ = fmt.Fprintf(out, "\nNext: kubectl dba adopt milvus run -f %s --yes\n", filepath.Join(outDir, adoptionFileName)) + } else { + _, _ = fmt.Fprintf(out, "\nNext: kubectl dba adopt milvus run %s -n %s --yes\n", o.Release, c.ns) + } return nil }, } addPlanFlags(cmd, &o) + addFileFlag(cmd, &fo) cmd.Flags().StringVar(&outDir, "output-dir", "", "Write the objects to this directory (Secrets unredacted, mode 0600)") return cmd } @@ -189,8 +196,9 @@ type runOptions struct { func newCmdMilvusRun(f cmdutil.Factory) *cobra.Command { var o runOptions + var fo fileOptions cmd := &cobra.Command{ - Use: "run RELEASE", + Use: "run [RELEASE]", Short: i18n.T("Adopt a running Milvus: create the Milvus object and hand the roles over"), Long: templates.LongDesc(` Runs the checks of "plan", creates the Secrets, the state ConfigMap @@ -215,17 +223,18 @@ func newCmdMilvusRun(f cmdutil.Factory) *cobra.Command { the client Service that KubeDB now uses.`), Example: templates.Examples(` kubectl dba adopt milvus run my-release -n milvus --yes - kubectl dba adopt milvus run my-release -n milvus --method stop-start --yes`), + kubectl dba adopt milvus run my-release -n milvus --method stop-start --yes + kubectl dba adopt milvus run -f ./adopt/adopt.yaml --yes`), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { var err error - if o.Release, err = releaseArg(args); err != nil { + if o.Release, err = fo.resolve(cmd, args); err != nil { return err } if o.Method != methodOverlap && o.Method != methodStopStart { return fmt.Errorf("--method must be %s or %s", methodOverlap, methodStopStart) } - c, err := newCluster(f, cmd.OutOrStdout()) + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) if err != nil { return err } @@ -234,6 +243,7 @@ func newCmdMilvusRun(f cmdutil.Factory) *cobra.Command { } addPlanFlags(cmd, &o.planOptions) addTimingFlags(cmd, &o.timing) + addFileFlag(cmd, &fo) cmd.Flags().StringVar(&o.Method, "method", methodOverlap, "Handover method: overlap or stop-start") cmd.Flags().BoolVar(&o.Yes, "yes", false, "Proceed; without it, run only shows what it would do") cmd.Flags().BoolVar(&o.SkipConfigCheck, "skip-config-check", false, "Do not stop when the rendered config differs from the running one") @@ -334,18 +344,19 @@ Next: } func newCmdMilvusStatus(f cmdutil.Factory) *cobra.Command { + var fo fileOptions cmd := &cobra.Command{ - Use: "status RELEASE", + Use: "status [RELEASE]", Short: i18n.T("Show where each role of an adoption runs"), Example: templates.Examples(`kubectl dba adopt milvus status my-release -n milvus`), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - release, err := releaseArg(args) + release, err := fo.resolve(cmd, args) if err != nil { return err } ctx := context.Background() - c, err := newCluster(f, cmd.OutOrStdout()) + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) if err != nil { return err } @@ -359,6 +370,7 @@ func newCmdMilvusStatus(f cmdutil.Factory) *cobra.Command { return c.printStatus(ctx, st) }, } + addFileFlag(cmd, &fo) return cmd } @@ -409,8 +421,9 @@ type rollbackOptions struct { func newCmdMilvusRollback(f cmdutil.Factory) *cobra.Command { var o rollbackOptions + var fo fileOptions cmd := &cobra.Command{ - Use: "rollback RELEASE", + Use: "rollback [RELEASE]", Short: i18n.T("Give an adopted (or partly adopted) Milvus back to the chart's workloads"), Long: templates.LongDesc(` Deletes the Milvus object with kubedb.com/milvus-orphan-on-delete, so its @@ -427,14 +440,15 @@ func newCmdMilvusRollback(f cmdutil.Factory) *cobra.Command { chart's replicas are then taken from the PetSets, and the client Service gets the chart proxies' selector.`), Example: templates.Examples(` - kubectl dba adopt milvus rollback my-release -n milvus --yes --cleanup`), + kubectl dba adopt milvus rollback my-release -n milvus --yes --cleanup + kubectl dba adopt milvus rollback -f ./adopt/adopt.yaml --yes`), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { var err error - if o.Release, err = releaseArg(args); err != nil { + if o.Release, err = fo.resolve(cmd, args); err != nil { return err } - c, err := newCluster(f, cmd.OutOrStdout()) + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) if err != nil { return err } @@ -443,6 +457,7 @@ func newCmdMilvusRollback(f cmdutil.Factory) *cobra.Command { } addSourceFlags(cmd, &o.sourceOptions) addTimingFlags(cmd, &o.timing) + addFileFlag(cmd, &fo) cmd.Flags().StringVar(&o.Name, "name", "", "Name of the Milvus, for an adoption done by hand (no state ConfigMap)") cmd.Flags().BoolVar(&o.Yes, "yes", false, "Proceed; without it, rollback only shows what it would do") cmd.Flags().BoolVar(&o.Cleanup, "cleanup", false, "Delete what KubeDB left behind once the chart serves again (PVCs are kept)") diff --git a/pkg/adopt/milvus_cluster.go b/pkg/adopt/milvus_cluster.go index b21810bc5..46d8ebf7c 100644 --- a/pkg/adopt/milvus_cluster.go +++ b/pkg/adopt/milvus_cluster.go @@ -43,11 +43,16 @@ type cluster struct { poll time.Duration } -func newCluster(f cmdutil.Factory, out io.Writer) (*cluster, error) { - ns, _, err := f.ToRawKubeConfigLoader().Namespace() +// newCluster connects to the cluster. fileNamespace, from an adoption file, is +// used when -n is not given. +func newCluster(f cmdutil.Factory, out io.Writer, fileNamespace string) (*cluster, error) { + ns, explicit, err := f.ToRawKubeConfigLoader().Namespace() if err != nil { return nil, err } + if !explicit && fileNamespace != "" { + ns = fileNamespace + } cfg, err := f.ToRESTConfig() if err != nil { return nil, err diff --git a/pkg/adopt/milvus_file.go b/pkg/adopt/milvus_file.go new file mode 100644 index 000000000..f9bf35905 --- /dev/null +++ b/pkg/adopt/milvus_file.go @@ -0,0 +1,290 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "errors" + "fmt" + "os" + "strconv" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +const ( + adoptionFileAPIVersion = "cli.kubedb.com/v1alpha1" + adoptionFileKind = "MilvusAdoption" + // adoptionFileName is the file "plan --output-dir" writes. + adoptionFileName = "adopt.yaml" +) + +// adoptionFile holds the options of an adoption, so they need not be typed as +// flags on every command. A flag given on the command line wins over the file. +type adoptionFile struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Release string `json:"release"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Source adoptionSource `json:"source,omitempty"` + Auth adoptionAuth `json:"auth,omitempty"` + RestrictedSCC *bool `json:"restrictedSCC,omitempty"` + Handover adoptionHandover `json:"handover,omitempty"` + Rollback adoptionRollback `json:"rollback,omitempty"` +} + +type adoptionSource struct { + Selector string `json:"selector,omitempty"` + RoleLabel string `json:"roleLabel,omitempty"` + Workloads []adoptionWorkload `json:"workloads,omitempty"` + ConfigFile string `json:"configFile,omitempty"` +} + +type adoptionWorkload struct { + Role string `json:"role"` + Group string `json:"group,omitempty"` + Kind string `json:"kind"` + Name string `json:"name"` +} + +type adoptionAuth struct { + AuthSecret string `json:"authSecret,omitempty"` + RootPasswordFile string `json:"rootPasswordFile,omitempty"` + EtcdAuthSecret string `json:"etcdAuthSecret,omitempty"` + SkipCredentialCheck *bool `json:"skipCredentialCheck,omitempty"` +} + +type adoptionHandover struct { + Method string `json:"method,omitempty"` + QueryNodeDelay *metav1.Duration `json:"querynodeDelay,omitempty"` + Settle *metav1.Duration `json:"settle,omitempty"` + Timeout *metav1.Duration `json:"timeout,omitempty"` + ReadyTimeout *metav1.Duration `json:"readyTimeout,omitempty"` + SkipConfigCheck *bool `json:"skipConfigCheck,omitempty"` +} + +type adoptionRollback struct { + Cleanup *bool `json:"cleanup,omitempty"` +} + +// loadAdoptionFile reads and checks an adoption file. Unknown keys are errors, +// so a misspelt option is not silently ignored. +func loadAdoptionFile(path string) (*adoptionFile, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + af := &adoptionFile{} + if err := yaml.UnmarshalStrict(raw, af); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if err := af.validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return af, nil +} + +func (af *adoptionFile) validate() error { + if af.APIVersion != adoptionFileAPIVersion || af.Kind != adoptionFileKind { + return fmt.Errorf("want apiVersion %s and kind %s, got %q and %q", adoptionFileAPIVersion, adoptionFileKind, af.APIVersion, af.Kind) + } + if af.Release == "" { + return errors.New("release is required") + } + if m := af.Handover.Method; m != "" && m != methodOverlap && m != methodStopStart { + return fmt.Errorf("handover.method must be %s or %s, got %q", methodOverlap, methodStopStart, m) + } + for i, w := range af.Source.Workloads { + if !contains(milvusRoles, w.Role) { + return fmt.Errorf("source.workloads[%d]: role %q is not one of %v", i, w.Role, milvusRoles) + } + if w.Kind != "Deployment" && w.Kind != "StatefulSet" { + return fmt.Errorf("source.workloads[%d]: kind must be Deployment or StatefulSet, got %q", i, w.Kind) + } + if w.Name == "" { + return fmt.Errorf("source.workloads[%d]: name is required", i) + } + } + return nil +} + +// workloadFlag renders a workload the way --workload takes it. +func (w adoptionWorkload) workloadFlag() string { + role := w.Role + if w.Group != "" { + role += "/" + w.Group + } + return role + "=" + w.Kind + "/" + w.Name +} + +type flagValue struct { + flag string + values []string +} + +// flagValues maps the file onto the command's flags. +func (af *adoptionFile) flagValues() []flagValue { + var out []flagValue + str := func(flag, v string) { + if v != "" { + out = append(out, flagValue{flag, []string{v}}) + } + } + boolean := func(flag string, v *bool) { + if v != nil { + out = append(out, flagValue{flag, []string{strconv.FormatBool(*v)}}) + } + } + duration := func(flag string, v *metav1.Duration) { + if v != nil { + out = append(out, flagValue{flag, []string{v.Duration.String()}}) + } + } + str("name", af.Name) + str("version", af.Version) + str("selector", af.Source.Selector) + str("role-label", af.Source.RoleLabel) + if len(af.Source.Workloads) > 0 { + fv := flagValue{flag: "workload"} + for _, w := range af.Source.Workloads { + fv.values = append(fv.values, w.workloadFlag()) + } + out = append(out, fv) + } + str("config-file", af.Source.ConfigFile) + str("auth-secret", af.Auth.AuthSecret) + str("root-password-file", af.Auth.RootPasswordFile) + str("etcd-auth-secret", af.Auth.EtcdAuthSecret) + boolean("skip-credential-check", af.Auth.SkipCredentialCheck) + boolean("restricted-scc", af.RestrictedSCC) + str("method", af.Handover.Method) + duration("querynode-delay", af.Handover.QueryNodeDelay) + duration("settle", af.Handover.Settle) + duration("timeout", af.Handover.Timeout) + duration("ready-timeout", af.Handover.ReadyTimeout) + boolean("skip-config-check", af.Handover.SkipConfigCheck) + boolean("cleanup", af.Rollback.Cleanup) + return out +} + +// applyFlags sets every flag of fs that the file gives and the command line +// did not. Options a command does not have are skipped. +func (af *adoptionFile) applyFlags(fs *pflag.FlagSet) error { + for _, fv := range af.flagValues() { + fl := fs.Lookup(fv.flag) + if fl == nil || fl.Changed { + continue + } + for _, v := range fv.values { + if err := fs.Set(fv.flag, v); err != nil { + return fmt.Errorf("--%s from the file: %w", fv.flag, err) + } + } + } + return nil +} + +// fileOptions is the -f flag every adopt milvus command has. +type fileOptions struct { + Filename string + file *adoptionFile +} + +func addFileFlag(cmd *cobra.Command, o *fileOptions) { + cmd.Flags().StringVarP(&o.Filename, "filename", "f", "", "File with the adoption options (kind MilvusAdoption; \"plan --output-dir\" writes one). Flags given on the command line win") +} + +// resolve loads the file, fills the flags it gives, and returns the release: +// the RELEASE argument, or release from the file. +func (o *fileOptions) resolve(cmd *cobra.Command, args []string) (string, error) { + if len(args) > 1 { + return "", errors.New("give at most one RELEASE") + } + if o.Filename != "" { + af, err := loadAdoptionFile(o.Filename) + if err != nil { + return "", err + } + if err := af.applyFlags(cmd.Flags()); err != nil { + return "", err + } + o.file = af + } + switch { + case len(args) == 1 && o.file != nil && o.file.Release != args[0]: + return "", fmt.Errorf("RELEASE %q differs from release %q in %s", args[0], o.file.Release, o.Filename) + case len(args) == 1: + return args[0], nil + case o.file != nil: + return o.file.Release, nil + } + return "", errors.New("give the RELEASE (the Helm release, or the instance label value of the workloads), or a file with -f") +} + +// namespace is the file's namespace, used when -n is not given. +func (o *fileOptions) namespace() string { + if o.file == nil { + return "" + } + return o.file.Namespace +} + +// adoptionFileFor writes down what a plan found, as a file to review and pass +// to run, status and rollback with -f. +func adoptionFileFor(p *adoptPlan, o planOptions, ns string) ([]byte, error) { + version, _ := p.Objects.Milvus.Object["spec"].(map[string]any)["version"].(string) + af := adoptionFile{ + APIVersion: adoptionFileAPIVersion, + Kind: adoptionFileKind, + Release: o.Release, + Namespace: ns, + Name: p.State.Name, + Version: version, + Source: adoptionSource{ConfigFile: o.ConfigFile}, + Auth: adoptionAuth{ + AuthSecret: o.AuthSecret, + RootPasswordFile: o.RootPasswordFile, + EtcdAuthSecret: o.EtcdAuthSecret, + }, + Handover: adoptionHandover{ + Method: methodOverlap, + QueryNodeDelay: &metav1.Duration{Duration: time.Minute}, + Settle: &metav1.Duration{Duration: 20 * time.Second}, + Timeout: &metav1.Duration{Duration: 15 * time.Minute}, + ReadyTimeout: &metav1.Duration{Duration: 15 * time.Minute}, + }, + Rollback: adoptionRollback{Cleanup: ptrBool(false)}, + } + if o.RestrictedSCC { + af.RestrictedSCC = ptrBool(true) + } + for _, u := range p.State.Units { + af.Source.Workloads = append(af.Source.Workloads, adoptionWorkload{Role: u.Role, Group: u.Group, Kind: u.Kind, Name: u.Name}) + } + b, err := yaml.Marshal(af) + if err != nil { + return nil, err + } + header := "# Options of the adoption, as plan found them. Review, then:\n" + + "# kubectl dba adopt milvus run -f " + adoptionFileName + " --yes\n" + + "# Raise handover.querynodeDelay for large collections (about the time one QueryNode takes to load).\n" + return append([]byte(header), b...), nil +} + +func ptrBool(b bool) *bool { return &b } diff --git a/pkg/adopt/milvus_test.go b/pkg/adopt/milvus_test.go index 5393f3a65..8d65656bb 100644 --- a/pkg/adopt/milvus_test.go +++ b/pkg/adopt/milvus_test.go @@ -14,10 +14,14 @@ limitations under the License. package adopt import ( + "os" + "path/filepath" "reflect" "strings" "testing" + "time" + "github.com/spf13/cobra" core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -505,3 +509,155 @@ func TestRestPort(t *testing.T) { t.Fatal("a set http port") } } + +const adoptionYAML = ` +apiVersion: cli.kubedb.com/v1alpha1 +kind: MilvusAdoption +release: pura-milvus +namespace: milvus-prod +source: + workloads: + - {role: proxy, kind: Deployment, name: mv-proxy} + - {role: querynode, group: l40s, kind: Deployment, name: mv-querynode-l40s} +auth: + authSecret: file-secret +restrictedSCC: true +handover: + method: stop-start + querynodeDelay: 10m +rollback: + cleanup: true +` + +func writeTemp(t *testing.T, content string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "adopt.yaml") + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestLoadAdoptionFile(t *testing.T) { + af, err := loadAdoptionFile(writeTemp(t, adoptionYAML)) + if err != nil { + t.Fatal(err) + } + if af.Release != "pura-milvus" || af.Namespace != "milvus-prod" || af.Handover.QueryNodeDelay.Duration != 10*time.Minute { + t.Fatalf("got %+v", af) + } + for name, content := range map[string]string{ + "unknown key": adoptionYAML + "querynodeDelay: 5m\n", + "wrong kind": strings.Replace(adoptionYAML, "kind: MilvusAdoption", "kind: Milvus", 1), + "no release": strings.Replace(adoptionYAML, "release: pura-milvus", "", 1), + "bad method": strings.Replace(adoptionYAML, "method: stop-start", "method: fast", 1), + "bad role": strings.Replace(adoptionYAML, "role: proxy", "role: rootcoord", 1), + "bad kind": strings.Replace(adoptionYAML, "{role: proxy, kind: Deployment", "{role: proxy, kind: Pod", 1), + "bad duration": strings.Replace(adoptionYAML, "querynodeDelay: 10m", "querynodeDelay: ten", 1), + "workload no name": strings.Replace(adoptionYAML, ", name: mv-proxy}", ", name: \"\"}", 1), + } { + if _, err := loadAdoptionFile(writeTemp(t, content)); err == nil { + t.Errorf("%s: want an error", name) + } + } +} + +func newRunCommandForTest(o *runOptions, fo *fileOptions) *cobra.Command { + cmd := &cobra.Command{Use: "run", RunE: func(*cobra.Command, []string) error { return nil }} + addPlanFlags(cmd, &o.planOptions) + addTimingFlags(cmd, &o.timing) + addFileFlag(cmd, fo) + cmd.Flags().StringVar(&o.Method, "method", methodOverlap, "") + cmd.Flags().DurationVar(&o.ReadyTimeout, "ready-timeout", 15*time.Minute, "") + return cmd +} + +func TestAdoptionFileFillsUnsetFlagsOnly(t *testing.T) { + var o runOptions + var fo fileOptions + cmd := newRunCommandForTest(&o, &fo) + path := writeTemp(t, adoptionYAML) + if err := cmd.ParseFlags([]string{"-f", path, "--auth-secret", "cli-secret", "--settle", "5s"}); err != nil { + t.Fatal(err) + } + release, err := fo.resolve(cmd, nil) + if err != nil { + t.Fatal(err) + } + if release != "pura-milvus" || fo.namespace() != "milvus-prod" { + t.Fatalf("release %q namespace %q", release, fo.namespace()) + } + if o.AuthSecret != "cli-secret" { + t.Fatalf("a flag on the command line must win over the file, got %q", o.AuthSecret) + } + if o.Method != methodStopStart || o.QueryNodeDelay != 10*time.Minute || !o.RestrictedSCC { + t.Fatalf("file values not applied: %+v", o) + } + if o.Settle != 5*time.Second || o.Timeout != 15*time.Minute { + t.Fatalf("flag and default must stay: settle %s timeout %s", o.Settle, o.Timeout) + } + want := []string{"proxy=Deployment/mv-proxy", "querynode/l40s=Deployment/mv-querynode-l40s"} + if !reflect.DeepEqual(o.Workloads, want) { + t.Fatalf("workloads %v", o.Workloads) + } +} + +func TestResolveRelease(t *testing.T) { + var o runOptions + var fo fileOptions + cmd := newRunCommandForTest(&o, &fo) + if _, err := fo.resolve(cmd, nil); err == nil { + t.Fatal("no RELEASE and no file must fail") + } + if r, err := fo.resolve(cmd, []string{"rel"}); err != nil || r != "rel" { + t.Fatalf("got %q %v", r, err) + } + if err := cmd.ParseFlags([]string{"-f", writeTemp(t, adoptionYAML)}); err != nil { + t.Fatal(err) + } + if _, err := fo.resolve(cmd, []string{"other"}); err == nil { + t.Fatal("a RELEASE that differs from the file must fail") + } +} + +func TestAdoptionFileForRoundTrip(t *testing.T) { + refs := allRoles() + refs[3] = ref("rel-milvus-querynode-l40s", "querynode", 4) + refs = append(refs, ref("rel-milvus-querynode-h200", "querynode", 2)) + units, tmpls, _, _ := assignUnits(refs, sourceOptions{Release: "rel", RoleLabel: "component"}) + in := renderInput{ + Namespace: "ns", Name: "rel-milvus", Version: "2.6.15-gpu", Release: "rel", + Source: &Source{Units: units, Templates: tmpls, Config: chartConfig(t)}, RootUser: "root", RootPassword: "x", + } + st := newState(in) + objs, err := render(in, st) + if err != nil { + t.Fatal(err) + } + o := planOptions{sourceOptions: sourceOptions{Release: "rel"}, AuthSecret: "milvus-root", RestrictedSCC: true} + raw, err := adoptionFileFor(&adoptPlan{State: st, Objects: objs}, o, "ns") + if err != nil { + t.Fatal(err) + } + af, err := loadAdoptionFile(writeTemp(t, string(raw))) + if err != nil { + t.Fatalf("the file plan writes must load again: %v\n%s", err, raw) + } + if af.Release != "rel" || af.Namespace != "ns" || af.Name != "rel-milvus" || af.Version != "2.6.15-gpu" || + af.Auth.AuthSecret != "milvus-root" || af.RestrictedSCC == nil || !*af.RestrictedSCC || len(af.Source.Workloads) != 6 { + t.Fatalf("got %+v", af) + } + var o2 runOptions + var fo fileOptions + cmd := newRunCommandForTest(&o2, &fo) + if err := cmd.ParseFlags([]string{"-f", writeTemp(t, string(raw))}); err != nil { + t.Fatal(err) + } + if _, err := fo.resolve(cmd, nil); err != nil { + t.Fatal(err) + } + gotUnits, _, fs, err := assignUnits(refs, o2.sourceOptions) + if err != nil || fs.hasErrors() || !reflect.DeepEqual(gotUnits, units) { + t.Fatalf("the written workloads must select the same units: %v %v\n%v\n%v", err, fs, gotUnits, units) + } +} From b9538fb8ac9ea627ec7593ffdb361531941c584f Mon Sep 17 00:00:00 2001 From: sabnaj Date: Fri, 30 Oct 2026 23:32:45 +0600 Subject: [PATCH 3/4] adopt milvus: take the command in applyFlags, keep pflag indirect Signed-off-by: sabnaj --- pkg/adopt/milvus_file.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/adopt/milvus_file.go b/pkg/adopt/milvus_file.go index f9bf35905..43b5ac038 100644 --- a/pkg/adopt/milvus_file.go +++ b/pkg/adopt/milvus_file.go @@ -21,7 +21,6 @@ import ( "time" "github.com/spf13/cobra" - "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" ) @@ -183,9 +182,10 @@ func (af *adoptionFile) flagValues() []flagValue { return out } -// applyFlags sets every flag of fs that the file gives and the command line +// applyFlags sets every flag of cmd that the file gives and the command line // did not. Options a command does not have are skipped. -func (af *adoptionFile) applyFlags(fs *pflag.FlagSet) error { +func (af *adoptionFile) applyFlags(cmd *cobra.Command) error { + fs := cmd.Flags() for _, fv := range af.flagValues() { fl := fs.Lookup(fv.flag) if fl == nil || fl.Changed { @@ -221,7 +221,7 @@ func (o *fileOptions) resolve(cmd *cobra.Command, args []string) (string, error) if err != nil { return "", err } - if err := af.applyFlags(cmd.Flags()); err != nil { + if err := af.applyFlags(cmd); err != nil { return "", err } o.file = af From ff23117eca241af34a2310a03ab0dc8cfe2159d9 Mon Sep 17 00:00:00 2001 From: sabnaj Date: Sun, 1 Nov 2026 13:25:47 +0600 Subject: [PATCH 4/4] adopt milvus: keep the TLS the cluster runs with plan finds client, internal and etcd TLS in the running config, traces each file to its Secret and reuses it, or copies it when its keys differ. A missing mTLS client certificate comes from --tls-client-secret or --tls-issuer. TLS that is off stays off. Certificates that cannot serve the cluster stop plan, and the old paths are dropped from the carried config, because it would win the merge over what the operator renders. Signed-off-by: sabnaj --- pkg/adopt/milvus.go | 2 + pkg/adopt/milvus_auth.go | 9 +- pkg/adopt/milvus_cluster.go | 3 + pkg/adopt/milvus_config.go | 17 ++ pkg/adopt/milvus_file.go | 9 + pkg/adopt/milvus_handover.go | 18 ++ pkg/adopt/milvus_plan.go | 8 +- pkg/adopt/milvus_render.go | 25 ++ pkg/adopt/milvus_tls.go | 504 +++++++++++++++++++++++++++++++++++ pkg/adopt/milvus_tls_test.go | 245 +++++++++++++++++ 10 files changed, 835 insertions(+), 5 deletions(-) create mode 100644 pkg/adopt/milvus_tls.go create mode 100644 pkg/adopt/milvus_tls_test.go diff --git a/pkg/adopt/milvus.go b/pkg/adopt/milvus.go index 3f3f4ddeb..331cc7a55 100644 --- a/pkg/adopt/milvus.go +++ b/pkg/adopt/milvus.go @@ -74,6 +74,8 @@ func addPlanFlags(cmd *cobra.Command, o *planOptions) { cmd.Flags().StringVar(&o.AuthSecret, "auth-secret", "", "Existing basic-auth Secret with the current root user and password") cmd.Flags().StringVar(&o.RootPasswordFile, "root-password-file", "", "File holding the current root password (default: common.security.defaultRootPassword of the running config)") cmd.Flags().StringVar(&o.EtcdAuthSecret, "etcd-auth-secret", "", "Basic-auth Secret for an etcd that uses authentication") + cmd.Flags().StringVar(&o.TLSIssuer, "tls-issuer", "", "cert-manager issuer for a certificate the cluster does not have, as name, Issuer/name or ClusterIssuer/name") + cmd.Flags().StringVar(&o.TLSClientSecret, "tls-client-secret", "", "Existing Secret (tls.crt, tls.key, ca.crt) with the client certificate KubeDB presents when client mTLS is on") cmd.Flags().StringVar(&o.ConfigFile, "config-file", "", "The effective milvus.yaml, when it is not mounted from a ConfigMap or Secret") cmd.Flags().BoolVar(&o.RestrictedSCC, "restricted-scc", false, "Set spec.restrictedSCC (OpenShift)") cmd.Flags().BoolVar(&o.SkipCredentialCheck, "skip-credential-check", false, "Do not log in to the running cluster to check the root credentials") diff --git a/pkg/adopt/milvus_auth.go b/pkg/adopt/milvus_auth.go index 23f6af7a6..56d9eee95 100644 --- a/pkg/adopt/milvus_auth.go +++ b/pkg/adopt/milvus_auth.go @@ -70,7 +70,12 @@ func (c *cluster) checkCredentials(ctx context.Context, proxy Unit, restPort int } defer tunnel.Close() - url := "http://127.0.0.1:" + strconv.Itoa(tunnel.Local) + "/v2/vectordb/collections/list" + scheme, httpc := "http", http.DefaultClient + if c.restTLS != nil { + scheme = "https" + httpc = &http.Client{Transport: &http.Transport{TLSClientConfig: c.restTLS}} + } + url := scheme + "://127.0.0.1:" + strconv.Itoa(tunnel.Local) + "/v2/vectordb/collections/list" reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, url, bytes.NewBufferString("{}")) @@ -79,7 +84,7 @@ func (c *cluster) checkCredentials(ctx context.Context, proxy Unit, restPort int } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+user+":"+password) - resp, err := http.DefaultClient.Do(req) + resp, err := httpc.Do(req) if err != nil { return credentialsUnverified, fmt.Sprintf("RESTful API of %s on port %d: %v", pod, restPort, err) } diff --git a/pkg/adopt/milvus_cluster.go b/pkg/adopt/milvus_cluster.go index 46d8ebf7c..e24d0d72f 100644 --- a/pkg/adopt/milvus_cluster.go +++ b/pkg/adopt/milvus_cluster.go @@ -15,6 +15,7 @@ package adopt import ( "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -41,6 +42,8 @@ type cluster struct { out io.Writer // poll is the interval of every wait. poll time.Duration + // restTLS is set when the running proxies serve TLS. + restTLS *tls.Config } // newCluster connects to the cluster. fileNamespace, from an adoption file, is diff --git a/pkg/adopt/milvus_config.go b/pkg/adopt/milvus_config.go index 5e7d41d79..e2ae279b0 100644 --- a/pkg/adopt/milvus_config.go +++ b/pkg/adopt/milvus_config.go @@ -185,6 +185,16 @@ func objectStorageData(cfg map[string]any) map[string]string { func carriedConfig(cfg map[string]any) (string, error) { out := deepCopy(cfg) delete(out, "etcd") + // The operator renders TLS for the certificates it mounts. The carried config + // wins a merge, so its paths would point at files the new pods do not have. + delete(out, "tls") + delete(out, "internaltls") + if common, ok := out["common"].(map[string]any); ok { + if sec, ok := common["security"].(map[string]any); ok { + delete(sec, "tlsMode") + delete(sec, "internaltlsEnabled") + } + } if v, ok := lookup(out, "proxy.http.port"); !ok || v == nil { proxy, _ := out["proxy"].(map[string]any) if proxy == nil { @@ -222,8 +232,12 @@ var criticalConfigPaths = []string{ "minio.useIAM", "minio.cloudProvider", "minio.region", "minio.accessKeyID", "minio.secretAccessKey", "mq.type", "woodpecker.storage.type", "woodpecker.storage.rootPath", "common.security.authorizationEnabled", "proxy.port", "proxy.http.port", + "common.security.tlsMode", "common.security.internaltlsEnabled", "etcd.ssl.enabled", } +// tlsSwitchPaths are off when the operator renders nothing for them. +var tlsSwitchPaths = []string{"common.security.tlsMode", "common.security.internaltlsEnabled", "etcd.ssl.enabled"} + // configMismatches compares the running config with the one the operator // rendered. A path the running config does not set is compared only for the // etcd root path, whose Milvus and KubeDB defaults differ. @@ -243,6 +257,9 @@ func configMismatches(running, rendered map[string]any) []string { } } a, b := scalarString(rv), lookupString(rendered, p) + if b == "" && (a == "false" || a == "0") && contains(tlsSwitchPaths, p) { + continue + } if a != b { if strings.Contains(strings.ToLower(p), "secret") || strings.HasSuffix(p, "accessKeyID") { out = append(out, p+": differs") diff --git a/pkg/adopt/milvus_file.go b/pkg/adopt/milvus_file.go index 43b5ac038..81693fb02 100644 --- a/pkg/adopt/milvus_file.go +++ b/pkg/adopt/milvus_file.go @@ -43,6 +43,7 @@ type adoptionFile struct { Version string `json:"version,omitempty"` Source adoptionSource `json:"source,omitempty"` Auth adoptionAuth `json:"auth,omitempty"` + TLS adoptionTLSFile `json:"tls,omitempty"` RestrictedSCC *bool `json:"restrictedSCC,omitempty"` Handover adoptionHandover `json:"handover,omitempty"` Rollback adoptionRollback `json:"rollback,omitempty"` @@ -69,6 +70,11 @@ type adoptionAuth struct { SkipCredentialCheck *bool `json:"skipCredentialCheck,omitempty"` } +type adoptionTLSFile struct { + Issuer string `json:"issuer,omitempty"` + ClientSecret string `json:"clientSecret,omitempty"` +} + type adoptionHandover struct { Method string `json:"method,omitempty"` QueryNodeDelay *metav1.Duration `json:"querynodeDelay,omitempty"` @@ -171,6 +177,8 @@ func (af *adoptionFile) flagValues() []flagValue { str("root-password-file", af.Auth.RootPasswordFile) str("etcd-auth-secret", af.Auth.EtcdAuthSecret) boolean("skip-credential-check", af.Auth.SkipCredentialCheck) + str("tls-issuer", af.TLS.Issuer) + str("tls-client-secret", af.TLS.ClientSecret) boolean("restricted-scc", af.RestrictedSCC) str("method", af.Handover.Method) duration("querynode-delay", af.Handover.QueryNodeDelay) @@ -262,6 +270,7 @@ func adoptionFileFor(p *adoptPlan, o planOptions, ns string) ([]byte, error) { RootPasswordFile: o.RootPasswordFile, EtcdAuthSecret: o.EtcdAuthSecret, }, + TLS: adoptionTLSFile{Issuer: o.TLSIssuer, ClientSecret: o.TLSClientSecret}, Handover: adoptionHandover{ Method: methodOverlap, QueryNodeDelay: &metav1.Duration{Duration: time.Minute}, diff --git a/pkg/adopt/milvus_handover.go b/pkg/adopt/milvus_handover.go index 41573c39f..9063902c0 100644 --- a/pkg/adopt/milvus_handover.go +++ b/pkg/adopt/milvus_handover.go @@ -448,6 +448,24 @@ func (c *cluster) cleanupKubeDBObjects(ctx context.Context, st *State) error { } c.logf("deleted Secret %s (rendered config)", s.Name) } + // Copies of certificates. The label keeps a Secret of the user's with the same + // name, which run reuses, out of this. + for _, name := range []string{serverTLSSecretName(st.Name), etcdTLSSecretName(st.Name)} { + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, name, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + continue + } + if err != nil { + return err + } + if s.Labels[LabelRelease] == "" { + continue + } + if err := c.kube.CoreV1().Secrets(c.ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted Secret %s (copy of a certificate)", name) + } for _, rec := range st.RemovedServices { svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{Name: rec.Name, Namespace: c.ns, Labels: rec.Labels, Annotations: rec.Annotations}, diff --git a/pkg/adopt/milvus_plan.go b/pkg/adopt/milvus_plan.go index b88febab0..aabbf4228 100644 --- a/pkg/adopt/milvus_plan.go +++ b/pkg/adopt/milvus_plan.go @@ -34,6 +34,8 @@ type planOptions struct { AuthSecret string RootPasswordFile string EtcdAuthSecret string + TLSIssuer string + TLSClientSecret string ConfigFile string RestrictedSCC bool SkipCredentialCheck bool @@ -132,9 +134,6 @@ func buildPlan(ctx context.Context, c *cluster, o planOptions) (*adoptPlan, erro } else { fs.infof("etcd %s, root path %q (kept)", strings.Join(etcd.Endpoints, ","), etcd.RootPath) } - if etcd.TLSEnabled { - fs.errorf("the etcd uses TLS; this command does not carry etcd client certificates yet, see the adoption guide") - } if etcd.AuthEnabled { if o.EtcdAuthSecret == "" { fs.errorf("the etcd uses authentication; give its root user in a basic-auth Secret with --etcd-auth-secret") @@ -174,6 +173,9 @@ func buildPlan(ctx context.Context, c *cluster, o planOptions) (*adoptPlan, erro Namespace: c.ns, Name: name, Version: version, Release: o.Release, Source: src, EtcdAuthSecret: o.EtcdAuthSecret, RestrictedSCC: o.RestrictedSCC, } + if err := c.planTLS(ctx, o, src, &in, fs); err != nil { + return nil, err + } if !lookupBool(cfg, "common.security.authorizationEnabled") { in.DisableAuth = true fs.infof("authentication is off; the Milvus gets spec.disableSecurity: true") diff --git a/pkg/adopt/milvus_render.go b/pkg/adopt/milvus_render.go index b005a2653..5ff7fcb02 100644 --- a/pkg/adopt/milvus_render.go +++ b/pkg/adopt/milvus_render.go @@ -48,6 +48,8 @@ const ( func stateConfigMapName(name string) string { return name + "-adopt" } func authSecretName(name string) string { return name + "-auth" } func objectStorageSecretName(name string) string { return name + "-object-storage" } +func serverTLSSecretName(name string) string { return name + "-tls-server" } +func etcdTLSSecretName(name string) string { return name + "-etcd-tls" } func configSecretName(name string) string { return name + "-adopt-config" } // petSetName is the KubeDB PetSet that replaces a Unit. @@ -87,6 +89,8 @@ type renderInput struct { RootPassword string DisableAuth bool EtcdAuthSecret string + EtcdTLS *tlsMaterial + TLS *adoptionTLS RestrictedSCC bool } @@ -141,6 +145,21 @@ func render(in renderInput, st *State) (*adoptObjects, error) { Data: map[string]string{stateKey: string(raw)}, } + // secretFor returns the Secret that holds m: the running one when KubeDB can + // read it as it is, otherwise a copy that rollback removes again. + secretFor := func(m *tlsMaterial, copyName string) string { + if m.Secret != "" { + return m.Secret + } + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: copyName, Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeOpaque, + StringData: m.secretData(), + }) + return copyName + } + etcd := etcdFromConfig(cfg) meta := map[string]any{ "externallyManaged": true, @@ -150,6 +169,9 @@ func render(in renderInput, st *State) (*adoptObjects, error) { if in.EtcdAuthSecret != "" { meta["authSecret"] = map[string]any{"name": in.EtcdAuthSecret} } + if in.EtcdTLS != nil { + meta["clientTLSSecret"] = map[string]any{"name": secretFor(in.EtcdTLS, etcdTLSSecretName(in.Name))} + } spec := map[string]any{ "version": in.Version, "deletionPolicy": "DoNotTerminate", @@ -171,6 +193,9 @@ func render(in renderInput, st *State) (*adoptObjects, error) { if in.RestrictedSCC { spec["restrictedSCC"] = true } + if in.TLS != nil { + spec["tls"] = tlsSpec(in.TLS, secretFor(in.TLS.Server, serverTLSSecretName(in.Name))) + } objs.Milvus = &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "kubedb.com/v1alpha2", "kind": "Milvus", diff --git a/pkg/adopt/milvus_tls.go b/pkg/adopt/milvus_tls.go new file mode 100644 index 000000000..ac04c7fe8 --- /dev/null +++ b/pkg/adopt/milvus_tls.go @@ -0,0 +1,504 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "path" + "strings" + "time" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// Secret keys the KubeDB operator reads a certificate from. +const ( + tlsCertKey = "tls.crt" + tlsKeyKey = "tls.key" + tlsCAKey = "ca.crt" + + certManagerGroup = "cert-manager.io" + milvusWorkDir = "/milvus" + certExpiryWarn = 30 * 24 * time.Hour +) + +var ( + certIssuerGVR = schema.GroupVersionResource{Group: certManagerGroup, Version: "v1", Resource: "issuers"} + certClusterIssuerGVR = schema.GroupVersionResource{Group: certManagerGroup, Version: "v1", Resource: "clusterissuers"} +) + +// tlsMaterial is a certificate as the running Milvus uses it. Secret is set when +// a Secret already has the keys KubeDB reads, so it can be referenced as it is +// and its owner's renewal keeps working. +type tlsMaterial struct { + Cert, Key, CA []byte + Secret string +} + +func (m *tlsMaterial) secretData() map[string]string { + return map[string]string{tlsCertKey: string(m.Cert), tlsKeyKey: string(m.Key), tlsCAKey: string(m.CA)} +} + +// adoptionTLS is the TLS the adopted Milvus keeps. +type adoptionTLS struct { + ExternalMode string // Disabled, TLS or mTLS + Internal bool + Server *tlsMaterial // nil when both client and internal TLS are off + ClientSecret string + IssuerKind string + IssuerName string +} + +// parseIssuerRef parses Kind/name; a bare name is an Issuer. +func parseIssuerRef(s string) (kind, name string, err error) { + kind, name = "Issuer", s + if k, n, ok := strings.Cut(s, "/"); ok { + kind, name = k, n + } + if name == "" || (kind != "Issuer" && kind != "ClusterIssuer") { + return "", "", fmt.Errorf("invalid issuer %q: want name, Issuer/name or ClusterIssuer/name", s) + } + return kind, name, nil +} + +// externalTLSMode reads common.security.tlsMode: 0 off, 1 TLS, 2 mTLS. +func externalTLSMode(cfg map[string]any) (string, error) { + switch v := lookupString(cfg, "common.security.tlsMode"); v { + case "", "0": + return "Disabled", nil + case "1": + return "TLS", nil + case "2": + return "mTLS", nil + default: + return "", fmt.Errorf("common.security.tlsMode is %q; Milvus knows 0, 1 and 2", v) + } +} + +// fileSource is one place a mounted file may come from. +type fileSource struct { + Kind string // Secret or ConfigMap + Name string + Key string +} + +// mountedFile lists where a file in the Milvus container comes from. Several +// sources are possible only for a projected volume without items. +func mountedFile(t core.PodTemplateSpec, file string) ([]fileSource, error) { + if !path.IsAbs(file) { + file = path.Join(milvusWorkDir, file) + } + c := milvusContainer(t) + if c == nil { + return nil, errors.New("no Milvus container") + } + var mount *core.VolumeMount + for i := range c.VolumeMounts { + m := &c.VolumeMounts[i] + mp := strings.TrimSuffix(m.MountPath, "/") + if file != mp && !strings.HasPrefix(file, mp+"/") { + continue + } + if mount == nil || len(mp) > len(strings.TrimSuffix(mount.MountPath, "/")) { + mount = m + } + } + if mount == nil { + return nil, fmt.Errorf("%s is not on a mounted volume", file) + } + rel := strings.TrimPrefix(strings.TrimPrefix(file, strings.TrimSuffix(mount.MountPath, "/")), "/") + rel = path.Join(mount.SubPath, rel) + if rel == "" || rel == "." { + return nil, fmt.Errorf("%s is a directory", file) + } + for _, v := range t.Spec.Volumes { + if v.Name == mount.Name { + return volumeFile(v, rel, file) + } + } + return nil, fmt.Errorf("volume %s of %s does not exist", mount.Name, file) +} + +func itemKey(items []core.KeyToPath, rel string) (string, bool) { + if len(items) == 0 { + return rel, true + } + for _, it := range items { + if it.Path == rel { + return it.Key, true + } + } + return "", false +} + +func volumeFile(v core.Volume, rel, file string) ([]fileSource, error) { + var out []fileSource + add := func(kind, name string, items []core.KeyToPath) { + if key, ok := itemKey(items, rel); ok { + out = append(out, fileSource{Kind: kind, Name: name, Key: key}) + } + } + switch { + case v.Secret != nil: + add("Secret", v.Secret.SecretName, v.Secret.Items) + case v.ConfigMap != nil: + add("ConfigMap", v.ConfigMap.Name, v.ConfigMap.Items) + case v.Projected != nil: + for _, s := range v.Projected.Sources { + if s.Secret != nil { + add("Secret", s.Secret.Name, s.Secret.Items) + } + if s.ConfigMap != nil { + add("ConfigMap", s.ConfigMap.Name, s.ConfigMap.Items) + } + } + default: + return nil, fmt.Errorf("%s does not come from a Secret or ConfigMap (volume %s), so it cannot be carried over", file, v.Name) + } + if len(out) == 0 { + return nil, fmt.Errorf("volume %s does not provide %s", v.Name, file) + } + return out, nil +} + +func (c *cluster) readSource(ctx context.Context, src fileSource) ([]byte, bool, error) { + if src.Kind == "ConfigMap" { + cm, err := c.kube.CoreV1().ConfigMaps(c.ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return nil, false, err + } + v, ok := cm.Data[src.Key] + return []byte(v), ok, nil + } + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return nil, false, err + } + v, ok := s.Data[src.Key] + return v, ok, nil +} + +func (c *cluster) readMounted(ctx context.Context, t core.PodTemplateSpec, file string) ([]byte, fileSource, error) { + sources, err := mountedFile(t, file) + if err != nil { + return nil, fileSource{}, err + } + for _, src := range sources { + data, ok, err := c.readSource(ctx, src) + if err != nil { + return nil, src, fmt.Errorf("failed to read %s %s: %w", src.Kind, src.Name, err) + } + if ok { + return data, src, nil + } + } + return nil, fileSource{}, fmt.Errorf("no mounted Secret or ConfigMap has %s", file) +} + +// readTLS reads the certificate, key and CA at the three config paths. +func (c *cluster) readTLS(ctx context.Context, t core.PodTemplateSpec, cfg map[string]any, certPath, keyPath, caPath string) (*tlsMaterial, error) { + m := &tlsMaterial{} + var srcs [3]fileSource + for i, p := range []string{certPath, keyPath, caPath} { + file := lookupString(cfg, p) + if file == "" { + return nil, fmt.Errorf("the config does not set %s", p) + } + data, src, err := c.readMounted(ctx, t, file) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + srcs[i] = src + switch i { + case 0: + m.Cert = data + case 1: + m.Key = data + case 2: + m.CA = data + } + } + name := srcs[0].Name + if srcs == [3]fileSource{{"Secret", name, tlsCertKey}, {"Secret", name, tlsKeyKey}, {"Secret", name, tlsCAKey}} { + m.Secret = name + } + return m, nil +} + +func parseLeaf(certPEM []byte) (*x509.Certificate, *x509.CertPool, error) { + var leaf *x509.Certificate + intermediates := x509.NewCertPool() + for rest := certPEM; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, err + } + if leaf == nil { + leaf = cert + } else { + intermediates.AddCert(cert) + } + } + if leaf == nil { + return nil, nil, errors.New("no certificate in the PEM data") + } + return leaf, intermediates, nil +} + +func verifyAgainstCA(certPEM, caPEM []byte, now time.Time) error { + leaf, intermediates, err := parseLeaf(certPEM) + if err != nil { + return err + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return errors.New("no certificate in the CA data") + } + _, err = leaf.Verify(x509.VerifyOptions{ + Roots: roots, Intermediates: intermediates, CurrentTime: now, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + }) + return err +} + +// checkCertificate reports a certificate that cannot serve this cluster. +func checkCertificate(label string, m *tlsMaterial, hosts []string, chain bool, now time.Time, fs *findings) { + if _, err := tls.X509KeyPair(m.Cert, m.Key); err != nil { + fs.errorf("%s: the certificate and key do not form a pair: %v", label, err) + return + } + leaf, _, err := parseLeaf(m.Cert) + if err != nil { + fs.errorf("%s: %v", label, err) + return + } + if chain { + if err := verifyAgainstCA(m.Cert, m.CA, now); err != nil { + fs.errorf("%s: the certificate does not verify against its CA: %v", label, err) + } + } + for _, h := range hosts { + if err := leaf.VerifyHostname(h); err != nil { + fs.errorf("%s: the certificate does not cover %s, which KubeDB connects to (it covers %s); reissue it with that name", label, h, strings.Join(leaf.DNSNames, ", ")) + } + } + if left := leaf.NotAfter.Sub(now); left > 0 && left < certExpiryWarn { + fs.warnf("%s: the certificate expires on %s", label, leaf.NotAfter.Format("2006-01-02")) + } +} + +// clientHosts are the names KubeDB and Milvus use to reach the proxies. +func clientHosts(name, ns string) []string { + return []string{name, name + "." + ns + ".svc", name + "." + ns + ".svc.cluster.local"} +} + +// planEtcdTLS carries the client certificate of an etcd that uses TLS. +func (c *cluster) planEtcdTLS(ctx context.Context, t core.PodTemplateSpec, cfg map[string]any, in *renderInput, fs *findings) { + if !etcdFromConfig(cfg).TLSEnabled { + return + } + m, err := c.readTLS(ctx, t, cfg, "etcd.ssl.tlsCert", "etcd.ssl.tlsKey", "etcd.ssl.tlsCACert") + if err != nil { + fs.errorf("etcd TLS: %v", err) + return + } + checkCertificate("etcd TLS", m, nil, false, time.Now(), fs) + in.EtcdTLS = m + fs.infof("etcd TLS: %s", materialNote(m)) +} + +func materialNote(m *tlsMaterial) string { + if m.Secret != "" { + return "Secret " + m.Secret + " is used as it is" + } + return "the certificate is copied to a new Secret, because its keys are not tls.crt, tls.key and ca.crt; renewing the copy is up to you" +} + +// planTLS keeps the TLS the running Milvus uses. What exists is reused, and +// KubeDB issues only a certificate that is missing. TLS that is off stays off: +// switching it on during a handover would split the old pods from the new ones. +func (c *cluster) planTLS(ctx context.Context, o planOptions, src *Source, in *renderInput, fs *findings) error { + cfg := src.Config + proxy := src.Templates[src.firstUnit(roleProxy).key()] + c.planEtcdTLS(ctx, proxy, cfg, in, fs) + mode, err := externalTLSMode(cfg) + if err != nil { + fs.errorf("%v", err) + return nil + } + internal := lookupBool(cfg, "common.security.internaltlsEnabled") + if mode == "Disabled" && !internal { + fs.infof("client and internal TLS are off and stay off") + return nil + } + out := &adoptionTLS{ExternalMode: mode, Internal: internal} + + var external, inner *tlsMaterial + if mode != "Disabled" { + if external, err = c.readTLS(ctx, proxy, cfg, "tls.serverPemPath", "tls.serverKeyPath", "tls.caPemPath"); err != nil { + fs.errorf("client TLS: %v", err) + return nil + } + out.Server = external + } + if internal { + if inner, err = c.readTLS(ctx, proxy, cfg, "internaltls.serverPemPath", "internaltls.serverKeyPath", "internaltls.caPemPath"); err != nil { + fs.errorf("internal TLS: %v", err) + return nil + } + out.Server = inner + } + if external != nil && inner != nil { + if !bytes.Equal(external.Cert, inner.Cert) { + fs.errorf("client and internal TLS use different certificates; KubeDB mounts one for both") + return nil + } + out.Server = external + } + checkCertificate("server certificate", out.Server, clientHosts(in.Name, c.ns), true, time.Now(), fs) + fs.infof("TLS: client %s, internal %v; %s", mode, internal, materialNote(out.Server)) + + if mode == "mTLS" { + if err := c.planClientCert(ctx, o, out, fs); err != nil { + return err + } + } else if o.TLSIssuer != "" || o.TLSClientSecret != "" { + fs.infof("no client certificate is needed without mTLS; --tls-issuer and --tls-client-secret are not used") + } + in.TLS = out + c.setRESTTLS(out, in.Name) + return nil +} + +// planClientCert finds the certificate KubeDB presents when Milvus asks clients +// for one: the user's, or else one from the issuer. +func (c *cluster) planClientCert(ctx context.Context, o planOptions, out *adoptionTLS, fs *findings) error { + switch { + case o.TLSClientSecret != "": + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, o.TLSClientSecret, metav1.GetOptions{}) + if err != nil { + fs.errorf("--tls-client-secret: %v", err) + return nil + } + for _, k := range []string{tlsCertKey, tlsKeyKey, tlsCAKey} { + if _, ok := s.Data[k]; !ok { + fs.errorf("--tls-client-secret: Secret %s has no key %s", s.Name, k) + return nil + } + } + checkCertificate("client certificate", &tlsMaterial{Cert: s.Data[tlsCertKey], Key: s.Data[tlsKeyKey], CA: out.Server.CA}, nil, true, time.Now(), fs) + if err := verifyAgainstCA(out.Server.Cert, s.Data[tlsCAKey], time.Now()); err != nil { + fs.errorf("--tls-client-secret: its ca.crt does not verify the server certificate: %v", err) + } + out.ClientSecret = s.Name + fs.infof("client certificate: Secret %s is used as it is", s.Name) + case o.TLSIssuer != "": + kind, name, err := parseIssuerRef(o.TLSIssuer) + if err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + out.IssuerKind, out.IssuerName = kind, name + return c.checkIssuer(ctx, kind, name, out.Server.CA, fs) + default: + fs.errorf("client mTLS is on, so KubeDB needs a client certificate that Milvus trusts; give it with --tls-client-secret, or give --tls-issuer to have one issued") + } + return nil +} + +// checkIssuer stops an issuer whose certificates Milvus would reject: in mTLS, +// Milvus trusts only the CA it runs with. +func (c *cluster) checkIssuer(ctx context.Context, kind, name string, serverCA []byte, fs *findings) error { + if kind == "ClusterIssuer" { + if _, err := c.dyn.Resource(certClusterIssuerGVR).Get(ctx, name, metav1.GetOptions{}); err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + fs.warnf("cannot confirm that ClusterIssuer %s signs with the CA Milvus trusts; client certificates from another CA are rejected", name) + return nil + } + obj, err := c.dyn.Resource(certIssuerGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + fs.errorf("--tls-issuer: Issuer %s does not exist", name) + return nil + } + if err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + spec, _ := obj.Object["spec"].(map[string]any) + ca, _ := spec["ca"].(map[string]any) + secret, _ := ca["secretName"].(string) + if secret == "" { + fs.warnf("cannot confirm that Issuer %s signs with the CA Milvus trusts; client certificates from another CA are rejected", name) + return nil + } + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, secret, metav1.GetOptions{}) + if err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + if !bytes.Equal(bytes.TrimSpace(s.Data[tlsCertKey]), bytes.TrimSpace(serverCA)) { + fs.errorf("Issuer %s signs with a different CA than the one Milvus trusts, so Milvus would reject its client certificates", name) + } + return nil +} + +// setRESTTLS makes the credential check speak TLS to the running proxies. +func (c *cluster) setRESTTLS(t *adoptionTLS, name string) { + if t.ExternalMode == "Disabled" { + return + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(t.Server.CA) + c.restTLS = &tls.Config{RootCAs: roots, ServerName: name, MinVersion: tls.VersionTLS12} +} + +// tlsSpec is spec.tls of the adopted Milvus. +func tlsSpec(t *adoptionTLS, serverSecret string) map[string]any { + internal := "Disabled" + if t.Internal { + internal = "TLS" + } + existing := map[string]any{"server": map[string]any{"name": serverSecret}} + if t.ClientSecret != "" { + existing["client"] = map[string]any{"name": t.ClientSecret} + } + spec := map[string]any{ + "external": map[string]any{"mode": t.ExternalMode}, + "internal": map[string]any{"mode": internal}, + "existingSecrets": existing, + } + if t.IssuerName != "" { + spec["issuerRef"] = map[string]any{"apiGroup": certManagerGroup, "kind": t.IssuerKind, "name": t.IssuerName} + } + return spec +} diff --git a/pkg/adopt/milvus_tls_test.go b/pkg/adopt/milvus_tls_test.go new file mode 100644 index 000000000..b710741c3 --- /dev/null +++ b/pkg/adopt/milvus_tls_test.go @@ -0,0 +1,245 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "strings" + "testing" + "time" + + core "k8s.io/api/core/v1" +) + +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + pem []byte +} + +func newTestCA(t *testing.T) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * 365 * time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + cert, _ := x509.ParseCertificate(der) + return &testCA{cert: cert, key: key, pem: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})} +} + +func (ca *testCA) issue(t *testing.T, ttl time.Duration, dns ...string) *tlsMaterial { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "milvus"}, DNSNames: dns, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(ttl), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + return &tlsMaterial{ + Cert: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + Key: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + CA: ca.pem, + } +} + +func TestCheckCertificate(t *testing.T) { + ca := newTestCA(t) + hosts := clientHosts("mv", "demo") + year := 365 * 24 * time.Hour + now := time.Now() + + good := ca.issue(t, year, hosts...) + fs := &findings{} + checkCertificate("server", good, hosts, true, now, fs) + if fs.hasErrors() || len(*fs) != 0 { + t.Fatalf("a good certificate has findings: %s", findingText(*fs)) + } + + fs = &findings{} + checkCertificate("server", ca.issue(t, year, "other.example.com"), hosts, true, now, fs) + if !fs.hasErrors() || !strings.Contains(findingText(*fs), "does not cover mv") { + t.Fatalf("a wrong name must stop the plan: %s", findingText(*fs)) + } + + fs = &findings{} + foreign := ca.issue(t, year, hosts...) + foreign.CA = newTestCA(t).pem + checkCertificate("server", foreign, hosts, true, now, fs) + if !fs.hasErrors() { + t.Fatal("a certificate from another CA must stop the plan") + } + + fs = &findings{} + checkCertificate("server", ca.issue(t, 5*24*time.Hour, hosts...), hosts, true, now, fs) + if fs.hasErrors() || !strings.Contains(findingText(*fs), "expires") { + t.Fatalf("a certificate near expiry is a warning: %s", findingText(*fs)) + } + + fs = &findings{} + mismatched := ca.issue(t, year, hosts...) + mismatched.Key = ca.issue(t, year, hosts...).Key + checkCertificate("server", mismatched, hosts, true, now, fs) + if !fs.hasErrors() { + t.Fatal("a key of another certificate must stop the plan") + } +} + +func TestExternalTLSMode(t *testing.T) { + for v, want := range map[any]string{nil: "Disabled", 0: "Disabled", 1: "TLS", float64(2): "mTLS"} { + cfg := map[string]any{} + if v != nil { + cfg = map[string]any{"common": map[string]any{"security": map[string]any{"tlsMode": v}}} + } + if got, err := externalTLSMode(cfg); err != nil || got != want { + t.Fatalf("tlsMode %v: got %q %v, want %q", v, got, err, want) + } + } + if _, err := externalTLSMode(map[string]any{"common": map[string]any{"security": map[string]any{"tlsMode": 3}}}); err == nil { + t.Fatal("an unknown mode must fail") + } +} + +func TestMountedFile(t *testing.T) { + tmpl := core.PodTemplateSpec{Spec: core.PodSpec{ + Containers: []core.Container{{Name: "milvus", Image: "milvusdb/milvus", VolumeMounts: []core.VolumeMount{ + {Name: "certs", MountPath: "/certs"}, + {Name: "items", MountPath: "/items"}, + {Name: "proj", MountPath: "/proj"}, + {Name: "sub", MountPath: "/single/server.pem", SubPath: "tls.crt"}, + {Name: "disk", MountPath: "/disk"}, + {Name: "certs", MountPath: "/milvus/configs/cert"}, + }}}, + Volumes: []core.Volume{ + {Name: "certs", VolumeSource: core.VolumeSource{Secret: &core.SecretVolumeSource{SecretName: "s1"}}}, + {Name: "items", VolumeSource: core.VolumeSource{Secret: &core.SecretVolumeSource{ + SecretName: "s2", Items: []core.KeyToPath{{Key: "tls.crt", Path: "server.pem"}}, + }}}, + {Name: "proj", VolumeSource: core.VolumeSource{Projected: &core.ProjectedVolumeSource{Sources: []core.VolumeProjection{ + {Secret: &core.SecretProjection{LocalObjectReference: core.LocalObjectReference{Name: "s3"}, Items: []core.KeyToPath{{Key: "tls.key", Path: "server.key"}}}}, + {ConfigMap: &core.ConfigMapProjection{LocalObjectReference: core.LocalObjectReference{Name: "cm"}, Items: []core.KeyToPath{{Key: "ca", Path: "ca.pem"}}}}, + }}}}, + {Name: "sub", VolumeSource: core.VolumeSource{Secret: &core.SecretVolumeSource{SecretName: "s4"}}}, + {Name: "disk", VolumeSource: core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{}}}, + }, + }} + for file, want := range map[string]fileSource{ + "/certs/server.pem": {"Secret", "s1", "server.pem"}, + "/items/server.pem": {"Secret", "s2", "tls.crt"}, + "/proj/server.key": {"Secret", "s3", "tls.key"}, + "/proj/ca.pem": {"ConfigMap", "cm", "ca"}, + "/single/server.pem": {"Secret", "s4", "tls.crt"}, + "configs/cert/ca.pem": {"Secret", "s1", "ca.pem"}, // relative to the Milvus work dir + } { + got, err := mountedFile(tmpl, file) + if err != nil || len(got) != 1 || got[0] != want { + t.Fatalf("%s: got %+v %v, want %+v", file, got, err, want) + } + } + for _, file := range []string{"/nowhere/x.pem", "/disk/x.pem", "/items/other.pem"} { + if _, err := mountedFile(tmpl, file); err == nil { + t.Fatalf("%s must fail", file) + } + } +} + +func TestParseIssuerRef(t *testing.T) { + for in, want := range map[string][2]string{ + "ca": {"Issuer", "ca"}, "Issuer/ca": {"Issuer", "ca"}, "ClusterIssuer/ca": {"ClusterIssuer", "ca"}, + } { + if k, n, err := parseIssuerRef(in); err != nil || k != want[0] || n != want[1] { + t.Fatalf("%s: got %s %s %v", in, k, n, err) + } + } + for _, in := range []string{"Certificate/ca", "Issuer/", ""} { + if _, _, err := parseIssuerRef(in); err == nil { + t.Fatalf("%q must fail", in) + } + } +} + +func TestCarriedConfigDropsTLS(t *testing.T) { + raw, err := carriedConfig(map[string]any{ + "tls": map[string]any{"serverPemPath": "/old/server.pem"}, + "internaltls": map[string]any{"serverPemPath": "/old/server.pem"}, + "common": map[string]any{"security": map[string]any{ + "tlsMode": 1, "internaltlsEnabled": true, "authorizationEnabled": true, + }}, + }) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{"/old/", "tlsMode", "internaltlsEnabled"} { + if strings.Contains(raw, s) { + t.Fatalf("%s must not be carried, the operator renders it: %s", s, raw) + } + } + if !strings.Contains(raw, "authorizationEnabled") { + t.Fatalf("other security settings stay: %s", raw) + } +} + +func TestConfigMismatchesTLS(t *testing.T) { + sec := func(v map[string]any) map[string]any { + return map[string]any{"common": map[string]any{"security": v}, "etcd": map[string]any{"rootPath": "by-dev"}} + } + if d := configMismatches(sec(map[string]any{"tlsMode": 0, "internaltlsEnabled": false}), sec(map[string]any{})); len(d) != 0 { + t.Fatalf("off and not rendered are the same: %v", d) + } + if d := configMismatches(sec(map[string]any{"tlsMode": 1}), sec(map[string]any{})); len(d) != 1 { + t.Fatalf("TLS on and not rendered must be reported: %v", d) + } +} + +func TestTLSSpec(t *testing.T) { + got := tlsSpec(&adoptionTLS{ExternalMode: "mTLS", Internal: true, IssuerKind: "Issuer", IssuerName: "ca"}, "theirs") + if got["external"].(map[string]any)["mode"] != "mTLS" || got["internal"].(map[string]any)["mode"] != "TLS" { + t.Fatalf("modes: %v", got) + } + if got["existingSecrets"].(map[string]any)["server"].(map[string]any)["name"] != "theirs" { + t.Fatalf("server Secret: %v", got) + } + if got["issuerRef"].(map[string]any)["apiGroup"] != certManagerGroup { + t.Fatalf("issuer: %v", got) + } + got = tlsSpec(&adoptionTLS{ExternalMode: "TLS"}, "theirs") + if _, ok := got["issuerRef"]; ok { + t.Fatalf("nothing to issue, no issuerRef: %v", got) + } +}