diff --git a/pkg/adopt/adopt.go b/pkg/adopt/adopt.go new file mode 100644 index 000000000..d5c04026b --- /dev/null +++ b/pkg/adopt/adopt.go @@ -0,0 +1,43 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package adopt implements `kubectl dba adopt`, which brings a database that +// already runs in Kubernetes (for example from a Helm chart) under KubeDB in +// place: the data stays where it is and the workloads change hands one role at +// a time. +package adopt + +import ( + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdAdopt returns the `adopt` command group. +func NewCmdAdopt(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "adopt", + Short: i18n.T("Bring a running database under KubeDB in place, without moving its data"), + Long: templates.LongDesc(` + Take over a database that already runs in Kubernetes, for example one + installed by a Helm chart. The data stays in its current storage, the + credentials stay the same, and clients keep their address; only the + workloads change hands, one role at a time.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand(NewCmdMilvus(f)) + return cmd +} diff --git a/pkg/adopt/findings.go b/pkg/adopt/findings.go new file mode 100644 index 000000000..aedfeb79b --- /dev/null +++ b/pkg/adopt/findings.go @@ -0,0 +1,61 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "fmt" + "io" +) + +const ( + levelError = "ERROR" + levelWarn = "WARN" + levelInfo = "OK" +) + +type finding struct { + Level string + Msg string +} + +// findings are the results of the pre-flight checks. An ERROR blocks the +// adoption; a WARN needs a look before the handover. +type findings []finding + +func (f *findings) errorf(format string, a ...any) { + *f = append(*f, finding{levelError, fmt.Sprintf(format, a...)}) +} + +func (f *findings) warnf(format string, a ...any) { + *f = append(*f, finding{levelWarn, fmt.Sprintf(format, a...)}) +} + +func (f *findings) infof(format string, a ...any) { + *f = append(*f, finding{levelInfo, fmt.Sprintf(format, a...)}) +} + +func (f findings) hasErrors() bool { + for _, x := range f { + if x.Level == levelError { + return true + } + } + return false +} + +func (f findings) print(w io.Writer) { + for _, x := range f { + _, _ = fmt.Fprintf(w, " [%-5s] %s\n", x.Level, x.Msg) + } +} diff --git a/pkg/adopt/milvus.go b/pkg/adopt/milvus.go new file mode 100644 index 000000000..331cc7a55 --- /dev/null +++ b/pkg/adopt/milvus.go @@ -0,0 +1,572 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdMilvus returns `adopt milvus`. +func NewCmdMilvus(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "milvus", + Short: i18n.T("Adopt a running Distributed Milvus 2.6 in place"), + Long: templates.LongDesc(` + Adopt a Distributed Milvus 2.6 that runs from a Helm chart (or any set of + Deployments and StatefulSets) in place. The metadata stays in its etcd, the + segments, indexes and write-ahead log stay in its bucket, the root + credentials and the client Service stay the same. + + * plan: check the cluster and show what an adoption creates (read-only) + * run: create the Milvus and hand the roles over, one at a time + * status: show where each role runs + * rollback: give the cluster back to the chart's workloads + + RELEASE names the running cluster: by default its workloads are those with + app.kubernetes.io/instance=RELEASE, and the "component" label names each + workload's role, as in the upstream Milvus Helm chart.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand(newCmdMilvusPlan(f), newCmdMilvusRun(f), newCmdMilvusStatus(f), newCmdMilvusRollback(f)) + return cmd +} + +func addSourceFlags(cmd *cobra.Command, o *sourceOptions) { + cmd.Flags().StringVar(&o.Selector, "selector", "", "Label selector of the cluster's workloads (default app.kubernetes.io/instance=RELEASE)") + cmd.Flags().StringVar(&o.RoleLabel, "role-label", "component", "Pod label that names a workload's Milvus role") + cmd.Flags().StringArrayVar(&o.Workloads, "workload", nil, "Map a workload to a role, as role[/group]=Kind/name (repeat for each workload; replaces --selector)") +} + +func addPlanFlags(cmd *cobra.Command, o *planOptions) { + addSourceFlags(cmd, &o.sourceOptions) + cmd.Flags().StringVar(&o.Name, "name", "", "Name of the Milvus object (default: the name of the Service clients use, which KubeDB then takes over)") + cmd.Flags().StringVar(&o.Version, "version", "", "MilvusVersion to use (default: the one matching the running image)") + cmd.Flags().StringVar(&o.AuthSecret, "auth-secret", "", "Existing basic-auth Secret with the current root user and password") + cmd.Flags().StringVar(&o.RootPasswordFile, "root-password-file", "", "File holding the current root password (default: common.security.defaultRootPassword of the running config)") + cmd.Flags().StringVar(&o.EtcdAuthSecret, "etcd-auth-secret", "", "Basic-auth Secret for an etcd that uses authentication") + cmd.Flags().StringVar(&o.TLSIssuer, "tls-issuer", "", "cert-manager issuer for a certificate the cluster does not have, as name, Issuer/name or ClusterIssuer/name") + cmd.Flags().StringVar(&o.TLSClientSecret, "tls-client-secret", "", "Existing Secret (tls.crt, tls.key, ca.crt) with the client certificate KubeDB presents when client mTLS is on") + cmd.Flags().StringVar(&o.ConfigFile, "config-file", "", "The effective milvus.yaml, when it is not mounted from a ConfigMap or Secret") + cmd.Flags().BoolVar(&o.RestrictedSCC, "restricted-scc", false, "Set spec.restrictedSCC (OpenShift)") + cmd.Flags().BoolVar(&o.SkipCredentialCheck, "skip-credential-check", false, "Do not log in to the running cluster to check the root credentials") +} + +func addTimingFlags(cmd *cobra.Command, t *timing) { + cmd.Flags().DurationVar(&t.Timeout, "timeout", 15*time.Minute, "Longest wait for one step (pods Ready or gone)") + cmd.Flags().DurationVar(&t.Settle, "settle", 20*time.Second, "Pause after each role") + cmd.Flags().DurationVar(&t.QueryNodeDelay, "querynode-delay", time.Minute, "Overlap: time a new QueryNode serves before an old one stops (raise it for large collections)") +} + +func newCmdMilvusPlan(f cmdutil.Factory) *cobra.Command { + var o planOptions + var fo fileOptions + var outDir string + cmd := &cobra.Command{ + Use: "plan [RELEASE]", + Short: i18n.T("Check a running Milvus and show what adopting it creates"), + Long: templates.LongDesc(` + Read-only. Finds the workloads, effective configuration, image and client + Service of the running cluster, checks everything an in-place adoption + depends on (Woodpecker WAL, etcd root path, object storage, a matching + MilvusVersion, the current root password), and shows the Secrets, the + state ConfigMap and the staged Milvus that "run" would create. + + With --output-dir the objects are written there, unredacted, together with + adopt.yaml: every option of this adoption as plan found it. Review it and + pass it to run, status and rollback with -f instead of repeating the flags. + Edit the objects if needed and apply them; "run" then uses what exists.`), + Example: templates.Examples(` + kubectl dba adopt milvus plan my-release -n milvus + kubectl dba adopt milvus plan my-release -n milvus --output-dir ./adopt + kubectl dba adopt milvus plan -f ./adopt/adopt.yaml`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + var err error + if o.Release, err = fo.resolve(cmd, args); err != nil { + return err + } + ctx := context.Background() + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) + if err != nil { + return err + } + p, err := buildPlan(ctx, c, o) + if err != nil { + return err + } + out := cmd.OutOrStdout() + _, _ = fmt.Fprintf(out, "Pre-flight checks for %s in namespace %s:\n", o.Release, c.ns) + p.Findings.print(out) + if p.Findings.hasErrors() { + return errors.New("the adoption is blocked; fix the ERROR findings") + } + docs, err := p.yamlDocs(outDir == "") + if err != nil { + return err + } + if outDir != "" { + if err := os.MkdirAll(outDir, 0o700); err != nil { + return err + } + af, err := adoptionFileFor(p, o, c.ns) + if err != nil { + return err + } + docs[adoptionFileName] = string(af) + for _, name := range sortedKeys(docs) { + if err := os.WriteFile(filepath.Join(outDir, name), []byte(docs[name]), 0o600); err != nil { + return err + } + } + _, _ = fmt.Fprintf(out, "\nWrote %s to %s\n", strings.Join(sortedKeys(docs), ", "), outDir) + } else { + for _, name := range sortedKeys(docs) { + _, _ = fmt.Fprintf(out, "\n# %s\n%s", name, docs[name]) + } + } + printHandoverPlan(out, p.State, methodOverlap) + if outDir != "" { + _, _ = fmt.Fprintf(out, "\nNext: kubectl dba adopt milvus run -f %s --yes\n", filepath.Join(outDir, adoptionFileName)) + } else { + _, _ = fmt.Fprintf(out, "\nNext: kubectl dba adopt milvus run %s -n %s --yes\n", o.Release, c.ns) + } + return nil + }, + } + addPlanFlags(cmd, &o) + addFileFlag(cmd, &fo) + cmd.Flags().StringVar(&outDir, "output-dir", "", "Write the objects to this directory (Secrets unredacted, mode 0600)") + return cmd +} + +func printHandoverPlan(out io.Writer, st *State, method string) { + _, _ = fmt.Fprintf(out, "\nHandover (%s), in this order:\n", method) + for _, role := range handoverOrder(method) { + for _, u := range st.unitsOf(role) { + if st.isDone(u) { + continue + } + how := "start KubeDB's, wait Ready, then stop the chart's" + if method == methodStopStart { + how = "stop the chart's, then start KubeDB's" + } else if u.Role == roleQueryNode && u.Replicas > 1 { + how = "one QueryNode at a time: start one, let it serve, stop one" + } + _, _ = fmt.Fprintf(out, " %-24s %d replicas: %s\n", u.key(), u.Replicas, how) + } + } +} + +type runOptions struct { + planOptions + timing + Method string + Yes bool + SkipConfigCheck bool + ReadyTimeout time.Duration +} + +func newCmdMilvusRun(f cmdutil.Factory) *cobra.Command { + var o runOptions + var fo fileOptions + cmd := &cobra.Command{ + Use: "run [RELEASE]", + Short: i18n.T("Adopt a running Milvus: create the Milvus object and hand the roles over"), + Long: templates.LongDesc(` + Runs the checks of "plan", creates the Secrets, the state ConfigMap + NAME-adopt (NAME is the Milvus) and the Milvus with every role at 0 replicas, and waits for + the operator. Before any pod moves, it compares the config the operator + rendered with the running one on every setting that decides where the data + is read from, and stops on a difference. + + Then it hands the roles over one at a time: + + * --method overlap (default): start KubeDB's pods of a role, wait until they + are Ready, then stop the chart's. QueryNodes move one at a time. No role + stops serving while it moves; it needs room for one extra set of pods of + a role (one extra QueryNode). + * --method stop-start: stop the chart's pods of a role, then start KubeDB's. + Each role is briefly unavailable; no extra capacity is needed. + + Progress is recorded in the state ConfigMap: if the command stops, run it + again and it continues where it was. "rollback" gives the cluster back. + + Do not remove the chart with "helm uninstall" afterwards: it would delete + the client Service that KubeDB now uses.`), + Example: templates.Examples(` + kubectl dba adopt milvus run my-release -n milvus --yes + kubectl dba adopt milvus run my-release -n milvus --method stop-start --yes + kubectl dba adopt milvus run -f ./adopt/adopt.yaml --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + var err error + if o.Release, err = fo.resolve(cmd, args); err != nil { + return err + } + if o.Method != methodOverlap && o.Method != methodStopStart { + return fmt.Errorf("--method must be %s or %s", methodOverlap, methodStopStart) + } + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) + if err != nil { + return err + } + return runAdoption(context.Background(), c, o) + }, + } + addPlanFlags(cmd, &o.planOptions) + addTimingFlags(cmd, &o.timing) + addFileFlag(cmd, &fo) + cmd.Flags().StringVar(&o.Method, "method", methodOverlap, "Handover method: overlap or stop-start") + cmd.Flags().BoolVar(&o.Yes, "yes", false, "Proceed; without it, run only shows what it would do") + cmd.Flags().BoolVar(&o.SkipConfigCheck, "skip-config-check", false, "Do not stop when the rendered config differs from the running one") + cmd.Flags().DurationVar(&o.ReadyTimeout, "ready-timeout", 15*time.Minute, "Longest wait for the operator, and for the Milvus to be Ready at the end") + return cmd +} + +func runAdoption(ctx context.Context, c *cluster, o runOptions) error { + out := c.out + st, _, err := c.loadState(ctx, o.Release) + if err != nil { + return err + } + var p *adoptPlan + if st == nil || st.Phase == phasePlanned { + o.allowExisting = st != nil + if p, err = buildPlan(ctx, c, o.planOptions); err != nil { + return err + } + _, _ = fmt.Fprintf(out, "Pre-flight checks for %s in namespace %s:\n", o.Release, c.ns) + p.Findings.print(out) + if p.Findings.hasErrors() { + return errors.New("the adoption is blocked; fix the ERROR findings") + } + if st == nil { + st = p.State + } + } else { + _, _ = fmt.Fprintf(out, "Resuming the adoption of %s as Milvus %s (%s, %s); handed over: %s\n", + o.Release, st.Name, st.Phase, st.Method, orNone(strings.Join(st.Done, ", "))) + if st.Method != "" && st.Method != o.Method { + c.logf("continuing with --method %s (started with %s)", o.Method, st.Method) + } + } + if st.Phase == phaseAdopted { + _, _ = fmt.Fprintf(out, "Milvus %s is already adopted.\n", st.Name) + return nil + } + if !o.Yes { + printHandoverPlan(out, st, o.Method) + return errors.New("re-run with --yes to proceed") + } + + if p != nil { + if err := c.createObjects(ctx, p); err != nil { + return err + } + if err := c.removeGoverningServices(ctx, st, p.GoverningServices); err != nil { + return err + } + if _, err := c.createMilvus(ctx, p, o.ReadyTimeout); err != nil { + return err + } + if !o.SkipConfigCheck { + diffs, err := c.checkRenderedConfig(ctx, st.Name, p.Source.Config) + if err != nil { + return err + } + if len(diffs) > 0 { + _, _ = fmt.Fprintf(out, "The config the operator rendered for Milvus %s differs from the running one:\n", st.Name) + for _, d := range diffs { + _, _ = fmt.Fprintf(out, " %s\n", d) + } + return fmt.Errorf("no pod was moved; fix Secret %s or %s and run again, or pass --skip-config-check", configSecretName(st.Name), objectStorageSecretName(st.Name)) + } + c.logf("rendered config matches the running one (storage, etcd root path, WAL, security, ports)") + } + } else if _, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, st.Name, metav1.GetOptions{}); kerr.IsNotFound(err) { + return fmt.Errorf("the Milvus %s is gone while %s; run \"rollback\" to give the cluster back", st.Name, st.Phase) + } else if err != nil { + return err + } + + st.Phase, st.Method = phaseHandingOver, o.Method + if err := c.saveState(ctx, st); err != nil { + return err + } + start := time.Now() + if err := c.handOver(ctx, st, o.Method, o.timing); err != nil { + return fmt.Errorf("%w\nThe handover stopped; run the same command to continue, or \"rollback\" to give the cluster back", err) + } + if err := c.waitMilvusReady(ctx, st.Name, o.ReadyTimeout); err != nil { + return err + } + st.Phase = phaseAdopted + if err := c.saveState(ctx, st); err != nil { + return err + } + c.logf("Milvus %s is Ready; adopted in %s", st.Name, time.Since(start).Round(time.Second)) + _, _ = fmt.Fprintf(out, ` +Next: + - check the data from a client (collection count, known queries) + - the chart's workloads are kept at 0 replicas; do not run "helm uninstall" (it would delete + the client Service KubeDB now uses), and do not "helm upgrade" (it would start them again) + - to give the cluster back: kubectl dba adopt milvus rollback %s -n %s --yes +`, o.Release, c.ns) + return nil +} + +func newCmdMilvusStatus(f cmdutil.Factory) *cobra.Command { + var fo fileOptions + cmd := &cobra.Command{ + Use: "status [RELEASE]", + Short: i18n.T("Show where each role of an adoption runs"), + Example: templates.Examples(`kubectl dba adopt milvus status my-release -n milvus`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + release, err := fo.resolve(cmd, args) + if err != nil { + return err + } + ctx := context.Background() + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) + if err != nil { + return err + } + st, _, err := c.loadState(ctx, release) + if err != nil { + return err + } + if st == nil { + return fmt.Errorf("no adoption of %s in namespace %s", release, c.ns) + } + return c.printStatus(ctx, st) + }, + } + addFileFlag(cmd, &fo) + return cmd +} + +func (c *cluster) printStatus(ctx context.Context, st *State) error { + phase := "not found" + milvusExists := false + if db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, st.Name, metav1.GetOptions{}); err == nil { + milvusExists = true + phase, _, _ = unstructured.NestedString(db.Object, "status", "phase") + } else if !kerr.IsNotFound(err) { + return err + } + _, _ = fmt.Fprintf(c.out, "Adoption of %s: %s (method %s). Milvus %s: %s\n\n", st.Release, st.Phase, orNone(st.Method), st.Name, phase) + w := tabwriter.NewWriter(c.out, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "ROLE\tCHART WORKLOAD\tCHART\tKUBEDB\tHANDED OVER") + for _, role := range milvusRoles { + for _, u := range st.unitsOf(role) { + var kd side = petSetSide{c, st.Name, u} + if milvusExists { + kd = kubedbSide{c, st.Name, u} + } + _, _ = fmt.Fprintf(w, "%s\t%s %s\t%s\t%s\t%v\n", u.key(), u.Kind, u.Name, + c.sideSummary(ctx, chartSide{c, u}), c.sideSummary(ctx, kd), st.isDone(u)) + } + } + return w.Flush() +} + +func (c *cluster) sideSummary(ctx context.Context, s side) string { + d, err := s.desired(ctx) + if err != nil { + return "?" + } + r, _, err := c.podCounts(ctx, s.selector()) + if err != nil { + return "?" + } + return fmt.Sprintf("%d/%d Ready", r, d) +} + +type rollbackOptions struct { + sourceOptions + timing + Name string + Yes bool + Cleanup bool +} + +func newCmdMilvusRollback(f cmdutil.Factory) *cobra.Command { + var o rollbackOptions + var fo fileOptions + cmd := &cobra.Command{ + Use: "rollback [RELEASE]", + Short: i18n.T("Give an adopted (or partly adopted) Milvus back to the chart's workloads"), + Long: templates.LongDesc(` + Deletes the Milvus object with kubedb.com/milvus-orphan-on-delete, so its + PetSets keep serving, then moves each role back to the chart's workloads, + starting the chart's pods before it stops KubeDB's. Once the chart's + proxies are Ready, the client Service gets back its original selector, + labels and ports. The data is never touched. + + With --cleanup it then deletes the PetSets (at 0 replicas), the Services + other than the client Service and the rendered config KubeDB left, and + recreates the chart Services the adoption removed. PVCs are kept. + + For a Milvus adopted by hand (no state ConfigMap), give its --name: the + chart's replicas are then taken from the PetSets, and the client Service + gets the chart proxies' selector.`), + Example: templates.Examples(` + kubectl dba adopt milvus rollback my-release -n milvus --yes --cleanup + kubectl dba adopt milvus rollback -f ./adopt/adopt.yaml --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + var err error + if o.Release, err = fo.resolve(cmd, args); err != nil { + return err + } + c, err := newCluster(f, cmd.OutOrStdout(), fo.namespace()) + if err != nil { + return err + } + return runRollback(context.Background(), c, o) + }, + } + addSourceFlags(cmd, &o.sourceOptions) + addTimingFlags(cmd, &o.timing) + addFileFlag(cmd, &fo) + cmd.Flags().StringVar(&o.Name, "name", "", "Name of the Milvus, for an adoption done by hand (no state ConfigMap)") + cmd.Flags().BoolVar(&o.Yes, "yes", false, "Proceed; without it, rollback only shows what it would do") + cmd.Flags().BoolVar(&o.Cleanup, "cleanup", false, "Delete what KubeDB left behind once the chart serves again (PVCs are kept)") + return cmd +} + +func runRollback(ctx context.Context, c *cluster, o rollbackOptions) error { + st, cm, err := c.loadState(ctx, o.Release) + if err != nil { + return err + } + if st == nil { + if o.Name == "" { + return fmt.Errorf("no adoption of %s in namespace %s; for a Milvus adopted by hand, give its --name", o.Release, c.ns) + } + if st, err = c.stateFromManualAdoption(ctx, o); err != nil { + return err + } + } + out := c.out + _, _ = fmt.Fprintf(out, "Rollback of Milvus %s to the workloads of %s:\n", st.Name, o.Release) + for _, role := range rollbackOrder { + for _, u := range st.unitsOf(role) { + _, _ = fmt.Fprintf(out, " %-24s -> %s %s, %d replicas\n", u.key(), u.Kind, u.Name, u.Replicas) + } + } + if st.Service != nil { + _, _ = fmt.Fprintf(out, " Service %s -> selector %v\n", st.Service.Name, st.Service.Selector) + } + if !o.Yes { + return errors.New("re-run with --yes to proceed") + } + start := time.Now() + if err := c.releaseMilvus(ctx, st.Name, o.Timeout); err != nil { + return err + } + if err := c.rollBack(ctx, st, o.timing); err != nil { + if cm != nil { + _ = c.saveState(ctx, st) + } + return fmt.Errorf("%w\nThe rollback stopped; run the same command to continue", err) + } + if o.Cleanup { + if err := c.cleanupKubeDBObjects(ctx, st); err != nil { + return err + } + } + if cm != nil { + if err := c.kube.CoreV1().ConfigMaps(c.ns).Delete(ctx, cm.Name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + } + c.logf("the chart's workloads serve %s again; rolled back in %s", o.Release, time.Since(start).Round(time.Second)) + if !o.Cleanup { + _, _ = fmt.Fprintf(out, "KubeDB's PetSets (0 replicas), Services and rendered config are left in place; re-run with --cleanup to delete them.\n") + } + _, _ = fmt.Fprintf(out, "The Secrets with label %s=%s (credentials and the carried config) are kept.\n", LabelRelease, o.Release) + return nil +} + +// stateFromManualAdoption rebuilds the state of a Milvus adopted by hand: the +// chart workloads from the release, their replicas from the KubeDB PetSets. +func (c *cluster) stateFromManualAdoption(ctx context.Context, o rollbackOptions) (*State, error) { + src, err := discover(ctx, c.kube, c.ns, o.sourceOptions) + if err != nil { + return nil, err + } + if len(src.Units) == 0 { + return nil, fmt.Errorf("found no workloads of %s", o.Release) + } + st := &State{Release: o.Release, Name: o.Name, Phase: phaseHandingOver, Units: src.Units} + for i := range st.Units { + u := &st.Units[i] + n, err := petSetSide{c, o.Name, *u}.desired(ctx) + if err != nil { + return nil, err + } + if chart, err := (chartSide{c, *u}).desired(ctx); err == nil && chart > n { + n = chart + } + if n == 0 { + return nil, fmt.Errorf("cannot tell how many replicas %s %s should run: it and PetSet %s are both at 0", u.Kind, u.Name, petSetName(o.Name, *u)) + } + u.Replicas = n + } + svc, err := c.kube.CoreV1().Services(c.ns).Get(ctx, o.Name, metav1.GetOptions{}) + if err == nil { + proxy := src.firstUnit(roleProxy) + st.Service = fallbackServiceRecord(*svc, proxy, src.Templates[proxy.key()]) + } else if !kerr.IsNotFound(err) { + return nil, err + } + return st, nil +} + +func orNone(s string) string { + if s == "" { + return "none" + } + return s +} + +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/pkg/adopt/milvus_auth.go b/pkg/adopt/milvus_auth.go new file mode 100644 index 000000000..56d9eee95 --- /dev/null +++ b/pkg/adopt/milvus_auth.go @@ -0,0 +1,119 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "kmodules.xyz/client-go/tools/portforward" +) + +// Milvus REST error codes for a failed login. +var milvusAuthErrorCodes = map[int]bool{1800: true, 2: true} + +type credentialResult int + +const ( + credentialsAccepted credentialResult = iota + credentialsRejected + credentialsUnverified +) + +// checkCredentials logs in to a running proxy of the release with the given +// root credentials, through a port-forward to its RESTful API. The API server's +// service proxy cannot be used: it does not pass the Authorization header on. +func (c *cluster) checkCredentials(ctx context.Context, proxy Unit, restPort int, user, password string) (credentialResult, string) { + pods, err := c.kube.CoreV1().Pods(c.ns).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(proxy.Selector).String()}) + if err != nil { + return credentialsUnverified, err.Error() + } + pod := "" + for _, p := range pods.Items { + if p.DeletionTimestamp == nil && podReady(p) { + pod = p.Name + break + } + } + if pod == "" { + return credentialsUnverified, "no Ready proxy pod to log in to" + } + tunnel := portforward.NewTunnel(portforward.TunnelOptions{ + Client: c.kube.CoreV1().RESTClient(), + Config: c.cfg, + Resource: "pods", + Name: pod, + Namespace: c.ns, + Remote: restPort, + }) + if err := tunnel.ForwardPort(); err != nil { + return credentialsUnverified, fmt.Sprintf("port-forward to %s: %v", pod, err) + } + defer tunnel.Close() + + scheme, httpc := "http", http.DefaultClient + if c.restTLS != nil { + scheme = "https" + httpc = &http.Client{Transport: &http.Transport{TLSClientConfig: c.restTLS}} + } + url := scheme + "://127.0.0.1:" + strconv.Itoa(tunnel.Local) + "/v2/vectordb/collections/list" + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, url, bytes.NewBufferString("{}")) + if err != nil { + return credentialsUnverified, err.Error() + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+user+":"+password) + resp, err := httpc.Do(req) + if err != nil { + return credentialsUnverified, fmt.Sprintf("RESTful API of %s on port %d: %v", pod, restPort, err) + } + defer resp.Body.Close() // nolint:errcheck + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + var r struct { + Code int `json:"code"` + Message string `json:"message"` + } + if err := json.Unmarshal(body, &r); err != nil { + return credentialsUnverified, fmt.Sprintf("RESTful API of %s answered HTTP %d: %.200s", pod, resp.StatusCode, body) + } + switch { + case r.Code == 0: + return credentialsAccepted, pod + case milvusAuthErrorCodes[r.Code]: + return credentialsRejected, r.Message + } + return credentialsUnverified, fmt.Sprintf("RESTful API of %s answered code %d: %s", pod, r.Code, r.Message) +} + +// restPort is where the running proxies serve the RESTful API: the gRPC port +// unless proxy.http.port moves it. +func restPort(cfg map[string]any) int { + if p, err := strconv.Atoi(lookupString(cfg, "proxy.http.port")); err == nil && p > 0 { + return p + } + if p, err := strconv.Atoi(lookupString(cfg, "proxy.port")); err == nil && p > 0 { + return p + } + return 19530 +} diff --git a/pkg/adopt/milvus_cluster.go b/pkg/adopt/milvus_cluster.go new file mode 100644 index 000000000..e24d0d72f --- /dev/null +++ b/pkg/adopt/milvus_cluster.go @@ -0,0 +1,370 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "time" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +// cluster bundles the clients and the namespace one command works in. +type cluster struct { + kube kubernetes.Interface + dyn dynamic.Interface + cfg *rest.Config + ns string + out io.Writer + // poll is the interval of every wait. + poll time.Duration + // restTLS is set when the running proxies serve TLS. + restTLS *tls.Config +} + +// newCluster connects to the cluster. fileNamespace, from an adoption file, is +// used when -n is not given. +func newCluster(f cmdutil.Factory, out io.Writer, fileNamespace string) (*cluster, error) { + ns, explicit, err := f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil, err + } + if !explicit && fileNamespace != "" { + ns = fileNamespace + } + cfg, err := f.ToRESTConfig() + if err != nil { + return nil, err + } + kc, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + return &cluster{kube: kc, dyn: dyn, cfg: cfg, ns: ns, out: out, poll: 3 * time.Second}, nil +} + +func (c *cluster) logf(format string, a ...any) { + _, _ = fmt.Fprintf(c.out, "%s %s\n", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...)) +} + +// side is one place a Unit's pods run: the chart workload, the KubeDB Milvus +// role, or (during a rollback, after the Milvus is gone) the orphaned PetSet. +type side interface { + String() string + desired(ctx context.Context) (int32, error) + scale(ctx context.Context, n int32) error + selector() map[string]string +} + +// podCounts returns the Ready pods (not terminating) and all pods a selector matches. +func (c *cluster) podCounts(ctx context.Context, sel map[string]string) (ready, total int32, err error) { + pods, err := c.kube.CoreV1().Pods(c.ns).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(sel).String()}) + if err != nil { + return 0, 0, err + } + for _, p := range pods.Items { + total++ + if p.DeletionTimestamp == nil && podReady(p) { + ready++ + } + } + return ready, total, nil +} + +func podReady(p core.Pod) bool { + for _, c := range p.Status.Conditions { + if c.Type == core.PodReady { + return c.Status == core.ConditionTrue + } + } + return false +} + +// waitReady waits until a side has at least n Ready pods. +func (c *cluster) waitReady(ctx context.Context, s side, n int32, timeout time.Duration) error { + return c.waitFor(ctx, timeout, fmt.Sprintf("%s: %d Ready", s, n), func() (bool, string, error) { + r, t, err := c.podCounts(ctx, s.selector()) + return r >= n, fmt.Sprintf("%d/%d Ready", r, t), err + }) +} + +// waitGone waits until a side has at most n pods, terminating ones included. +func (c *cluster) waitGone(ctx context.Context, s side, n int32, timeout time.Duration) error { + return c.waitFor(ctx, timeout, fmt.Sprintf("%s: at most %d pods", s, n), func() (bool, string, error) { + _, t, err := c.podCounts(ctx, s.selector()) + return t <= n, fmt.Sprintf("%d pods", t), err + }) +} + +func (c *cluster) waitFor(ctx context.Context, timeout time.Duration, what string, cond func() (bool, string, error)) error { + deadline := time.Now().Add(timeout) + lastReport := time.Now() + for { + ok, now, err := cond() + if err != nil { + return fmt.Errorf("waiting for %s: %w", what, err) + } + if ok { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for %s (now %s)", timeout, what, now) + } + if time.Since(lastReport) >= 30*time.Second { + c.logf(" waiting for %s (now %s)", what, now) + lastReport = time.Now() + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.poll): + } + } +} + +// chartSide is the workload the chart created. +type chartSide struct { + c *cluster + u Unit +} + +func (s chartSide) String() string { return fmt.Sprintf("%s %s", s.u.Kind, s.u.Name) } + +func (s chartSide) selector() map[string]string { return s.u.Selector } + +func (s chartSide) desired(ctx context.Context) (int32, error) { + switch s.u.Kind { + case "Deployment": + d, err := s.c.kube.AppsV1().Deployments(s.c.ns).Get(ctx, s.u.Name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + if d.Spec.Replicas == nil { + return 1, nil + } + return *d.Spec.Replicas, nil + case "StatefulSet": + st, err := s.c.kube.AppsV1().StatefulSets(s.c.ns).Get(ctx, s.u.Name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + if st.Spec.Replicas == nil { + return 1, nil + } + return *st.Spec.Replicas, nil + } + return 0, fmt.Errorf("unsupported workload kind %s", s.u.Kind) +} + +func (s chartSide) scale(ctx context.Context, n int32) error { + patch := []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, n)) + var err error + switch s.u.Kind { + case "Deployment": + _, err = s.c.kube.AppsV1().Deployments(s.c.ns).Patch(ctx, s.u.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + case "StatefulSet": + _, err = s.c.kube.AppsV1().StatefulSets(s.c.ns).Patch(ctx, s.u.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + default: + err = fmt.Errorf("unsupported workload kind %s", s.u.Kind) + } + if err != nil { + return fmt.Errorf("failed to scale %s to %d: %w", s, n, err) + } + return nil +} + +// kubedbSide is the role (or node group) of the KubeDB Milvus. +type kubedbSide struct { + c *cluster + name string + u Unit +} + +func (s kubedbSide) String() string { + if s.u.Group != "" { + return fmt.Sprintf("Milvus %s %s group %s", s.name, s.u.Role, s.u.Group) + } + return fmt.Sprintf("Milvus %s %s", s.name, s.u.Role) +} + +func (s kubedbSide) selector() map[string]string { return kubedbSelector(s.name, s.u) } + +// replicasPath returns the field path of the Unit's replicas in the Milvus. +func (s kubedbSide) replicasPath(db *unstructured.Unstructured) ([]string, int, error) { + base := []string{"spec", "topology", "distributed", s.u.Role} + if s.u.Group == "" { + return append(base, "replicas"), -1, nil + } + groups, _, _ := unstructured.NestedSlice(db.Object, append(base, "groups")...) + for i, g := range groups { + if m, ok := g.(map[string]any); ok && m["name"] == s.u.Group { + return nil, i, nil + } + } + return nil, -1, fmt.Errorf("no %s group %q in Milvus %s", s.u.Role, s.u.Group, s.name) +} + +func (s kubedbSide) desired(ctx context.Context) (int32, error) { + db, err := s.c.dyn.Resource(MilvusGVR).Namespace(s.c.ns).Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + path, idx, err := s.replicasPath(db) + if err != nil { + return 0, err + } + if idx < 0 { + n, _, _ := unstructured.NestedInt64(db.Object, path...) + return int32(n), nil + } + groups, _, _ := unstructured.NestedSlice(db.Object, "spec", "topology", "distributed", s.u.Role, "groups") + n, _, _ := unstructured.NestedInt64(groups[idx].(map[string]any), "replicas") + return int32(n), nil +} + +func (s kubedbSide) scale(ctx context.Context, n int32) error { + db, err := s.c.dyn.Resource(MilvusGVR).Namespace(s.c.ns).Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + return err + } + _, idx, err := s.replicasPath(db) + if err != nil { + return err + } + var patch []byte + pt := types.MergePatchType + if idx < 0 { + patch = []byte(fmt.Sprintf(`{"spec":{"topology":{"distributed":{%q:{"replicas":%d}}}}}`, s.u.Role, n)) + } else { + pt = types.JSONPatchType + patch = []byte(fmt.Sprintf(`[{"op":"add","path":"/spec/topology/distributed/%s/groups/%d/replicas","value":%d}]`, s.u.Role, idx, n)) + } + if _, err := s.c.dyn.Resource(MilvusGVR).Namespace(s.c.ns).Patch(ctx, s.name, pt, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to set %s to %d replicas: %w", s, n, err) + } + return nil +} + +// petSetSide is a KubeDB PetSet after its Milvus was deleted with the +// orphan-on-delete annotation: the pods keep running under the PetSet controller. +type petSetSide struct { + c *cluster + name string + u Unit +} + +func (s petSetSide) String() string { return "PetSet " + petSetName(s.name, s.u) } + +func (s petSetSide) selector() map[string]string { return kubedbSelector(s.name, s.u) } + +func (s petSetSide) desired(ctx context.Context) (int32, error) { + ps, err := s.c.dyn.Resource(PetSetGVR).Namespace(s.c.ns).Get(ctx, petSetName(s.name, s.u), metav1.GetOptions{}) + if kerr.IsNotFound(err) { + return 0, nil + } + if err != nil { + return 0, err + } + n, found, _ := unstructured.NestedInt64(ps.Object, "spec", "replicas") + if !found { + return 1, nil + } + return int32(n), nil +} + +func (s petSetSide) scale(ctx context.Context, n int32) error { + patch := []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, n)) + _, err := s.c.dyn.Resource(PetSetGVR).Namespace(s.c.ns).Patch(ctx, petSetName(s.name, s.u), types.MergePatchType, patch, metav1.PatchOptions{}) + if kerr.IsNotFound(err) && n == 0 { + return nil + } + if err != nil { + return fmt.Errorf("failed to scale %s to %d: %w", s, n, err) + } + return nil +} + +// loadState finds the adoption of a release in the namespace. +func (c *cluster) loadState(ctx context.Context, release string) (*State, *core.ConfigMap, error) { + cms, err := c.kube.CoreV1().ConfigMaps(c.ns).List(ctx, metav1.ListOptions{LabelSelector: LabelRelease + "=" + release}) + if err != nil { + return nil, nil, err + } + for i := range cms.Items { + cm := &cms.Items[i] + raw, ok := cm.Data[stateKey] + if !ok { + continue + } + st := &State{} + if err := json.Unmarshal([]byte(raw), st); err != nil { + return nil, nil, fmt.Errorf("ConfigMap %s holds an unreadable adoption state: %w", cm.Name, err) + } + return st, cm, nil + } + return nil, nil, nil +} + +func (c *cluster) saveState(ctx context.Context, st *State) error { + raw, err := json.Marshal(st) + if err != nil { + return err + } + cm, err := c.kube.CoreV1().ConfigMaps(c.ns).Get(ctx, stateConfigMapName(st.Name), metav1.GetOptions{}) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[stateKey] = string(raw) + _, err = c.kube.CoreV1().ConfigMaps(c.ns).Update(ctx, cm, metav1.UpdateOptions{}) + return err +} + +// renderedConfig returns the milvus.yaml the operator rendered for a Milvus: +// the Secret it owns that carries that key. +func (c *cluster) renderedConfig(ctx context.Context, db *unstructured.Unstructured) (string, error) { + secs, err := c.kube.CoreV1().Secrets(c.ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return "", err + } + for _, s := range secs.Items { + for _, o := range s.OwnerReferences { + if o.UID == db.GetUID() { + if v, ok := s.Data["milvus.yaml"]; ok { + return string(v), nil + } + } + } + } + return "", nil +} diff --git a/pkg/adopt/milvus_config.go b/pkg/adopt/milvus_config.go new file mode 100644 index 000000000..e2ae279b0 --- /dev/null +++ b/pkg/adopt/milvus_config.go @@ -0,0 +1,281 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "fmt" + "sort" + "strings" + + "sigs.k8s.io/yaml" +) + +// Milvus reads its configuration files from /milvus/configs in this order, each +// overriding the one before; a chart mounts some of them from a ConfigMap. +var milvusConfigFiles = []string{"milvus.yaml", "default.yaml", "user.yaml"} + +const defaultEtcdRootPath = "by-dev" + +// The minio.* keys the KubeDB operator reads from spec.objectStorage.configSecret +// and writes back into the rendered milvus.yaml. +var objectStorageKeys = []string{ + "address", "port", "useSSL", "bucketName", "rootPath", "useIAM", "cloudProvider", + "region", "iamEndpoint", "useVirtualHost", "accessKeyID", "secretAccessKey", +} + +// mergeConfigFiles merges the files of one Milvus in load order. Unknown file +// names are returned so the caller can report them. +func mergeConfigFiles(files map[string]string) (map[string]any, []string, error) { + out := map[string]any{} + for _, name := range milvusConfigFiles { + raw, ok := files[name] + if !ok { + continue + } + var doc map[string]any + if err := yaml.Unmarshal([]byte(raw), &doc); err != nil { + return nil, nil, fmt.Errorf("failed to parse %s: %w", name, err) + } + out = deepMerge(out, doc) + } + var unknown []string + for name := range files { + if !contains(milvusConfigFiles, name) { + unknown = append(unknown, name) + } + } + sort.Strings(unknown) + return out, unknown, nil +} + +func deepMerge(dst, src map[string]any) map[string]any { + for k, v := range src { + if sm, ok := v.(map[string]any); ok { + if dm, ok := dst[k].(map[string]any); ok { + dst[k] = deepMerge(dm, sm) + continue + } + } + dst[k] = v + } + return dst +} + +// lookup returns the value at a dotted path, e.g. "minio.bucketName". +func lookup(cfg map[string]any, path string) (any, bool) { + var cur any = cfg + for _, p := range strings.Split(path, ".") { + m, ok := cur.(map[string]any) + if !ok { + return nil, false + } + if cur, ok = m[p]; !ok { + return nil, false + } + } + return cur, true +} + +func lookupString(cfg map[string]any, path string) string { + v, ok := lookup(cfg, path) + if !ok || v == nil { + return "" + } + return scalarString(v) +} + +func lookupBool(cfg map[string]any, path string) bool { + return strings.EqualFold(lookupString(cfg, path), "true") +} + +// scalarString renders a YAML scalar the way it would be written back: numbers +// without exponent, booleans as true/false. +func scalarString(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + case float64: + if t == float64(int64(t)) { + return fmt.Sprintf("%d", int64(t)) + } + return fmt.Sprintf("%v", t) + default: + return fmt.Sprintf("%v", t) + } +} + +// etcdSettings is what the running Milvus uses to reach its metadata. +type etcdSettings struct { + Endpoints []string + RootPath string + AuthEnabled bool + TLSEnabled bool +} + +func etcdFromConfig(cfg map[string]any) etcdSettings { + s := etcdSettings{RootPath: lookupString(cfg, "etcd.rootPath")} + if s.RootPath == "" { + s.RootPath = defaultEtcdRootPath + } + s.AuthEnabled = lookupBool(cfg, "etcd.auth.enabled") + s.TLSEnabled = lookupBool(cfg, "etcd.ssl.enabled") + scheme := "http://" + if s.TLSEnabled { + scheme = "https://" + } + var eps []string + switch v, _ := lookup(cfg, "etcd.endpoints"); t := v.(type) { + case []any: + for _, e := range t { + eps = append(eps, scalarString(e)) + } + case string: + eps = strings.Split(t, ",") + } + for _, e := range eps { + if e = strings.TrimSpace(e); e == "" { + continue + } + if !strings.Contains(e, "://") { + e = scheme + e + } + s.Endpoints = append(s.Endpoints, e) + } + return s +} + +// walType returns the message queue the config selects ("" when it does not say). +func walType(cfg map[string]any) string { + if t := lookupString(cfg, "mq.type"); t != "" && t != "default" { + return t + } + return lookupString(cfg, "messageQueue") +} + +// objectStorageData returns the Secret data for spec.objectStorage.configSecret: +// every minio.* key the running config sets. +func objectStorageData(cfg map[string]any) map[string]string { + out := map[string]string{} + for _, k := range objectStorageKeys { + if v, ok := lookup(cfg, "minio."+k); ok && v != nil && scalarString(v) != "" { + out[k] = scalarString(v) + } + } + return out +} + +// carriedConfig is the milvus.yaml handed to KubeDB through +// spec.configuration.secretName. The etcd section is dropped (the Milvus object +// sets the endpoints and root path); everything else is kept as it runs today. +// Milvus serves its RESTful API on the gRPC port unless proxy.http.port is set, +// while KubeDB defaults it to 8080, so an unset port is pinned to "". +func carriedConfig(cfg map[string]any) (string, error) { + out := deepCopy(cfg) + delete(out, "etcd") + // The operator renders TLS for the certificates it mounts. The carried config + // wins a merge, so its paths would point at files the new pods do not have. + delete(out, "tls") + delete(out, "internaltls") + if common, ok := out["common"].(map[string]any); ok { + if sec, ok := common["security"].(map[string]any); ok { + delete(sec, "tlsMode") + delete(sec, "internaltlsEnabled") + } + } + if v, ok := lookup(out, "proxy.http.port"); !ok || v == nil { + proxy, _ := out["proxy"].(map[string]any) + if proxy == nil { + proxy = map[string]any{} + out["proxy"] = proxy + } + http, _ := proxy["http"].(map[string]any) + if http == nil { + http = map[string]any{} + proxy["http"] = http + } + http["port"] = "" + } + b, err := yaml.Marshal(out) + return string(b), err +} + +func deepCopy(in map[string]any) map[string]any { + out := make(map[string]any, len(in)) + for k, v := range in { + if m, ok := v.(map[string]any); ok { + out[k] = deepCopy(m) + continue + } + out[k] = v + } + return out +} + +// criticalConfigPaths are the settings that decide where the data is read from +// and how clients log in. After the Milvus is created, the config the operator +// renders must agree with the running one on each of them. +var criticalConfigPaths = []string{ + "minio.address", "minio.port", "minio.bucketName", "minio.rootPath", "minio.useSSL", + "minio.useIAM", "minio.cloudProvider", "minio.region", "minio.accessKeyID", "minio.secretAccessKey", + "mq.type", "woodpecker.storage.type", "woodpecker.storage.rootPath", + "common.security.authorizationEnabled", "proxy.port", "proxy.http.port", + "common.security.tlsMode", "common.security.internaltlsEnabled", "etcd.ssl.enabled", +} + +// tlsSwitchPaths are off when the operator renders nothing for them. +var tlsSwitchPaths = []string{"common.security.tlsMode", "common.security.internaltlsEnabled", "etcd.ssl.enabled"} + +// configMismatches compares the running config with the one the operator +// rendered. A path the running config does not set is compared only for the +// etcd root path, whose Milvus and KubeDB defaults differ. +func configMismatches(running, rendered map[string]any) []string { + var out []string + want := etcdFromConfig(running).RootPath + if got := lookupString(rendered, "etcd.rootPath"); got != want { + out = append(out, fmt.Sprintf("etcd.rootPath: running %q, rendered %q", want, got)) + } + for _, p := range criticalConfigPaths { + rv, ok := lookup(running, p) + if !ok || rv == nil { + if p == "proxy.http.port" { + rv = "" // carriedConfig pins it + } else { + continue + } + } + a, b := scalarString(rv), lookupString(rendered, p) + if b == "" && (a == "false" || a == "0") && contains(tlsSwitchPaths, p) { + continue + } + if a != b { + if strings.Contains(strings.ToLower(p), "secret") || strings.HasSuffix(p, "accessKeyID") { + out = append(out, p+": differs") + continue + } + out = append(out, fmt.Sprintf("%s: running %q, rendered %q", p, a, b)) + } + } + return out +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} diff --git a/pkg/adopt/milvus_file.go b/pkg/adopt/milvus_file.go new file mode 100644 index 000000000..81693fb02 --- /dev/null +++ b/pkg/adopt/milvus_file.go @@ -0,0 +1,299 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "errors" + "fmt" + "os" + "strconv" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +const ( + adoptionFileAPIVersion = "cli.kubedb.com/v1alpha1" + adoptionFileKind = "MilvusAdoption" + // adoptionFileName is the file "plan --output-dir" writes. + adoptionFileName = "adopt.yaml" +) + +// adoptionFile holds the options of an adoption, so they need not be typed as +// flags on every command. A flag given on the command line wins over the file. +type adoptionFile struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Release string `json:"release"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Source adoptionSource `json:"source,omitempty"` + Auth adoptionAuth `json:"auth,omitempty"` + TLS adoptionTLSFile `json:"tls,omitempty"` + RestrictedSCC *bool `json:"restrictedSCC,omitempty"` + Handover adoptionHandover `json:"handover,omitempty"` + Rollback adoptionRollback `json:"rollback,omitempty"` +} + +type adoptionSource struct { + Selector string `json:"selector,omitempty"` + RoleLabel string `json:"roleLabel,omitempty"` + Workloads []adoptionWorkload `json:"workloads,omitempty"` + ConfigFile string `json:"configFile,omitempty"` +} + +type adoptionWorkload struct { + Role string `json:"role"` + Group string `json:"group,omitempty"` + Kind string `json:"kind"` + Name string `json:"name"` +} + +type adoptionAuth struct { + AuthSecret string `json:"authSecret,omitempty"` + RootPasswordFile string `json:"rootPasswordFile,omitempty"` + EtcdAuthSecret string `json:"etcdAuthSecret,omitempty"` + SkipCredentialCheck *bool `json:"skipCredentialCheck,omitempty"` +} + +type adoptionTLSFile struct { + Issuer string `json:"issuer,omitempty"` + ClientSecret string `json:"clientSecret,omitempty"` +} + +type adoptionHandover struct { + Method string `json:"method,omitempty"` + QueryNodeDelay *metav1.Duration `json:"querynodeDelay,omitempty"` + Settle *metav1.Duration `json:"settle,omitempty"` + Timeout *metav1.Duration `json:"timeout,omitempty"` + ReadyTimeout *metav1.Duration `json:"readyTimeout,omitempty"` + SkipConfigCheck *bool `json:"skipConfigCheck,omitempty"` +} + +type adoptionRollback struct { + Cleanup *bool `json:"cleanup,omitempty"` +} + +// loadAdoptionFile reads and checks an adoption file. Unknown keys are errors, +// so a misspelt option is not silently ignored. +func loadAdoptionFile(path string) (*adoptionFile, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + af := &adoptionFile{} + if err := yaml.UnmarshalStrict(raw, af); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if err := af.validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return af, nil +} + +func (af *adoptionFile) validate() error { + if af.APIVersion != adoptionFileAPIVersion || af.Kind != adoptionFileKind { + return fmt.Errorf("want apiVersion %s and kind %s, got %q and %q", adoptionFileAPIVersion, adoptionFileKind, af.APIVersion, af.Kind) + } + if af.Release == "" { + return errors.New("release is required") + } + if m := af.Handover.Method; m != "" && m != methodOverlap && m != methodStopStart { + return fmt.Errorf("handover.method must be %s or %s, got %q", methodOverlap, methodStopStart, m) + } + for i, w := range af.Source.Workloads { + if !contains(milvusRoles, w.Role) { + return fmt.Errorf("source.workloads[%d]: role %q is not one of %v", i, w.Role, milvusRoles) + } + if w.Kind != "Deployment" && w.Kind != "StatefulSet" { + return fmt.Errorf("source.workloads[%d]: kind must be Deployment or StatefulSet, got %q", i, w.Kind) + } + if w.Name == "" { + return fmt.Errorf("source.workloads[%d]: name is required", i) + } + } + return nil +} + +// workloadFlag renders a workload the way --workload takes it. +func (w adoptionWorkload) workloadFlag() string { + role := w.Role + if w.Group != "" { + role += "/" + w.Group + } + return role + "=" + w.Kind + "/" + w.Name +} + +type flagValue struct { + flag string + values []string +} + +// flagValues maps the file onto the command's flags. +func (af *adoptionFile) flagValues() []flagValue { + var out []flagValue + str := func(flag, v string) { + if v != "" { + out = append(out, flagValue{flag, []string{v}}) + } + } + boolean := func(flag string, v *bool) { + if v != nil { + out = append(out, flagValue{flag, []string{strconv.FormatBool(*v)}}) + } + } + duration := func(flag string, v *metav1.Duration) { + if v != nil { + out = append(out, flagValue{flag, []string{v.Duration.String()}}) + } + } + str("name", af.Name) + str("version", af.Version) + str("selector", af.Source.Selector) + str("role-label", af.Source.RoleLabel) + if len(af.Source.Workloads) > 0 { + fv := flagValue{flag: "workload"} + for _, w := range af.Source.Workloads { + fv.values = append(fv.values, w.workloadFlag()) + } + out = append(out, fv) + } + str("config-file", af.Source.ConfigFile) + str("auth-secret", af.Auth.AuthSecret) + str("root-password-file", af.Auth.RootPasswordFile) + str("etcd-auth-secret", af.Auth.EtcdAuthSecret) + boolean("skip-credential-check", af.Auth.SkipCredentialCheck) + str("tls-issuer", af.TLS.Issuer) + str("tls-client-secret", af.TLS.ClientSecret) + boolean("restricted-scc", af.RestrictedSCC) + str("method", af.Handover.Method) + duration("querynode-delay", af.Handover.QueryNodeDelay) + duration("settle", af.Handover.Settle) + duration("timeout", af.Handover.Timeout) + duration("ready-timeout", af.Handover.ReadyTimeout) + boolean("skip-config-check", af.Handover.SkipConfigCheck) + boolean("cleanup", af.Rollback.Cleanup) + return out +} + +// applyFlags sets every flag of cmd that the file gives and the command line +// did not. Options a command does not have are skipped. +func (af *adoptionFile) applyFlags(cmd *cobra.Command) error { + fs := cmd.Flags() + for _, fv := range af.flagValues() { + fl := fs.Lookup(fv.flag) + if fl == nil || fl.Changed { + continue + } + for _, v := range fv.values { + if err := fs.Set(fv.flag, v); err != nil { + return fmt.Errorf("--%s from the file: %w", fv.flag, err) + } + } + } + return nil +} + +// fileOptions is the -f flag every adopt milvus command has. +type fileOptions struct { + Filename string + file *adoptionFile +} + +func addFileFlag(cmd *cobra.Command, o *fileOptions) { + cmd.Flags().StringVarP(&o.Filename, "filename", "f", "", "File with the adoption options (kind MilvusAdoption; \"plan --output-dir\" writes one). Flags given on the command line win") +} + +// resolve loads the file, fills the flags it gives, and returns the release: +// the RELEASE argument, or release from the file. +func (o *fileOptions) resolve(cmd *cobra.Command, args []string) (string, error) { + if len(args) > 1 { + return "", errors.New("give at most one RELEASE") + } + if o.Filename != "" { + af, err := loadAdoptionFile(o.Filename) + if err != nil { + return "", err + } + if err := af.applyFlags(cmd); err != nil { + return "", err + } + o.file = af + } + switch { + case len(args) == 1 && o.file != nil && o.file.Release != args[0]: + return "", fmt.Errorf("RELEASE %q differs from release %q in %s", args[0], o.file.Release, o.Filename) + case len(args) == 1: + return args[0], nil + case o.file != nil: + return o.file.Release, nil + } + return "", errors.New("give the RELEASE (the Helm release, or the instance label value of the workloads), or a file with -f") +} + +// namespace is the file's namespace, used when -n is not given. +func (o *fileOptions) namespace() string { + if o.file == nil { + return "" + } + return o.file.Namespace +} + +// adoptionFileFor writes down what a plan found, as a file to review and pass +// to run, status and rollback with -f. +func adoptionFileFor(p *adoptPlan, o planOptions, ns string) ([]byte, error) { + version, _ := p.Objects.Milvus.Object["spec"].(map[string]any)["version"].(string) + af := adoptionFile{ + APIVersion: adoptionFileAPIVersion, + Kind: adoptionFileKind, + Release: o.Release, + Namespace: ns, + Name: p.State.Name, + Version: version, + Source: adoptionSource{ConfigFile: o.ConfigFile}, + Auth: adoptionAuth{ + AuthSecret: o.AuthSecret, + RootPasswordFile: o.RootPasswordFile, + EtcdAuthSecret: o.EtcdAuthSecret, + }, + TLS: adoptionTLSFile{Issuer: o.TLSIssuer, ClientSecret: o.TLSClientSecret}, + Handover: adoptionHandover{ + Method: methodOverlap, + QueryNodeDelay: &metav1.Duration{Duration: time.Minute}, + Settle: &metav1.Duration{Duration: 20 * time.Second}, + Timeout: &metav1.Duration{Duration: 15 * time.Minute}, + ReadyTimeout: &metav1.Duration{Duration: 15 * time.Minute}, + }, + Rollback: adoptionRollback{Cleanup: ptrBool(false)}, + } + if o.RestrictedSCC { + af.RestrictedSCC = ptrBool(true) + } + for _, u := range p.State.Units { + af.Source.Workloads = append(af.Source.Workloads, adoptionWorkload{Role: u.Role, Group: u.Group, Kind: u.Kind, Name: u.Name}) + } + b, err := yaml.Marshal(af) + if err != nil { + return nil, err + } + header := "# Options of the adoption, as plan found them. Review, then:\n" + + "# kubectl dba adopt milvus run -f " + adoptionFileName + " --yes\n" + + "# Raise handover.querynodeDelay for large collections (about the time one QueryNode takes to load).\n" + return append([]byte(header), b...), nil +} + +func ptrBool(b bool) *bool { return &b } diff --git a/pkg/adopt/milvus_handover.go b/pkg/adopt/milvus_handover.go new file mode 100644 index 000000000..9063902c0 --- /dev/null +++ b/pkg/adopt/milvus_handover.go @@ -0,0 +1,524 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "fmt" + "time" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" +) + +// Handover methods. +const ( + // methodOverlap starts the KubeDB pods of a role before it stops the chart's, + // so the role never stops serving. It needs room for one extra set of pods + // (one extra QueryNode at a time). + methodOverlap = "overlap" + // methodStopStart stops the chart's pods of a role before it starts KubeDB's. + // Each role is briefly unavailable; no extra capacity is needed. + methodStopStart = "stop-start" +) + +// handoverOrder is the order the roles change hands in. With the overlap +// method the proxies go first, so clients reach the KubeDB pods early; a +// stop-start handover moves the coordinators first and the proxies last. +func handoverOrder(method string) []string { + if method == methodStopStart { + return []string{roleMixCoord, roleDataNode, roleQueryNode, roleStreamingNode, roleProxy} + } + return []string{roleProxy, roleMixCoord, roleDataNode, roleStreamingNode, roleQueryNode} +} + +// rollbackOrder returns the roles to the chart, the proxies last so the +// client Service moves once every other role serves from the chart again. +var rollbackOrder = []string{roleQueryNode, roleStreamingNode, roleDataNode, roleMixCoord, roleProxy} + +type timing struct { + Timeout time.Duration // per wait + Settle time.Duration // after each role + QueryNodeDelay time.Duration // after each new QueryNode, before an old one stops +} + +// step is one replica change on the way from one side to the other: grow +// `to` to toN, then shrink `from` to fromN. +type step struct { + toN, fromN int32 +} + +// swapSteps returns the steps that move n replicas from one side to the +// other. QueryNodes go one at a time, so the cluster never holds more than one +// extra QueryNode and every old one stops only after a new one serves. +func swapSteps(n int32, oneByOne bool) []step { + if !oneByOne || n <= 1 { + return []step{{toN: n, fromN: 0}} + } + var out []step + for i := int32(1); i <= n; i++ { + out = append(out, step{toN: i, fromN: n - i}) + } + return out +} + +// overlapSwap moves a Unit from one side to the other without a gap: each +// step grows the target first, waits for it to be Ready, then shrinks the +// source. It is safe to repeat: the target never shrinks and the source never +// grows. +func (c *cluster) overlapSwap(ctx context.Context, from, to side, n int32, oneByOne bool, t timing, beforeShrink func() error) error { + steps := swapSteps(n, oneByOne) + for i, s := range steps { + cur, err := to.desired(ctx) + if err != nil { + return err + } + if cur < s.toN { + c.logf("%s -> %d replicas", to, s.toN) + if err := to.scale(ctx, s.toN); err != nil { + return err + } + } + if err := c.waitReady(ctx, to, s.toN, t.Timeout); err != nil { + return err + } + if oneByOne && t.QueryNodeDelay > 0 && cur < s.toN { + c.logf(" %s serves %d; waiting %s before stopping one of %s", to, s.toN, t.QueryNodeDelay, from) + sleep(ctx, t.QueryNodeDelay) + } + if i == len(steps)-1 && beforeShrink != nil { + if err := beforeShrink(); err != nil { + return err + } + } + cur, err = from.desired(ctx) + if err != nil { + return err + } + if cur > s.fromN { + c.logf("%s -> %d replicas", from, s.fromN) + if err := from.scale(ctx, s.fromN); err != nil { + return err + } + } + if err := c.waitGone(ctx, from, s.fromN, t.Timeout); err != nil { + return err + } + } + return nil +} + +// stopStartSwap stops the source, waits until its pods are gone, then starts +// the target. +func (c *cluster) stopStartSwap(ctx context.Context, from, to side, n int32, t timing) error { + if cur, err := from.desired(ctx); err != nil { + return err + } else if cur > 0 { + c.logf("%s -> 0 replicas", from) + if err := from.scale(ctx, 0); err != nil { + return err + } + } + if err := c.waitGone(ctx, from, 0, t.Timeout); err != nil { + return err + } + if cur, err := to.desired(ctx); err != nil { + return err + } else if cur < n { + c.logf("%s -> %d replicas", to, n) + if err := to.scale(ctx, n); err != nil { + return err + } + } + return c.waitReady(ctx, to, n, t.Timeout) +} + +// handOver moves every Unit of the adoption to KubeDB, recording each one. +func (c *cluster) handOver(ctx context.Context, st *State, method string, t timing) error { + for _, role := range handoverOrder(method) { + for _, u := range st.unitsOf(role) { + if st.isDone(u) { + continue + } + start := time.Now() + from, to := chartSide{c, u}, kubedbSide{c, st.Name, u} + c.logf("== %s (%s): %d replicas, %s", u.key(), u.Name, u.Replicas, method) + var err error + if method == methodStopStart { + err = c.stopStartSwap(ctx, from, to, u.Replicas, t) + } else { + err = c.overlapSwap(ctx, from, to, u.Replicas, u.Role == roleQueryNode, t, nil) + } + if err != nil { + return fmt.Errorf("%s: %w", u.key(), err) + } + st.markDone(u) + if err := c.saveState(ctx, st); err != nil { + return err + } + c.logf("== %s handed over in %s", u.key(), time.Since(start).Round(time.Second)) + sleep(ctx, t.Settle) + } + } + return nil +} + +// rollBack returns every Unit to the chart workloads. The Milvus must be gone +// (deleted with the orphan-on-delete annotation), so its PetSets are scaled +// directly and the operator no longer moves the client Service. +func (c *cluster) rollBack(ctx context.Context, st *State, t timing) error { + for _, role := range rollbackOrder { + for _, u := range st.unitsOf(role) { + from, to := petSetSide{c, st.Name, u}, chartSide{c, u} + cur, err := from.desired(ctx) + if err != nil { + return err + } + _, pods, err := c.podCounts(ctx, from.selector()) + if err != nil { + return err + } + var restore func() error + if u.Role == roleProxy && st.Service != nil { + restore = func() error { return c.restoreClientService(ctx, st) } + } + if cur == 0 && pods == 0 { + // Nothing of this Unit runs on KubeDB (it was never handed over, or a + // stop-start handover stopped between the two sides). + if n, err := to.desired(ctx); err != nil { + return err + } else if n < u.Replicas { + c.logf("%s -> %d replicas", to, u.Replicas) + if err := to.scale(ctx, u.Replicas); err != nil { + return err + } + } + if err := c.waitReady(ctx, to, u.Replicas, t.Timeout); err != nil { + return err + } + if restore != nil { + if err := restore(); err != nil { + return err + } + } + st.unmarkDone(u) + continue + } + start := time.Now() + c.logf("== %s (%s): back to the chart, %d replicas", u.key(), u.Name, u.Replicas) + if err := c.overlapSwap(ctx, from, to, u.Replicas, u.Role == roleQueryNode, t, restore); err != nil { + return fmt.Errorf("%s: %w", u.key(), err) + } + st.unmarkDone(u) + c.logf("== %s back on the chart in %s", u.key(), time.Since(start).Round(time.Second)) + sleep(ctx, t.Settle) + } + } + return nil +} + +// restoreClientService points the client Service back at the chart's proxies, +// with the labels and ports it had before the adoption. +func (c *cluster) restoreClientService(ctx context.Context, st *State) error { + rec := st.Service + svc, err := c.kube.CoreV1().Services(c.ns).Get(ctx, rec.Name, metav1.GetOptions{}) + if err != nil { + return err + } + svc.Spec.Selector = rec.Selector + if rec.Labels != nil { + svc.Labels = rec.Labels + } + if len(rec.Ports) > 0 { + svc.Spec.Ports = rec.Ports + } + if _, err := c.kube.CoreV1().Services(c.ns).Update(ctx, svc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to point Service %s back at the chart proxies: %w", rec.Name, err) + } + c.logf("Service %s points at the chart proxies again", rec.Name) + return nil +} + +// releaseMilvus deletes the Milvus with the orphan-on-delete annotation, so +// its PetSets, Services and pods stay as they are. +func (c *cluster) releaseMilvus(ctx context.Context, name string, timeout time.Duration) error { + ri := c.dyn.Resource(MilvusGVR).Namespace(c.ns) + if _, err := ri.Get(ctx, name, metav1.GetOptions{}); kerr.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:"true"}},"spec":{"deletionPolicy":"Halt"}}`, annOrphanOnDelete) + if _, err := ri.Patch(ctx, name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to prepare Milvus %s for release: %w", name, err) + } + c.logf("deleting Milvus %s with %s; its pods keep running", name, annOrphanOnDelete) + if err := ri.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + return c.waitFor(ctx, timeout, "Milvus "+name+" to be released", func() (bool, string, error) { + _, err := ri.Get(ctx, name, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + return true, "", nil + } + return false, "still present", err + }) +} + +// createObjects creates what a plan holds, keeping any object that already +// exists (for example one edited from "plan --output-dir"). +func (c *cluster) createObjects(ctx context.Context, p *adoptPlan) error { + for _, s := range p.Objects.Secrets { + _, err := c.kube.CoreV1().Secrets(c.ns).Create(ctx, s, metav1.CreateOptions{}) + switch { + case kerr.IsAlreadyExists(err): + c.logf("Secret %s exists; using it", s.Name) + case err != nil: + return fmt.Errorf("failed to create Secret %s: %w", s.Name, err) + default: + c.logf("created Secret %s", s.Name) + } + } + _, err := c.kube.CoreV1().ConfigMaps(c.ns).Create(ctx, p.Objects.State, metav1.CreateOptions{}) + switch { + case kerr.IsAlreadyExists(err): + case err != nil: + return fmt.Errorf("failed to create ConfigMap %s: %w", p.Objects.State.Name, err) + default: + c.logf("created ConfigMap %s (adoption state)", p.Objects.State.Name) + } + return nil +} + +// removeGoverningServices deletes the chart's non-headless Services on names +// KubeDB needs headless, recording them so a rollback can recreate them. +func (c *cluster) removeGoverningServices(ctx context.Context, st *State, svcs []core.Service) error { + for _, svc := range svcs { + st.RemovedServices = append(st.RemovedServices, recordService(svc)) + } + if len(svcs) > 0 { + if err := c.saveState(ctx, st); err != nil { + return err + } + } + for _, svc := range svcs { + err := c.kube.CoreV1().Services(c.ns).Delete(ctx, svc.Name, metav1.DeleteOptions{}) + if err != nil && !kerr.IsNotFound(err) { + return fmt.Errorf("failed to delete Service %s: %w", svc.Name, err) + } + c.logf("deleted the chart's Service %s (KubeDB recreates it headless)", svc.Name) + } + return nil +} + +func recordService(svc core.Service) ServiceRecord { + return ServiceRecord{ + Name: svc.Name, Selector: svc.Spec.Selector, Labels: svc.Labels, + Annotations: svc.Annotations, Ports: svc.Spec.Ports, Type: svc.Spec.Type, + } +} + +// createMilvus creates the staged Milvus and waits until the operator has +// rendered its config and created a PetSet for every Unit. +func (c *cluster) createMilvus(ctx context.Context, p *adoptPlan, timeout time.Duration) (*unstructured.Unstructured, error) { + ri := c.dyn.Resource(MilvusGVR).Namespace(c.ns) + db, err := ri.Create(ctx, p.Objects.Milvus, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + db, err = ri.Get(ctx, p.State.Name, metav1.GetOptions{}) + } else if err == nil { + c.logf("created Milvus %s, every role at 0 replicas", p.State.Name) + } + if err != nil { + return nil, fmt.Errorf("failed to create Milvus %s: %w", p.State.Name, err) + } + return db, c.waitOperator(ctx, p.State, timeout) +} + +func (c *cluster) waitOperator(ctx context.Context, st *State, timeout time.Duration) error { + return c.waitFor(ctx, timeout, "the operator to create the PetSets and render the config of Milvus "+st.Name, func() (bool, string, error) { + db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, st.Name, metav1.GetOptions{}) + if err != nil { + return false, "", err + } + cfg, err := c.renderedConfig(ctx, db) + if err != nil { + return false, "", err + } + if cfg == "" { + return false, "no rendered config yet; see kubectl describe milvus " + st.Name, nil + } + for _, u := range st.Units { + if _, err := c.dyn.Resource(PetSetGVR).Namespace(c.ns).Get(ctx, petSetName(st.Name, u), metav1.GetOptions{}); kerr.IsNotFound(err) { + return false, "PetSet " + petSetName(st.Name, u) + " not created yet", nil + } else if err != nil { + return false, "", err + } + } + return true, "", nil + }) +} + +// checkRenderedConfig compares the config the operator rendered with the one +// the cluster runs, on every setting that decides where the data is read from. +func (c *cluster) checkRenderedConfig(ctx context.Context, name string, running map[string]any) ([]string, error) { + db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + raw, err := c.renderedConfig(ctx, db) + if err != nil { + return nil, err + } + var rendered map[string]any + if err := yaml.Unmarshal([]byte(raw), &rendered); err != nil { + return nil, fmt.Errorf("failed to parse the rendered config: %w", err) + } + return configMismatches(running, rendered), nil +} + +// waitMilvusReady waits for the Milvus to report Ready. +func (c *cluster) waitMilvusReady(ctx context.Context, name string, timeout time.Duration) error { + return c.waitFor(ctx, timeout, "Milvus "+name+" to be Ready", func() (bool, string, error) { + db, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, "", err + } + phase, _, _ := unstructured.NestedString(db.Object, "status", "phase") + return phase == "Ready", "phase " + phase, nil + }) +} + +// cleanupKubeDBObjects deletes what the released Milvus left behind (its +// PetSets, Services other than the client Service, rendered config and RBAC), +// then recreates the chart Services the adoption removed. PVCs are kept. +func (c *cluster) cleanupKubeDBObjects(ctx context.Context, st *State) error { + sel := metav1.ListOptions{LabelSelector: labelInstance + "=" + st.Name + "," + labelManagedBy + "=kubedb.com"} + pss, err := c.dyn.Resource(PetSetGVR).Namespace(c.ns).List(ctx, sel) + if err != nil { + return err + } + for _, ps := range pss.Items { + if n, _, _ := unstructured.NestedInt64(ps.Object, "spec", "replicas"); n > 0 { + return fmt.Errorf("PetSet %s still runs %d replicas; not deleting", ps.GetName(), n) + } + if err := c.dyn.Resource(PetSetGVR).Namespace(c.ns).Delete(ctx, ps.GetName(), metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted PetSet %s", ps.GetName()) + } + svcs, err := c.kube.CoreV1().Services(c.ns).List(ctx, sel) + if err != nil { + return err + } + for _, svc := range svcs.Items { + if st.Service != nil && svc.Name == st.Service.Name { + continue + } + if err := c.kube.CoreV1().Services(c.ns).Delete(ctx, svc.Name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted Service %s", svc.Name) + } + secs, err := c.kube.CoreV1().Secrets(c.ns).List(ctx, sel) + if err != nil { + return err + } + for _, s := range secs.Items { + if _, ok := s.Data["milvus.yaml"]; !ok { + continue // credentials stay + } + if err := c.kube.CoreV1().Secrets(c.ns).Delete(ctx, s.Name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted Secret %s (rendered config)", s.Name) + } + // Copies of certificates. The label keeps a Secret of the user's with the same + // name, which run reuses, out of this. + for _, name := range []string{serverTLSSecretName(st.Name), etcdTLSSecretName(st.Name)} { + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, name, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + continue + } + if err != nil { + return err + } + if s.Labels[LabelRelease] == "" { + continue + } + if err := c.kube.CoreV1().Secrets(c.ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !kerr.IsNotFound(err) { + return err + } + c.logf("deleted Secret %s (copy of a certificate)", name) + } + for _, rec := range st.RemovedServices { + svc := &core.Service{ + ObjectMeta: metav1.ObjectMeta{Name: rec.Name, Namespace: c.ns, Labels: rec.Labels, Annotations: rec.Annotations}, + Spec: core.ServiceSpec{Type: rec.Type, Selector: rec.Selector, Ports: withoutNodePorts(rec.Ports)}, + } + if _, err := c.kube.CoreV1().Services(c.ns).Create(ctx, svc, metav1.CreateOptions{}); err != nil && !kerr.IsAlreadyExists(err) { + return fmt.Errorf("failed to recreate the chart's Service %s: %w", rec.Name, err) + } + c.logf("recreated the chart's Service %s", rec.Name) + } + return nil +} + +func withoutNodePorts(in []core.ServicePort) []core.ServicePort { + out := make([]core.ServicePort, len(in)) + for i, p := range in { + p.NodePort = 0 + out[i] = p + } + return out +} + +// fallbackServiceRecord rebuilds the client Service record for a Milvus that +// was adopted by hand: the chart proxies' selector, and the Service's current +// ports with the port names only KubeDB pods define replaced by numbers. +func fallbackServiceRecord(svc core.Service, proxy Unit, t core.PodTemplateSpec) *ServiceRecord { + rec := &ServiceRecord{Name: svc.Name, Selector: proxy.Selector} + named := map[string]bool{} + numbers := map[int32]bool{} + if c := milvusContainer(t); c != nil { + for _, p := range c.Ports { + named[p.Name] = true + numbers[p.ContainerPort] = true + } + } + for _, p := range svc.Spec.Ports { + if p.TargetPort.Type == intstr.String && !named[p.TargetPort.StrVal] { + if !numbers[p.Port] { + continue // a port only the KubeDB pods serve + } + p.TargetPort = intstr.FromInt32(p.Port) + } + rec.Ports = append(rec.Ports, p) + } + return rec +} + +func sleep(ctx context.Context, d time.Duration) { + if d <= 0 { + return + } + select { + case <-ctx.Done(): + case <-time.After(d): + } +} diff --git a/pkg/adopt/milvus_plan.go b/pkg/adopt/milvus_plan.go new file mode 100644 index 000000000..aabbf4228 --- /dev/null +++ b/pkg/adopt/milvus_plan.go @@ -0,0 +1,352 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "fmt" + "os" + "strings" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/yaml" +) + +// planOptions are the inputs of an adoption that discovery cannot find. +type planOptions struct { + sourceOptions + Name string + Version string + AuthSecret string + RootPasswordFile string + EtcdAuthSecret string + TLSIssuer string + TLSClientSecret string + ConfigFile string + RestrictedSCC bool + SkipCredentialCheck bool + // allowExisting accepts a Milvus of that name: run resumes an adoption + // whose Milvus it already created. + allowExisting bool +} + +// adoptPlan is a checked adoption: the findings and, when nothing blocks it, +// the objects to create. +type adoptPlan struct { + Findings findings + Source *Source + State *State + Objects *adoptObjects + // GoverningServices are the chart's Services on names KubeDB needs headless; + // run deletes them before it creates the Milvus. + GoverningServices []core.Service +} + +// governingRoles have a headless governing Service named -. +var governingRoles = []string{roleMixCoord, roleDataNode, roleQueryNode, roleStreamingNode} + +func buildPlan(ctx context.Context, c *cluster, o planOptions) (*adoptPlan, error) { + src, err := discover(ctx, c.kube, c.ns, o.sourceOptions) + if err != nil { + return nil, err + } + p := &adoptPlan{Source: src, Findings: src.Findings} + fs := &p.Findings + if o.ConfigFile != "" { + raw, err := os.ReadFile(o.ConfigFile) + if err != nil { + return nil, err + } + cfg, _, err := mergeConfigFiles(map[string]string{"user.yaml": string(raw)}) + if err != nil { + return nil, err + } + src.Config = cfg + src.ConfigFrom = []string{o.ConfigFile} + p.Findings = withoutConfigError(p.Findings) + } + if fs.hasErrors() { + return p, nil + } + for _, u := range src.Units { + fs.infof("%-13s %s %s, %d replicas%s", u.Role, u.Kind, u.Name, u.Replicas, groupNote(u)) + } + fs.infof("config read from %s", strings.Join(src.ConfigFrom, "; ")) + cfg := src.Config + + // The Milvus is named after the client Service, so KubeDB takes it over. + name := o.Name + switch { + case name == "" && src.Service != nil: + name = src.Service.Name + case name == "": + fs.errorf("no Service selects the proxy pods; name the Milvus with --name") + return p, nil + case src.Service != nil && name != src.Service.Name: + fs.warnf("clients use Service %s; the Milvus %s gets a new Service, and clients must move to it", src.Service.Name, name) + } + if src.Service != nil && src.Service.Name == name { + fs.infof("client Service %s (%s) is taken over: same name, IP and ports", name, src.Service.Spec.ClusterIP) + } + if _, err := c.dyn.Resource(MilvusGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}); err == nil { + if !o.allowExisting { + fs.errorf("a Milvus %s already exists", name) + } + } else if !kerr.IsNotFound(err) { + return nil, err + } + + // Write-ahead log: KubeDB runs Woodpecker on object storage only. + switch wt := walType(cfg); wt { + case "woodpecker": + fs.infof("write-ahead log: woodpecker") + case "": + if c.streamingNodeLogsMention(ctx, src, "woodpecker") { + fs.infof("write-ahead log: woodpecker (from the StreamingNode logs; the config does not set mq.type)") + } else { + fs.errorf("cannot confirm the write-ahead log is Woodpecker: the config sets no mq.type and the StreamingNode logs do not mention it") + } + default: + fs.errorf("write-ahead log is %s; KubeDB runs Woodpecker only, and changing the WAL of a running cluster is a migration", wt) + } + if t := lookupString(cfg, "woodpecker.storage.type"); t != "" && t != "minio" { + fs.errorf("woodpecker.storage.type is %s; KubeDB keeps the write-ahead log on the object storage (minio)", t) + } + + // Metadata. + etcd := etcdFromConfig(cfg) + if len(etcd.Endpoints) == 0 { + fs.errorf("the config sets no etcd.endpoints") + } else { + fs.infof("etcd %s, root path %q (kept)", strings.Join(etcd.Endpoints, ","), etcd.RootPath) + } + if etcd.AuthEnabled { + if o.EtcdAuthSecret == "" { + fs.errorf("the etcd uses authentication; give its root user in a basic-auth Secret with --etcd-auth-secret") + } else if err := c.basicAuthSecretExists(ctx, o.EtcdAuthSecret); err != nil { + fs.errorf("--etcd-auth-secret: %v", err) + } + } + + // Object storage. + osData := objectStorageData(cfg) + if osData["bucketName"] == "" || osData["address"] == "" { + fs.errorf("the config does not set minio.address and minio.bucketName") + } else { + fs.infof("object storage %s, bucket %s, root path %q (kept)", osData["address"], osData["bucketName"], osData["rootPath"]) + } + if osData["accessKeyID"] == "" && !strings.EqualFold(osData["useIAM"], "true") { + fs.warnf("the config has no minio.accessKeyID and minio.useIAM is not true; if the credentials come from environment variables, add them to Secret %s", objectStorageSecretName(name)) + } + + // Version. + version := o.Version + if version == "" { + list, err := c.dyn.Resource(MilvusVersionGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list MilvusVersions: %w", err) + } + if version, err = pickVersion(src.Image, list.Items); err != nil { + fs.errorf("%v", err) + } + } + if version != "" { + fs.infof("image %s -> MilvusVersion %s", src.Image, version) + } + + // Credentials. + in := renderInput{ + Namespace: c.ns, Name: name, Version: version, Release: o.Release, Source: src, + EtcdAuthSecret: o.EtcdAuthSecret, RestrictedSCC: o.RestrictedSCC, + } + if err := c.planTLS(ctx, o, src, &in, fs); err != nil { + return nil, err + } + if !lookupBool(cfg, "common.security.authorizationEnabled") { + in.DisableAuth = true + fs.infof("authentication is off; the Milvus gets spec.disableSecurity: true") + } else if err := c.resolveRootCredentials(ctx, o, src, &in, fs); err != nil { + return nil, err + } + + // Services KubeDB needs headless. + for _, role := range governingRoles { + svc, err := c.kube.CoreV1().Services(c.ns).Get(ctx, name+"-"+role, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + continue + } + if err != nil { + return nil, err + } + if svc.Spec.ClusterIP == core.ClusterIPNone { + continue + } + if belongsToRelease(*svc, src) { + p.GoverningServices = append(p.GoverningServices, *svc) + fs.infof("Service %s is not headless; run deletes it (KubeDB recreates it headless; Milvus does not use it)", svc.Name) + } else { + fs.errorf("Service %s exists, is not headless and does not belong to the release; KubeDB needs that name for a headless Service", svc.Name) + } + } + + if fs.hasErrors() { + return p, nil + } + p.State = newState(in) + if p.Objects, err = render(in, p.State); err != nil { + return nil, err + } + return p, nil +} + +func groupNote(u Unit) string { + if u.Group == "" { + return "" + } + return ", node group " + u.Group +} + +func withoutConfigError(fs findings) findings { + var out findings + for _, f := range fs { + if f.Level == levelError && strings.Contains(f.Msg, "--config-file") { + continue + } + out = append(out, f) + } + return out +} + +// belongsToRelease reports whether a Service selects pods of the release, or +// carries the release's instance label. +func belongsToRelease(svc core.Service, src *Source) bool { + for _, u := range src.Units { + if len(svc.Spec.Selector) > 0 && labels.SelectorFromSet(svc.Spec.Selector).Matches(labels.Set(src.Templates[u.key()].Labels)) { + return true + } + } + for _, u := range src.Units { + if v := u.Selector[labelInstance]; v != "" && svc.Labels[labelInstance] == v { + return true + } + } + return false +} + +func (c *cluster) basicAuthSecretExists(ctx context.Context, name string) error { + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + if len(s.Data[core.BasicAuthUsernameKey]) == 0 || len(s.Data[core.BasicAuthPasswordKey]) == 0 { + return fmt.Errorf("the Secret %s needs the keys %s and %s", name, core.BasicAuthUsernameKey, core.BasicAuthPasswordKey) + } + return nil +} + +// resolveRootCredentials picks the root credentials (an existing Secret, a +// password file, or the chart's defaultRootPassword) and logs in with them. +// Milvus applies a root password only on its first start, so a password that +// does not log in today must not be handed to KubeDB. +func (c *cluster) resolveRootCredentials(ctx context.Context, o planOptions, src *Source, in *renderInput, fs *findings) error { + user, password, from := "root", "", "" + switch { + case o.AuthSecret != "": + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, o.AuthSecret, metav1.GetOptions{}) + if err != nil { + fs.errorf("--auth-secret: %v", err) + return nil + } + user, password = string(s.Data[core.BasicAuthUsernameKey]), string(s.Data[core.BasicAuthPasswordKey]) + if user == "" || password == "" { + fs.errorf("Secret %s needs the keys %s and %s", o.AuthSecret, core.BasicAuthUsernameKey, core.BasicAuthPasswordKey) + return nil + } + in.AuthSecret, from = o.AuthSecret, "Secret "+o.AuthSecret + case o.RootPasswordFile != "": + raw, err := os.ReadFile(o.RootPasswordFile) + if err != nil { + return err + } + password, from = strings.TrimRight(string(raw), "\r\n"), o.RootPasswordFile + default: + password = lookupString(src.Config, "common.security.defaultRootPassword") + if password == "" { + fs.errorf("authentication is on and the config has no defaultRootPassword; give the current root password with --auth-secret or --root-password-file") + return nil + } + from = "common.security.defaultRootPassword" + } + in.RootUser, in.RootPassword = user, password + if o.SkipCredentialCheck { + fs.warnf("root credentials from %s are not checked (--skip-credential-check)", from) + return nil + } + res, detail := c.checkCredentials(ctx, src.firstUnit(roleProxy), restPort(src.Config), user, password) + switch res { + case credentialsAccepted: + fs.infof("root credentials from %s log in to %s", from, detail) + case credentialsRejected: + fs.errorf("root credentials from %s are rejected by the running cluster (%s); Milvus applies a root password only on its first start, so give the current one with --auth-secret or --root-password-file", from, detail) + default: + fs.warnf("root credentials from %s could not be checked: %s", from, detail) + } + return nil +} + +// streamingNodeLogsMention reports whether a StreamingNode's recent log mentions s. +func (c *cluster) streamingNodeLogsMention(ctx context.Context, src *Source, s string) bool { + u := src.firstUnit(roleStreamingNode) + pods, err := c.kube.CoreV1().Pods(c.ns).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(u.Selector).String()}) + if err != nil || len(pods.Items) == 0 { + return false + } + limit := int64(4 << 20) + raw, err := c.kube.CoreV1().Pods(c.ns).GetLogs(pods.Items[0].Name, &core.PodLogOptions{LimitBytes: &limit}).DoRaw(ctx) + return err == nil && strings.Contains(strings.ToLower(string(raw)), s) +} + +// yamlDocs renders the plan's objects as YAML, one document set per file. +func (p *adoptPlan) yamlDocs(redact bool) (map[string]string, error) { + out := map[string]string{} + var secrets []string + for _, s := range p.Objects.Secrets { + cp := s.DeepCopy() + if redact { + for k := range cp.StringData { + cp.StringData[k] = "" + } + } + b, err := yaml.Marshal(cp) + if err != nil { + return nil, err + } + secrets = append(secrets, string(b)) + } + out["00-secrets.yaml"] = strings.Join(secrets, "---\n") + b, err := yaml.Marshal(p.Objects.State) + if err != nil { + return nil, err + } + out["01-adopt-state.yaml"] = string(b) + b, err = yaml.Marshal(p.Objects.Milvus.Object) + if err != nil { + return nil, err + } + out["02-milvus.yaml"] = string(b) + return out, nil +} diff --git a/pkg/adopt/milvus_render.go b/pkg/adopt/milvus_render.go new file mode 100644 index 000000000..5ff7fcb02 --- /dev/null +++ b/pkg/adopt/milvus_render.go @@ -0,0 +1,381 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + MilvusGVR = schema.GroupVersionResource{Group: "kubedb.com", Version: "v1alpha2", Resource: "milvuses"} + MilvusVersionGVR = schema.GroupVersionResource{Group: "catalog.kubedb.com", Version: "v1alpha1", Resource: "milvusversions"} + PetSetGVR = schema.GroupVersionResource{Group: "apps.k8s.appscode.com", Version: "v1", Resource: "petsets"} +) + +const ( + // LabelRelease marks the objects this command creates for one adoption. + LabelRelease = "adopt.kubedb.com/release" + stateKey = "state.json" + + annOrphanOnDelete = "kubedb.com/milvus-orphan-on-delete" + labelRole = "kubedb.com/role" + labelNodeGroup = "kubedb.com/node-group" + labelInstance = "app.kubernetes.io/instance" + labelManagedBy = "app.kubernetes.io/managed-by" + + milvusContainerName = "milvus" +) + +func stateConfigMapName(name string) string { return name + "-adopt" } +func authSecretName(name string) string { return name + "-auth" } +func objectStorageSecretName(name string) string { return name + "-object-storage" } +func serverTLSSecretName(name string) string { return name + "-tls-server" } +func etcdTLSSecretName(name string) string { return name + "-etcd-tls" } +func configSecretName(name string) string { return name + "-adopt-config" } + +// petSetName is the KubeDB PetSet that replaces a Unit. +func petSetName(name string, u Unit) string { + n := name + "-" + u.Role + if u.Group != "" { + n += "-" + u.Group + } + return n +} + +// kubedbSelector selects the KubeDB pods of a Unit. +func kubedbSelector(name string, u Unit) map[string]string { + sel := map[string]string{labelInstance: name, labelRole: u.Role} + if u.Group != "" { + sel[labelNodeGroup] = u.Group + } + return sel +} + +// adoptObjects are the objects an adoption creates, in creation order. +type adoptObjects struct { + Secrets []*core.Secret + State *core.ConfigMap + Milvus *unstructured.Unstructured +} + +// renderInput is everything the objects are built from. +type renderInput struct { + Namespace string + Name string + Version string + Release string + Source *Source + AuthSecret string // existing Secret to use; "" with RootUser set creates one + RootUser string + RootPassword string + DisableAuth bool + EtcdAuthSecret string + EtcdTLS *tlsMaterial + TLS *adoptionTLS + RestrictedSCC bool +} + +func newState(in renderInput) *State { + st := &State{Release: in.Release, Name: in.Name, Phase: phasePlanned, Units: in.Source.Units} + if svc := in.Source.Service; svc != nil && svc.Name == in.Name { + rec := recordService(*svc) + st.Service = &rec + } + return st +} + +func render(in renderInput, st *State) (*adoptObjects, error) { + cfg := in.Source.Config + objs := &adoptObjects{} + lbl := map[string]string{LabelRelease: in.Release} + + authName := in.AuthSecret + if !in.DisableAuth && authName == "" { + authName = authSecretName(in.Name) + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: authName, Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeBasicAuth, + StringData: map[string]string{core.BasicAuthUsernameKey: in.RootUser, core.BasicAuthPasswordKey: in.RootPassword}, + }) + } + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: objectStorageSecretName(in.Name), Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeOpaque, + StringData: objectStorageData(cfg), + }) + carried, err := carriedConfig(cfg) + if err != nil { + return nil, err + } + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: configSecretName(in.Name), Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeOpaque, + StringData: map[string]string{"milvus.yaml": carried}, + }) + + raw, err := json.Marshal(st) + if err != nil { + return nil, err + } + objs.State = &core.ConfigMap{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"}, + ObjectMeta: metav1.ObjectMeta{Name: stateConfigMapName(in.Name), Namespace: in.Namespace, Labels: lbl}, + Data: map[string]string{stateKey: string(raw)}, + } + + // secretFor returns the Secret that holds m: the running one when KubeDB can + // read it as it is, otherwise a copy that rollback removes again. + secretFor := func(m *tlsMaterial, copyName string) string { + if m.Secret != "" { + return m.Secret + } + objs.Secrets = append(objs.Secrets, &core.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: copyName, Namespace: in.Namespace, Labels: lbl}, + Type: core.SecretTypeOpaque, + StringData: m.secretData(), + }) + return copyName + } + + etcd := etcdFromConfig(cfg) + meta := map[string]any{ + "externallyManaged": true, + "endpoints": toAnySlice(etcd.Endpoints), + "rootPath": etcd.RootPath, + } + if in.EtcdAuthSecret != "" { + meta["authSecret"] = map[string]any{"name": in.EtcdAuthSecret} + } + if in.EtcdTLS != nil { + meta["clientTLSSecret"] = map[string]any{"name": secretFor(in.EtcdTLS, etcdTLSSecretName(in.Name))} + } + spec := map[string]any{ + "version": in.Version, + "deletionPolicy": "DoNotTerminate", + "autoOps": map[string]any{"disabled": true}, + "healthChecker": map[string]any{"disableWriteCheck": true}, + "metaStorage": meta, + "objectStorage": map[string]any{"configSecret": map[string]any{"name": objectStorageSecretName(in.Name)}}, + "configuration": map[string]any{"secretName": configSecretName(in.Name)}, + "topology": map[string]any{ + "mode": "Distributed", + "distributed": distributedSpec(in.Source), + }, + } + if in.DisableAuth { + spec["disableSecurity"] = true + } else { + spec["authSecret"] = map[string]any{"name": authName} + } + if in.RestrictedSCC { + spec["restrictedSCC"] = true + } + if in.TLS != nil { + spec["tls"] = tlsSpec(in.TLS, secretFor(in.TLS.Server, serverTLSSecretName(in.Name))) + } + objs.Milvus = &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kubedb.com/v1alpha2", + "kind": "Milvus", + "metadata": map[string]any{ + "name": in.Name, + "namespace": in.Namespace, + "labels": map[string]any{LabelRelease: in.Release}, + "annotations": map[string]any{annOrphanOnDelete: "true"}, + }, + "spec": spec, + }} + return objs, nil +} + +// distributedSpec stages every role at 0 replicas, with the chart's resources, +// node selector, tolerations and GPU count. A role with several workloads +// becomes node groups. +func distributedSpec(src *Source) map[string]any { + out := map[string]any{} + for _, role := range milvusRoles { + var units []Unit + for _, u := range src.Units { + if u.Role == role { + units = append(units, u) + } + } + if len(units) == 0 { + continue + } + if len(units) == 1 && units[0].Group == "" { + node := nodeSpec(src.Templates[units[0].key()]) + node["replicas"] = int64(0) + out[role] = node + continue + } + groups := make([]any, 0, len(units)) + for _, u := range units { + g := nodeSpec(src.Templates[u.key()]) + g["name"] = u.Group + g["replicas"] = int64(0) + groups = append(groups, g) + } + out[role] = map[string]any{"groups": groups} + } + return out +} + +// Resources a KubeDB Milvus container takes from the chart as they are. GPUs +// go to the gpu field; SR-IOV and RDMA devices come from the network field. +func carriedResource(name core.ResourceName) bool { + n := string(name) + return name == core.ResourceCPU || name == core.ResourceMemory || name == core.ResourceEphemeralStorage || + strings.HasPrefix(n, core.ResourceHugePagesPrefix) +} + +func gpuResource(name core.ResourceName) bool { + return strings.HasSuffix(string(name), "/gpu") +} + +func nodeSpec(t core.PodTemplateSpec) map[string]any { + node := map[string]any{} + podSpec := map[string]any{} + c := milvusContainer(t) + if c != nil { + res := map[string]any{} + var gpuName core.ResourceName + var gpuCount int64 + for _, part := range []struct { + key string + list core.ResourceList + }{{"requests", c.Resources.Requests}, {"limits", c.Resources.Limits}} { + m := map[string]any{} + for _, n := range sortedResourceNames(part.list) { + q := part.list[n] + switch { + case carriedResource(n): + m[string(n)] = q.String() + case gpuResource(n): + gpuName, gpuCount = n, q.Value() + } + } + if len(m) > 0 { + res[part.key] = m + } + } + if len(res) > 0 { + podSpec["containers"] = []any{map[string]any{"name": milvusContainerName, "resources": res}} + } + if gpuCount > 0 { + gpu := map[string]any{"count": gpuCount} + if gpuName != "nvidia.com/gpu" { + gpu["resourceName"] = string(gpuName) + } + node["gpu"] = gpu + } + } + if len(t.Spec.NodeSelector) > 0 { + ns := map[string]any{} + for k, v := range t.Spec.NodeSelector { + ns[k] = v + } + podSpec["nodeSelector"] = ns + } + if len(t.Spec.Tolerations) > 0 { + tols := make([]any, 0, len(t.Spec.Tolerations)) + for _, tol := range t.Spec.Tolerations { + b, _ := json.Marshal(tol) + var m map[string]any + _ = json.Unmarshal(b, &m) + tols = append(tols, m) + } + podSpec["tolerations"] = tols + } + if len(podSpec) > 0 { + node["podTemplate"] = map[string]any{"spec": podSpec} + } + return node +} + +func sortedResourceNames(l core.ResourceList) []core.ResourceName { + out := make([]core.ResourceName, 0, len(l)) + for n := range l { + out = append(out, n) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +func toAnySlice(in []string) []any { + out := make([]any, 0, len(in)) + for _, s := range in { + out = append(out, s) + } + return out +} + +// imageVersion splits a Milvus image into its version and whether it is a GPU +// build: "milvusdb/milvus:v2.6.15-gpu" -> ("2.6.15", true). +func imageVersion(image string) (string, bool, error) { + if strings.Contains(image, "@") { + return "", false, fmt.Errorf("image %s is pinned by digest; pass the version with --version", image) + } + i := strings.LastIndex(image, ":") + if i < 0 || strings.Contains(image[i:], "/") { + return "", false, fmt.Errorf("image %s has no tag; pass the version with --version", image) + } + tag := strings.TrimPrefix(image[i+1:], "v") + gpu := strings.Contains(tag, "-gpu") + ver, _, _ := strings.Cut(tag, "-") + return ver, gpu, nil +} + +// pickVersion returns the MilvusVersion for an image: same version, same GPU +// support, not deprecated. +func pickVersion(image string, versions []unstructured.Unstructured) (string, error) { + ver, gpu, err := imageVersion(image) + if err != nil { + return "", err + } + var names []string + for _, v := range versions { + sv, _, _ := unstructured.NestedString(v.Object, "spec", "version") + g, _, _ := unstructured.NestedBool(v.Object, "spec", "db", "gpu", "supported") + dep, _, _ := unstructured.NestedBool(v.Object, "spec", "deprecated") + if sv == ver && g == gpu && !dep { + names = append(names, v.GetName()) + } + } + if len(names) == 0 { + build := "CPU" + if gpu { + build = "GPU" + } + return "", fmt.Errorf("no MilvusVersion for %s (%s build of %s); install the KubeDB catalog that has it, or pass --version", image, build, ver) + } + sort.Strings(names) + for _, n := range names { // prefer the canonical name + if n == ver || n == ver+"-gpu" { + return n, nil + } + } + return names[0], nil +} diff --git a/pkg/adopt/milvus_source.go b/pkg/adopt/milvus_source.go new file mode 100644 index 000000000..f38c86ca9 --- /dev/null +++ b/pkg/adopt/milvus_source.go @@ -0,0 +1,543 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "context" + "fmt" + "path" + "regexp" + "sort" + "strings" + + apps "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" +) + +// Distributed Milvus 2.6 roles, in the order the CR lists them. +const ( + roleProxy = "proxy" + roleMixCoord = "mixcoord" + roleDataNode = "datanode" + roleQueryNode = "querynode" + roleStreamingNode = "streamingnode" +) + +var milvusRoles = []string{roleProxy, roleMixCoord, roleDataNode, roleQueryNode, roleStreamingNode} + +// Components of Milvus before 2.6 (separate coordinators, IndexNode) and the +// Standalone mode. A cluster with any of them is not a 2.6 Distributed cluster. +var unsupportedComponents = []string{"rootcoord", "querycoord", "datacoord", "indexcoord", "indexnode", "standalone"} + +// Unit is one workload of the running cluster and the KubeDB PetSet that +// replaces it: a role, or one node group of a role when the role runs as +// several workloads (for example QueryNodes on two GPU pools). +type Unit struct { + Role string `json:"role"` + Group string `json:"group,omitempty"` + Kind string `json:"kind"` + Name string `json:"name"` + Replicas int32 `json:"replicas"` + Selector map[string]string `json:"selector"` +} + +func (u Unit) key() string { + if u.Group == "" { + return u.Role + } + return u.Role + "/" + u.Group +} + +// ServiceRecord is the client Service as the chart left it, so a rollback can +// give it back unchanged. +type ServiceRecord struct { + Name string `json:"name"` + Type core.ServiceType `json:"type,omitempty"` + Selector map[string]string `json:"selector"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Ports []core.ServicePort `json:"ports"` +} + +// Phases of an adoption, recorded in the state ConfigMap. +const ( + phasePlanned = "Planned" + phaseHandingOver = "HandingOver" + phaseAdopted = "Adopted" + phaseRolledBack = "RolledBack" +) + +// State is the record of one adoption, kept in the ConfigMap -adopt so +// that run can resume and rollback knows what to restore. +type State struct { + Release string `json:"release"` + Name string `json:"name"` + Method string `json:"method,omitempty"` + Phase string `json:"phase"` + Units []Unit `json:"units"` + Service *ServiceRecord `json:"service,omitempty"` + // RemovedServices are chart Services deleted because KubeDB needs their + // names for headless Services; a rollback recreates them. + RemovedServices []ServiceRecord `json:"removedServices,omitempty"` + Done []string `json:"done,omitempty"` +} + +func (s *State) unitsOf(role string) []Unit { + var out []Unit + for _, u := range s.Units { + if u.Role == role { + out = append(out, u) + } + } + return out +} + +func (s *State) isDone(u Unit) bool { return contains(s.Done, u.key()) } + +func (s *State) markDone(u Unit) { + if !s.isDone(u) { + s.Done = append(s.Done, u.key()) + } +} + +func (s *State) unmarkDone(u Unit) { + out := s.Done[:0] + for _, k := range s.Done { + if k != u.key() { + out = append(out, k) + } + } + s.Done = out +} + +// Source is what discovery found about the running cluster. +type Source struct { + Units []Unit + Templates map[string]core.PodTemplateSpec // by Unit.key() + Config map[string]any + ConfigFrom []string + Image string + Service *core.Service + Findings findings +} + +type workloadRef struct { + Kind string + Name string + Labels map[string]string + Replicas int32 + Selector map[string]string + Template core.PodTemplateSpec +} + +func refFromDeployment(d apps.Deployment) workloadRef { + r := workloadRef{Kind: "Deployment", Name: d.Name, Labels: d.Labels, Replicas: 1, Template: d.Spec.Template} + if d.Spec.Replicas != nil { + r.Replicas = *d.Spec.Replicas + } + if d.Spec.Selector != nil { + r.Selector = d.Spec.Selector.MatchLabels + } + return r +} + +func refFromStatefulSet(s apps.StatefulSet) workloadRef { + r := workloadRef{Kind: "StatefulSet", Name: s.Name, Labels: s.Labels, Replicas: 1, Template: s.Spec.Template} + if s.Spec.Replicas != nil { + r.Replicas = *s.Spec.Replicas + } + if s.Spec.Selector != nil { + r.Selector = s.Spec.Selector.MatchLabels + } + return r +} + +// sourceOptions select the running cluster. +type sourceOptions struct { + Release string + Selector string // label selector of the release's workloads + RoleLabel string // label that names a workload's role + Workloads []string // explicit role[/group]=Kind/name, instead of the selector +} + +func (o sourceOptions) selector() string { + if o.Selector != "" { + return o.Selector + } + return "app.kubernetes.io/instance=" + o.Release +} + +// assignUnits turns the release's workloads into Units. Workloads of one role +// become node groups named after the workload. +func assignUnits(refs []workloadRef, o sourceOptions) ([]Unit, map[string]core.PodTemplateSpec, findings, error) { + var fs findings + explicit := map[string]string{} // Kind/name -> role[/group] + for _, w := range o.Workloads { + k, v, ok := strings.Cut(w, "=") + if !ok { + return nil, nil, nil, fmt.Errorf("--workload %q: want role[/group]=Kind/name", w) + } + explicit[strings.ToLower(v)] = k + } + byRole := map[string][]workloadRef{} + groupOf := map[string]string{} + for _, r := range refs { + var role, group string + if len(explicit) > 0 { + rg, ok := explicit[strings.ToLower(r.Kind+"/"+r.Name)] + if !ok { + continue + } + role, group, _ = strings.Cut(rg, "/") + } else { + role = r.Template.Labels[o.RoleLabel] + if role == "" { + role = r.Labels[o.RoleLabel] + } + } + switch { + case role == "": + continue // etcd, MinIO and other workloads of the release + case contains(unsupportedComponents, role): + fs.errorf("%s %s is a %q component: only a Distributed Milvus 2.6 (MixCoord, Proxy, DataNode, QueryNode, StreamingNode) can be adopted", r.Kind, r.Name, role) + continue + case !contains(milvusRoles, role): + fs.infof("%s %s has %s=%s, not a Milvus role; ignored", r.Kind, r.Name, o.RoleLabel, role) + continue + } + byRole[role] = append(byRole[role], r) + if group != "" { + groupOf[r.Kind+"/"+r.Name] = group + } + } + if len(explicit) > 0 { + found := 0 + for _, rs := range byRole { + found += len(rs) + } + if found != len(explicit) { + return nil, nil, nil, fmt.Errorf("found %d of the %d workloads given with --workload", found, len(explicit)) + } + } + + var units []Unit + templates := map[string]core.PodTemplateSpec{} + for _, role := range milvusRoles { + rs := byRole[role] + if len(rs) == 0 { + fs.errorf("no workload found for the %s role (selector %q, role label %q)", role, o.selector(), o.RoleLabel) + continue + } + sort.Slice(rs, func(i, j int) bool { return rs[i].Name < rs[j].Name }) + used := map[string]bool{} + for _, r := range rs { + u := Unit{Role: role, Kind: r.Kind, Name: r.Name, Replicas: r.Replicas, Selector: r.Selector} + if len(rs) > 1 { + g := groupOf[r.Kind+"/"+r.Name] + if g == "" { + g = groupName(r.Name, role) + } + if used[g] { + return nil, nil, nil, fmt.Errorf("workloads of role %s map to the same group %q; name them with --workload", role, g) + } + used[g] = true + u.Group = g + } + if len(u.Selector) == 0 { + return nil, nil, nil, fmt.Errorf("%s %s has no matchLabels selector", r.Kind, r.Name) + } + if u.Replicas == 0 { + fs.errorf("%s %s runs 0 replicas: nothing to hand over. If an adoption already moved it, run \"status\"", r.Kind, r.Name) + } + units = append(units, u) + templates[u.key()] = r.Template + } + } + return units, templates, fs, nil +} + +var dnsUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// groupName derives a node group name from a workload name, e.g. +// "milvus-querynode-h200" -> "h200". +func groupName(workload, role string) string { + n := strings.ToLower(workload) + if i := strings.LastIndex(n, role+"-"); i >= 0 && i+len(role)+1 < len(n) { + n = n[i+len(role)+1:] + } + n = strings.Trim(dnsUnsafe.ReplaceAllString(n, "-"), "-") + if len(n) > 20 { + n = strings.Trim(n[len(n)-20:], "-") + } + return n +} + +// milvusContainer returns the Milvus container of a pod template. +func milvusContainer(t core.PodTemplateSpec) *core.Container { + for i := range t.Spec.Containers { + c := &t.Spec.Containers[i] + if strings.Contains(c.Image, "milvus") { + return c + } + } + if len(t.Spec.Containers) > 0 { + return &t.Spec.Containers[0] + } + return nil +} + +// configMountDir is where Milvus reads its configuration files. +const configMountDir = "/milvus/configs" + +// configSource is one configuration file mounted into the Milvus container. +type configSource struct { + File string // milvus.yaml, default.yaml, user.yaml + Kind string // ConfigMap or Secret + Name string + Key string // "" for every key of a directory mount + IsDir bool + Volume string +} + +// configSources lists the configuration files a pod template mounts. +func configSources(t core.PodTemplateSpec) []configSource { + c := milvusContainer(t) + if c == nil { + return nil + } + vols := map[string]core.Volume{} + for _, v := range t.Spec.Volumes { + vols[v.Name] = v + } + var out []configSource + for _, m := range c.VolumeMounts { + v, ok := vols[m.Name] + if !ok { + continue + } + src := configSource{Volume: m.Name} + switch { + case v.ConfigMap != nil: + src.Kind, src.Name = "ConfigMap", v.ConfigMap.Name + case v.Secret != nil: + src.Kind, src.Name = "Secret", v.Secret.SecretName + default: + continue + } + mp := strings.TrimSuffix(m.MountPath, "/") + switch { + case mp == configMountDir: + src.IsDir = true + case path.Dir(mp) == configMountDir: + src.File = path.Base(mp) + src.Key = m.SubPath + if src.Key == "" { + continue // a directory mounted over a file name; not a config file + } + default: + continue + } + out = append(out, src) + } + return out +} + +// discover reads the release's workloads, configuration, image and client +// Service. Problems that block an adoption are returned as error findings. +func discover(ctx context.Context, kc kubernetes.Interface, ns string, o sourceOptions) (*Source, error) { + sel := o.selector() + if _, err := labels.Parse(sel); err != nil { + return nil, fmt.Errorf("invalid selector %q: %w", sel, err) + } + lo := metav1.ListOptions{LabelSelector: sel} + if len(o.Workloads) > 0 { + lo = metav1.ListOptions{} + } + var refs []workloadRef + deps, err := kc.AppsV1().Deployments(ns).List(ctx, lo) + if err != nil { + return nil, fmt.Errorf("failed to list Deployments: %w", err) + } + for _, d := range deps.Items { + refs = append(refs, refFromDeployment(d)) + } + sts, err := kc.AppsV1().StatefulSets(ns).List(ctx, lo) + if err != nil { + return nil, fmt.Errorf("failed to list StatefulSets: %w", err) + } + for _, s := range sts.Items { + refs = append(refs, refFromStatefulSet(s)) + } + units, templates, fs, err := assignUnits(refs, o) + if err != nil { + return nil, err + } + src := &Source{Units: units, Templates: templates, Findings: fs} + if src.Findings.hasErrors() { + return src, nil + } + + proxy := templates[src.firstUnit(roleProxy).key()] + if c := milvusContainer(proxy); c != nil { + src.Image = c.Image + } + if err := src.readConfig(ctx, kc, ns, proxy); err != nil { + return nil, err + } + if err := src.findClientService(ctx, kc, ns, proxy); err != nil { + return nil, err + } + src.checkTemplates() + return src, nil +} + +func (s *Source) firstUnit(role string) Unit { + for _, u := range s.Units { + if u.Role == role { + return u + } + } + return Unit{} +} + +func (s *Source) readConfig(ctx context.Context, kc kubernetes.Interface, ns string, t core.PodTemplateSpec) error { + files := map[string]string{} + for _, src := range configSources(t) { + var data map[string]string + switch src.Kind { + case "ConfigMap": + cm, err := kc.CoreV1().ConfigMaps(ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read the Milvus config ConfigMap %s: %w", src.Name, err) + } + data = cm.Data + case "Secret": + sec, err := kc.CoreV1().Secrets(ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read the Milvus config Secret %s: %w", src.Name, err) + } + data = map[string]string{} + for k, v := range sec.Data { + data[k] = string(v) + } + } + if src.IsDir { + for k, v := range data { + if strings.HasSuffix(k, ".yaml") { + files[k] = v + s.ConfigFrom = append(s.ConfigFrom, fmt.Sprintf("%s %s key %s", src.Kind, src.Name, k)) + } + } + continue + } + v, ok := data[src.Key] + if !ok { + return fmt.Errorf("%s %s has no key %q mounted as %s", src.Kind, src.Name, src.Key, src.File) + } + files[src.File] = v + s.ConfigFrom = append(s.ConfigFrom, fmt.Sprintf("%s %s key %s as %s", src.Kind, src.Name, src.Key, src.File)) + } + if len(files) == 0 { + s.Findings.errorf("found no configuration file mounted in %s; pass the effective milvus.yaml with --config-file", configMountDir) + s.Config = map[string]any{} + return nil + } + cfg, unknown, err := mergeConfigFiles(files) + if err != nil { + return err + } + for _, f := range unknown { + s.Findings.warnf("%s/%s is mounted but Milvus does not read it; not carried over", configMountDir, f) + } + s.Config = cfg + return nil +} + +// findClientService finds the Service whose selector picks the proxy pods. +func (s *Source) findClientService(ctx context.Context, kc kubernetes.Interface, ns string, proxy core.PodTemplateSpec) error { + svcs, err := kc.CoreV1().Services(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list Services: %w", err) + } + var matches []core.Service + for _, svc := range svcs.Items { + if len(svc.Spec.Selector) == 0 || svc.Spec.ClusterIP == core.ClusterIPNone { + continue + } + if labels.SelectorFromSet(svc.Spec.Selector).Matches(labels.Set(proxy.Labels)) { + matches = append(matches, svc) + } + } + sort.Slice(matches, func(i, j int) bool { + pi, pj := hasPort(matches[i], 19530), hasPort(matches[j], 19530) + if pi != pj { + return pi + } + return matches[i].Name < matches[j].Name + }) + switch len(matches) { + case 0: + s.Findings.warnf("no Service selects the proxy pods; clients will need the new Service ") + case 1: + default: + var names []string + for _, m := range matches[1:] { + names = append(names, m.Name) + } + s.Findings.warnf("Services %s also select the proxy pods; they keep pointing at the chart proxies and stop working once those are gone", + strings.Join(names, ", ")) + } + if len(matches) > 0 { + s.Service = &matches[0] + } + return nil +} + +func hasPort(svc core.Service, port int32) bool { + for _, p := range svc.Spec.Ports { + if p.Port == port { + return true + } + } + return false +} + +// checkTemplates reports pod settings that the Milvus object does not carry over +// by itself. +func (s *Source) checkTemplates() { + for _, u := range s.Units { + t := s.Templates[u.key()] + id := u.Kind + " " + u.Name + if t.Spec.Affinity != nil { + s.Findings.warnf("%s sets affinity, which is not carried over; use a PlacementPolicy (spec.topology.distributed.%s.podTemplate.spec.podPlacementPolicy)", id, u.Role) + } + if n := t.Annotations["k8s.v1.cni.cncf.io/networks"]; n != "" { + s.Findings.warnf("%s attaches secondary networks (%s); set spec.topology.distributed.%s.network before handing it over (see the GPU and SR-IOV guide)", id, n, u.Role) + } + for _, ic := range t.Spec.InitContainers { + s.Findings.warnf("%s has init container %q, which is not carried over", id, ic.Name) + } + if c := milvusContainer(t); c != nil { + for _, e := range c.Env { + s.Findings.warnf("%s sets environment variable %s; Milvus may read it as configuration, and it is not carried over", id, e.Name) + } + if c.Image != s.Image { + s.Findings.errorf("%s runs image %s, the proxy runs %s; adopt a cluster that runs one version", id, c.Image, s.Image) + } + } + } +} diff --git a/pkg/adopt/milvus_test.go b/pkg/adopt/milvus_test.go new file mode 100644 index 000000000..8d65656bb --- /dev/null +++ b/pkg/adopt/milvus_test.go @@ -0,0 +1,663 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" +) + +// The ConfigMap the upstream Milvus Helm chart (5.0.x) renders, trimmed. +const chartDefaultYAML = ` +etcd: + endpoints: + - mv-etcd-0.mv-etcd-headless.ns.svc.cluster.local:2379 + - mv-etcd-1.mv-etcd-headless.ns.svc.cluster.local:2379 +minio: + address: minio.storage.svc + port: 9000 + accessKeyID: ak + secretAccessKey: sk + useSSL: false + bucketName: milvus-data + rootPath: prod + useIAM: false + cloudProvider: aws + iamEndpoint: + region: +mq: + type: woodpecker +messageQueue: woodpecker +proxy: + port: 19530 +woodpecker: + storage: + type: minio + rootPath: /woodpecker/data +` + +const chartUserYAML = ` +common: + security: + authorizationEnabled: true + defaultRootPassword: S3cret +minio: + rootPath: prod-override +` + +func chartConfig(t *testing.T) map[string]any { + t.Helper() + cfg, unknown, err := mergeConfigFiles(map[string]string{"default.yaml": chartDefaultYAML, "user.yaml": chartUserYAML}) + if err != nil { + t.Fatal(err) + } + if len(unknown) != 0 { + t.Fatalf("unexpected unknown files %v", unknown) + } + return cfg +} + +func TestMergeConfigFilesUserOverridesDefault(t *testing.T) { + cfg := chartConfig(t) + if got := lookupString(cfg, "minio.rootPath"); got != "prod-override" { + t.Fatalf("user.yaml must override default.yaml: rootPath = %q", got) + } + if got := lookupString(cfg, "minio.bucketName"); got != "milvus-data" { + t.Fatalf("keys only in default.yaml are kept: bucketName = %q", got) + } + _, unknown, err := mergeConfigFiles(map[string]string{"user.yaml": "a: 1", "extra.yaml": "b: 2"}) + if err != nil || !reflect.DeepEqual(unknown, []string{"extra.yaml"}) { + t.Fatalf("unknown files = %v, %v", unknown, err) + } +} + +func TestEtcdFromConfig(t *testing.T) { + s := etcdFromConfig(chartConfig(t)) + want := []string{ + "http://mv-etcd-0.mv-etcd-headless.ns.svc.cluster.local:2379", + "http://mv-etcd-1.mv-etcd-headless.ns.svc.cluster.local:2379", + } + if !reflect.DeepEqual(s.Endpoints, want) { + t.Fatalf("endpoints = %v", s.Endpoints) + } + if s.RootPath != "by-dev" { + t.Fatalf("an unset etcd.rootPath is the Milvus default by-dev, got %q", s.RootPath) + } + s = etcdFromConfig(map[string]any{"etcd": map[string]any{ + "endpoints": "a:2379, b:2379", "rootPath": "custom", + "ssl": map[string]any{"enabled": true}, "auth": map[string]any{"enabled": "true"}, + }}) + if !reflect.DeepEqual(s.Endpoints, []string{"https://a:2379", "https://b:2379"}) || s.RootPath != "custom" || !s.AuthEnabled || !s.TLSEnabled { + t.Fatalf("got %+v", s) + } +} + +func TestObjectStorageData(t *testing.T) { + got := objectStorageData(chartConfig(t)) + want := map[string]string{ + "address": "minio.storage.svc", "port": "9000", "accessKeyID": "ak", "secretAccessKey": "sk", + "useSSL": "false", "bucketName": "milvus-data", "rootPath": "prod-override", "useIAM": "false", "cloudProvider": "aws", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v", got) + } +} + +func TestCarriedConfig(t *testing.T) { + raw, err := carriedConfig(chartConfig(t)) + if err != nil { + t.Fatal(err) + } + var cfg map[string]any + if err := yaml.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatal(err) + } + if _, ok := cfg["etcd"]; ok { + t.Fatal("the etcd section must be dropped: the Milvus object sets it") + } + if v, ok := lookup(cfg, "proxy.http.port"); !ok || v != "" { + t.Fatalf("an unset proxy.http.port must be pinned to \"\" to keep REST on the gRPC port, got %v %v", v, ok) + } + if lookupString(cfg, "woodpecker.storage.rootPath") != "/woodpecker/data" { + t.Fatal("the woodpecker section must be kept as it runs") + } + raw, _ = carriedConfig(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": nil}}}) + _ = yaml.Unmarshal([]byte(raw), &cfg) + if v, ok := lookup(cfg, "proxy.http.port"); !ok || v != "" { + t.Fatalf("a null proxy.http.port must be pinned to \"\" as well, got %v %v", v, ok) + } + raw, _ = carriedConfig(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": float64(8080)}}}) + if !strings.Contains(raw, "port: 8080") { + t.Fatalf("a set proxy.http.port is kept: %s", raw) + } +} + +func TestWalType(t *testing.T) { + for _, tc := range []struct { + cfg map[string]any + want string + }{ + {map[string]any{"mq": map[string]any{"type": "woodpecker"}}, "woodpecker"}, + {map[string]any{"mq": map[string]any{"type": "default"}, "messageQueue": "pulsar"}, "pulsar"}, + {map[string]any{"messageQueue": "kafka"}, "kafka"}, + {map[string]any{}, ""}, + } { + if got := walType(tc.cfg); got != tc.want { + t.Errorf("walType(%v) = %q, want %q", tc.cfg, got, tc.want) + } + } +} + +func TestConfigMismatches(t *testing.T) { + running := chartConfig(t) + carried, _ := carriedConfig(running) + var rendered map[string]any + _ = yaml.Unmarshal([]byte(carried), &rendered) + rendered["etcd"] = map[string]any{"rootPath": "by-dev"} + rendered["minio"].(map[string]any)["port"] = float64(9000) // the operator writes the port as a number + if d := configMismatches(running, rendered); len(d) != 0 { + t.Fatalf("identical settings reported as different: %v", d) + } + + rendered["etcd"] = map[string]any{"rootPath": "mv-etcd"} + rendered["minio"].(map[string]any)["rootPath"] = "files" + rendered["minio"].(map[string]any)["secretAccessKey"] = "other" + d := configMismatches(running, rendered) + if len(d) != 3 { + t.Fatalf("want 3 differences, got %v", d) + } + for _, s := range d { + if strings.Contains(s, "other") || strings.Contains(s, `"sk"`) { + t.Fatalf("a credential leaked into the report: %s", s) + } + } +} + +func ref(name, role string, replicas int32) workloadRef { + sel := map[string]string{"app.kubernetes.io/instance": "rel", "component": role} + t := core.PodTemplateSpec{Spec: core.PodSpec{Containers: []core.Container{{Name: role, Image: "milvusdb/milvus:v2.6.15"}}}} + t.Labels = sel + return workloadRef{Kind: "Deployment", Name: name, Replicas: replicas, Selector: sel, Template: t} +} + +func allRoles() []workloadRef { + return []workloadRef{ + ref("rel-milvus-proxy", "proxy", 3), + ref("rel-milvus-mixcoord", "mixcoord", 2), + ref("rel-milvus-datanode", "datanode", 2), + ref("rel-milvus-querynode", "querynode", 6), + ref("rel-milvus-streamingnode", "streamingnode", 2), + {Kind: "StatefulSet", Name: "rel-etcd", Replicas: 3, Selector: map[string]string{"app": "etcd"}}, + } +} + +func TestAssignUnits(t *testing.T) { + o := sourceOptions{Release: "rel", RoleLabel: "component"} + units, tmpls, fs, err := assignUnits(allRoles(), o) + if err != nil || fs.hasErrors() { + t.Fatalf("err %v, findings %v", err, fs) + } + if len(units) != 5 || len(tmpls) != 5 { + t.Fatalf("want the 5 roles and not etcd, got %v", units) + } + for _, u := range units { + if u.Group != "" { + t.Fatalf("one workload per role needs no group: %+v", u) + } + } + + // Two QueryNode pools become node groups named after the workloads. + refs := allRoles() + refs[3] = ref("rel-milvus-querynode-l40s", "querynode", 4) + refs = append(refs, ref("rel-milvus-querynode-h200", "querynode", 2)) + units, _, fs, err = assignUnits(refs, o) + if err != nil || fs.hasErrors() { + t.Fatalf("err %v, findings %v", err, fs) + } + var groups []string + for _, u := range units { + if u.Role == roleQueryNode { + groups = append(groups, u.Group) + } + } + if !reflect.DeepEqual(groups, []string{"h200", "l40s"}) { + t.Fatalf("groups = %v", groups) + } + + // A pre-2.6 component blocks the adoption; a missing role does too. + refs = append(allRoles()[1:], ref("rel-milvus-rootcoord", "rootcoord", 1)) + _, _, fs, _ = assignUnits(refs, o) + if !fs.hasErrors() || !strings.Contains(findingText(fs), "rootcoord") || !strings.Contains(findingText(fs), "proxy role") { + t.Fatalf("findings %v", fs) + } + + // A role at 0 replicas has nothing to hand over. + refs = allRoles() + refs[0].Replicas = 0 + _, _, fs, _ = assignUnits(refs, o) + if !fs.hasErrors() { + t.Fatal("a workload at 0 replicas must be reported") + } + + // Explicit mapping, for charts without a role label. + refs = allRoles() + for i := range refs { + delete(refs[i].Template.Labels, "component") + } + o.Workloads = []string{ + "proxy=Deployment/rel-milvus-proxy", "mixcoord=Deployment/rel-milvus-mixcoord", "datanode=Deployment/rel-milvus-datanode", + "querynode/gpu=Deployment/rel-milvus-querynode", "streamingnode=Deployment/rel-milvus-streamingnode", + } + units, _, fs, err = assignUnits(refs, o) + if err != nil || fs.hasErrors() || len(units) != 5 { + t.Fatalf("explicit mapping: units %v, findings %v, err %v", units, fs, err) + } +} + +func findingText(fs findings) string { + var b strings.Builder + for _, f := range fs { + b.WriteString(f.Msg + "\n") + } + return b.String() +} + +func TestGroupName(t *testing.T) { + for in, want := range map[string]string{ + "milvus-querynode-h200": "h200", + "milvus-querynode-L40S_gpu": "l40s-gpu", + "qn-pool-a": "qn-pool-a", + } { + if got := groupName(in, roleQueryNode); got != want { + t.Errorf("groupName(%q) = %q, want %q", in, got, want) + } + } +} + +func TestConfigSources(t *testing.T) { + tpl := core.PodTemplateSpec{Spec: core.PodSpec{ + Containers: []core.Container{{Name: "proxy", Image: "milvusdb/milvus:v2.6.15", VolumeMounts: []core.VolumeMount{ + {Name: "milvus-config", MountPath: "/milvus/configs/default.yaml", SubPath: "default.yaml"}, + {Name: "milvus-config", MountPath: "/milvus/configs/user.yaml", SubPath: "user.yaml"}, + {Name: "tools", MountPath: "/milvus/tools"}, + }}}, + Volumes: []core.Volume{ + {Name: "milvus-config", VolumeSource: core.VolumeSource{ConfigMap: &core.ConfigMapVolumeSource{LocalObjectReference: core.LocalObjectReference{Name: "rel-milvus"}}}}, + {Name: "tools", VolumeSource: core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{}}}, + }, + }} + got := configSources(tpl) + if len(got) != 2 || got[0].File != "default.yaml" || got[1].Key != "user.yaml" || got[0].Name != "rel-milvus" { + t.Fatalf("got %+v", got) + } +} + +func TestImageVersionAndPickVersion(t *testing.T) { + v, gpu, err := imageVersion("milvusdb/milvus:v2.6.15-gpu") + if err != nil || v != "2.6.15" || !gpu { + t.Fatalf("got %s %v %v", v, gpu, err) + } + if _, _, err := imageVersion("registry:5000/milvus"); err == nil { + t.Fatal("an image without a tag must fail") + } + mk := func(name, ver string, gpu, deprecated bool) unstructured.Unstructured { + u := unstructured.Unstructured{Object: map[string]any{"spec": map[string]any{ + "version": ver, "deprecated": deprecated, "db": map[string]any{"gpu": map[string]any{"supported": gpu}}, + }}} + u.SetName(name) + return u + } + versions := []unstructured.Unstructured{ + mk("2.6.15", "2.6.15", false, false), mk("2.6.15-gpu", "2.6.15", true, false), + mk("2.6.11", "2.6.11", false, false), mk("2.6.9-old", "2.6.9", false, true), + } + for image, want := range map[string]string{ + "milvusdb/milvus:v2.6.15": "2.6.15", + "milvusdb/milvus:v2.6.15-gpu": "2.6.15-gpu", + } { + if got, err := pickVersion(image, versions); err != nil || got != want { + t.Errorf("pickVersion(%s) = %s, %v; want %s", image, got, err, want) + } + } + for _, image := range []string{"milvusdb/milvus:v2.6.9", "milvusdb/milvus:v2.6.11-gpu"} { + if _, err := pickVersion(image, versions); err == nil { + t.Errorf("pickVersion(%s): a deprecated or missing build must fail", image) + } + } +} + +func TestRenderStagedMilvus(t *testing.T) { + refs := allRoles() + qn := &refs[3].Template.Spec.Containers[0] + qn.Resources = core.ResourceRequirements{ + Requests: core.ResourceList{core.ResourceCPU: resource.MustParse("8"), core.ResourceMemory: resource.MustParse("144Gi")}, + Limits: core.ResourceList{ + core.ResourceMemory: resource.MustParse("144Gi"), "nvidia.com/gpu": resource.MustParse("1"), + "intel.com/sriov_rdma": resource.MustParse("1"), + }, + } + refs[3].Template.Spec.NodeSelector = map[string]string{"pool": "gpu"} + refs[3].Template.Spec.Tolerations = []core.Toleration{{Key: "nvidia.com/gpu", Operator: core.TolerationOpExists, Effect: core.TaintEffectNoSchedule}} + units, tmpls, _, _ := assignUnits(refs, sourceOptions{Release: "rel", RoleLabel: "component"}) + svc := &core.Service{Spec: core.ServiceSpec{Selector: map[string]string{"component": "proxy"}, Ports: []core.ServicePort{{Name: "milvus", Port: 19530}}}} + svc.Name = "rel-milvus" + src := &Source{Units: units, Templates: tmpls, Config: chartConfig(t), Service: svc} + in := renderInput{ + Namespace: "ns", Name: "rel-milvus", Version: "2.6.15-gpu", Release: "rel", Source: src, + RootUser: "root", RootPassword: "S3cret", + } + st := newState(in) + if st.Service == nil || st.Service.Name != "rel-milvus" || len(st.Service.Ports) != 1 { + t.Fatalf("the client Service must be recorded for a rollback: %+v", st.Service) + } + objs, err := render(in, st) + if err != nil { + t.Fatal(err) + } + if len(objs.Secrets) != 3 || objs.Secrets[0].Name != "rel-milvus-auth" || objs.Secrets[0].StringData["password"] != "S3cret" { + t.Fatalf("secrets %v", objs.Secrets) + } + m := objs.Milvus.Object + str := func(path ...string) string { s, _, _ := unstructured.NestedString(m, path...); return s } + if str("spec", "metaStorage", "rootPath") != "by-dev" || str("spec", "authSecret", "name") != "rel-milvus-auth" || + str("spec", "deletionPolicy") != "DoNotTerminate" || str("metadata", "annotations", annOrphanOnDelete) != "true" { + t.Fatalf("Milvus %v", m) + } + if b, _, _ := unstructured.NestedBool(m, "spec", "metaStorage", "externallyManaged"); !b { + t.Fatal("the etcd must be externally managed") + } + for _, role := range milvusRoles { + n, found, _ := unstructured.NestedInt64(m, "spec", "topology", "distributed", role, "replicas") + if !found || n != 0 { + t.Fatalf("%s must be staged at 0 replicas, got %d %v", role, n, found) + } + } + q, _, _ := unstructured.NestedMap(m, "spec", "topology", "distributed", "querynode") + if c, _, _ := unstructured.NestedInt64(q, "gpu", "count"); c != 1 { + t.Fatalf("GPU count not carried: %v", q) + } + ctrs, _, _ := unstructured.NestedSlice(q, "podTemplate", "spec", "containers") + lim := ctrs[0].(map[string]any)["resources"].(map[string]any)["limits"].(map[string]any) + if lim["memory"] != "144Gi" || lim["nvidia.com/gpu"] != nil || lim["intel.com/sriov_rdma"] != nil { + t.Fatalf("limits %v: memory is carried, GPU and SR-IOV devices are not", lim) + } + if ns, _, _ := unstructured.NestedStringMap(q, "podTemplate", "spec", "nodeSelector"); ns["pool"] != "gpu" { + t.Fatalf("nodeSelector not carried: %v", q) + } + + // Without authentication there is no auth Secret. + in.DisableAuth = true + objs, _ = render(in, st) + if len(objs.Secrets) != 2 { + t.Fatalf("want no auth Secret, got %d secrets", len(objs.Secrets)) + } + if b, _, _ := unstructured.NestedBool(objs.Milvus.Object, "spec", "disableSecurity"); !b { + t.Fatal("disableSecurity must be set") + } +} + +func TestRenderNodeGroups(t *testing.T) { + refs := allRoles() + refs[3] = ref("rel-milvus-querynode-l40s", "querynode", 4) + refs = append(refs, ref("rel-milvus-querynode-h200", "querynode", 2)) + units, tmpls, _, _ := assignUnits(refs, sourceOptions{Release: "rel", RoleLabel: "component"}) + spec := distributedSpec(&Source{Units: units, Templates: tmpls}) + groups, _, _ := unstructured.NestedSlice(spec, "querynode", "groups") + if len(groups) != 2 || groups[0].(map[string]any)["name"] != "h200" || groups[0].(map[string]any)["replicas"] != int64(0) { + t.Fatalf("groups %v", groups) + } + u := units[3] + if petSetName("db", u) != "db-querynode-h200" || kubedbSelector("db", u)[labelNodeGroup] != "h200" { + t.Fatalf("group naming: %s %v", petSetName("db", u), kubedbSelector("db", u)) + } +} + +func TestSwapSteps(t *testing.T) { + if got := swapSteps(3, false); !reflect.DeepEqual(got, []step{{3, 0}}) { + t.Fatalf("whole role: %v", got) + } + if got := swapSteps(3, true); !reflect.DeepEqual(got, []step{{1, 2}, {2, 1}, {3, 0}}) { + t.Fatalf("one by one: %v", got) + } +} + +func TestHandoverOrder(t *testing.T) { + if o := handoverOrder(methodOverlap); o[0] != roleProxy || o[len(o)-1] != roleQueryNode { + t.Fatalf("overlap moves the proxies first and the QueryNodes last: %v", o) + } + if o := handoverOrder(methodStopStart); o[0] != roleMixCoord || o[len(o)-1] != roleProxy { + t.Fatalf("stop-start moves the coordinators first and the proxies last: %v", o) + } + if len(handoverOrder(methodOverlap)) != len(milvusRoles) || len(rollbackOrder) != len(milvusRoles) { + t.Fatal("every role is handed over and rolled back") + } +} + +func TestStateDone(t *testing.T) { + st := &State{} + u := Unit{Role: roleQueryNode, Group: "h200"} + st.markDone(u) + st.markDone(u) + if !st.isDone(u) || len(st.Done) != 1 || st.Done[0] != "querynode/h200" { + t.Fatalf("done %v", st.Done) + } + st.unmarkDone(u) + if st.isDone(u) { + t.Fatal("unmarkDone") + } +} + +func TestFallbackServiceRecord(t *testing.T) { + // A client Service KubeDB took over: its ports target the KubeDB container's + // port names. The chart's proxy names its gRPC port "milvus". + svc := core.Service{Spec: core.ServiceSpec{Ports: []core.ServicePort{ + {Name: "proxy", Port: 19530, TargetPort: intstr.FromString("proxy")}, + {Name: "metrics", Port: 9091, TargetPort: intstr.FromString("metrics")}, + {Name: "http", Port: 8080, TargetPort: intstr.FromString("http")}, + {Name: "admin", Port: 9999, TargetPort: intstr.FromInt32(9999)}, + }}} + svc.Name = "rel-milvus" + proxy := Unit{Role: roleProxy, Selector: map[string]string{"component": "proxy"}} + tpl := core.PodTemplateSpec{Spec: core.PodSpec{Containers: []core.Container{{ + Name: "proxy", Image: "milvusdb/milvus:v2.6.15", + Ports: []core.ContainerPort{{Name: "milvus", ContainerPort: 19530}, {Name: "metrics", ContainerPort: 9091}}, + }}}} + rec := fallbackServiceRecord(svc, proxy, tpl) + if !reflect.DeepEqual(rec.Selector, proxy.Selector) { + t.Fatalf("selector %v", rec.Selector) + } + got := make([]string, 0, len(rec.Ports)) + for _, p := range rec.Ports { + got = append(got, p.Name+"->"+p.TargetPort.String()) + } + want := []string{"proxy->19530", "metrics->metrics", "admin->9999"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ports %v, want %v (8080 only exists on KubeDB pods)", got, want) + } +} + +func TestRestPort(t *testing.T) { + if restPort(map[string]any{}) != 19530 { + t.Fatal("default") + } + if restPort(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": ""}, "port": float64(19531)}}) != 19531 { + t.Fatal("an empty http port serves REST on the gRPC port") + } + if restPort(map[string]any{"proxy": map[string]any{"http": map[string]any{"port": float64(8080)}}}) != 8080 { + t.Fatal("a set http port") + } +} + +const adoptionYAML = ` +apiVersion: cli.kubedb.com/v1alpha1 +kind: MilvusAdoption +release: pura-milvus +namespace: milvus-prod +source: + workloads: + - {role: proxy, kind: Deployment, name: mv-proxy} + - {role: querynode, group: l40s, kind: Deployment, name: mv-querynode-l40s} +auth: + authSecret: file-secret +restrictedSCC: true +handover: + method: stop-start + querynodeDelay: 10m +rollback: + cleanup: true +` + +func writeTemp(t *testing.T, content string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "adopt.yaml") + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestLoadAdoptionFile(t *testing.T) { + af, err := loadAdoptionFile(writeTemp(t, adoptionYAML)) + if err != nil { + t.Fatal(err) + } + if af.Release != "pura-milvus" || af.Namespace != "milvus-prod" || af.Handover.QueryNodeDelay.Duration != 10*time.Minute { + t.Fatalf("got %+v", af) + } + for name, content := range map[string]string{ + "unknown key": adoptionYAML + "querynodeDelay: 5m\n", + "wrong kind": strings.Replace(adoptionYAML, "kind: MilvusAdoption", "kind: Milvus", 1), + "no release": strings.Replace(adoptionYAML, "release: pura-milvus", "", 1), + "bad method": strings.Replace(adoptionYAML, "method: stop-start", "method: fast", 1), + "bad role": strings.Replace(adoptionYAML, "role: proxy", "role: rootcoord", 1), + "bad kind": strings.Replace(adoptionYAML, "{role: proxy, kind: Deployment", "{role: proxy, kind: Pod", 1), + "bad duration": strings.Replace(adoptionYAML, "querynodeDelay: 10m", "querynodeDelay: ten", 1), + "workload no name": strings.Replace(adoptionYAML, ", name: mv-proxy}", ", name: \"\"}", 1), + } { + if _, err := loadAdoptionFile(writeTemp(t, content)); err == nil { + t.Errorf("%s: want an error", name) + } + } +} + +func newRunCommandForTest(o *runOptions, fo *fileOptions) *cobra.Command { + cmd := &cobra.Command{Use: "run", RunE: func(*cobra.Command, []string) error { return nil }} + addPlanFlags(cmd, &o.planOptions) + addTimingFlags(cmd, &o.timing) + addFileFlag(cmd, fo) + cmd.Flags().StringVar(&o.Method, "method", methodOverlap, "") + cmd.Flags().DurationVar(&o.ReadyTimeout, "ready-timeout", 15*time.Minute, "") + return cmd +} + +func TestAdoptionFileFillsUnsetFlagsOnly(t *testing.T) { + var o runOptions + var fo fileOptions + cmd := newRunCommandForTest(&o, &fo) + path := writeTemp(t, adoptionYAML) + if err := cmd.ParseFlags([]string{"-f", path, "--auth-secret", "cli-secret", "--settle", "5s"}); err != nil { + t.Fatal(err) + } + release, err := fo.resolve(cmd, nil) + if err != nil { + t.Fatal(err) + } + if release != "pura-milvus" || fo.namespace() != "milvus-prod" { + t.Fatalf("release %q namespace %q", release, fo.namespace()) + } + if o.AuthSecret != "cli-secret" { + t.Fatalf("a flag on the command line must win over the file, got %q", o.AuthSecret) + } + if o.Method != methodStopStart || o.QueryNodeDelay != 10*time.Minute || !o.RestrictedSCC { + t.Fatalf("file values not applied: %+v", o) + } + if o.Settle != 5*time.Second || o.Timeout != 15*time.Minute { + t.Fatalf("flag and default must stay: settle %s timeout %s", o.Settle, o.Timeout) + } + want := []string{"proxy=Deployment/mv-proxy", "querynode/l40s=Deployment/mv-querynode-l40s"} + if !reflect.DeepEqual(o.Workloads, want) { + t.Fatalf("workloads %v", o.Workloads) + } +} + +func TestResolveRelease(t *testing.T) { + var o runOptions + var fo fileOptions + cmd := newRunCommandForTest(&o, &fo) + if _, err := fo.resolve(cmd, nil); err == nil { + t.Fatal("no RELEASE and no file must fail") + } + if r, err := fo.resolve(cmd, []string{"rel"}); err != nil || r != "rel" { + t.Fatalf("got %q %v", r, err) + } + if err := cmd.ParseFlags([]string{"-f", writeTemp(t, adoptionYAML)}); err != nil { + t.Fatal(err) + } + if _, err := fo.resolve(cmd, []string{"other"}); err == nil { + t.Fatal("a RELEASE that differs from the file must fail") + } +} + +func TestAdoptionFileForRoundTrip(t *testing.T) { + refs := allRoles() + refs[3] = ref("rel-milvus-querynode-l40s", "querynode", 4) + refs = append(refs, ref("rel-milvus-querynode-h200", "querynode", 2)) + units, tmpls, _, _ := assignUnits(refs, sourceOptions{Release: "rel", RoleLabel: "component"}) + in := renderInput{ + Namespace: "ns", Name: "rel-milvus", Version: "2.6.15-gpu", Release: "rel", + Source: &Source{Units: units, Templates: tmpls, Config: chartConfig(t)}, RootUser: "root", RootPassword: "x", + } + st := newState(in) + objs, err := render(in, st) + if err != nil { + t.Fatal(err) + } + o := planOptions{sourceOptions: sourceOptions{Release: "rel"}, AuthSecret: "milvus-root", RestrictedSCC: true} + raw, err := adoptionFileFor(&adoptPlan{State: st, Objects: objs}, o, "ns") + if err != nil { + t.Fatal(err) + } + af, err := loadAdoptionFile(writeTemp(t, string(raw))) + if err != nil { + t.Fatalf("the file plan writes must load again: %v\n%s", err, raw) + } + if af.Release != "rel" || af.Namespace != "ns" || af.Name != "rel-milvus" || af.Version != "2.6.15-gpu" || + af.Auth.AuthSecret != "milvus-root" || af.RestrictedSCC == nil || !*af.RestrictedSCC || len(af.Source.Workloads) != 6 { + t.Fatalf("got %+v", af) + } + var o2 runOptions + var fo fileOptions + cmd := newRunCommandForTest(&o2, &fo) + if err := cmd.ParseFlags([]string{"-f", writeTemp(t, string(raw))}); err != nil { + t.Fatal(err) + } + if _, err := fo.resolve(cmd, nil); err != nil { + t.Fatal(err) + } + gotUnits, _, fs, err := assignUnits(refs, o2.sourceOptions) + if err != nil || fs.hasErrors() || !reflect.DeepEqual(gotUnits, units) { + t.Fatalf("the written workloads must select the same units: %v %v\n%v\n%v", err, fs, gotUnits, units) + } +} diff --git a/pkg/adopt/milvus_tls.go b/pkg/adopt/milvus_tls.go new file mode 100644 index 000000000..ac04c7fe8 --- /dev/null +++ b/pkg/adopt/milvus_tls.go @@ -0,0 +1,504 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "path" + "strings" + "time" + + core "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// Secret keys the KubeDB operator reads a certificate from. +const ( + tlsCertKey = "tls.crt" + tlsKeyKey = "tls.key" + tlsCAKey = "ca.crt" + + certManagerGroup = "cert-manager.io" + milvusWorkDir = "/milvus" + certExpiryWarn = 30 * 24 * time.Hour +) + +var ( + certIssuerGVR = schema.GroupVersionResource{Group: certManagerGroup, Version: "v1", Resource: "issuers"} + certClusterIssuerGVR = schema.GroupVersionResource{Group: certManagerGroup, Version: "v1", Resource: "clusterissuers"} +) + +// tlsMaterial is a certificate as the running Milvus uses it. Secret is set when +// a Secret already has the keys KubeDB reads, so it can be referenced as it is +// and its owner's renewal keeps working. +type tlsMaterial struct { + Cert, Key, CA []byte + Secret string +} + +func (m *tlsMaterial) secretData() map[string]string { + return map[string]string{tlsCertKey: string(m.Cert), tlsKeyKey: string(m.Key), tlsCAKey: string(m.CA)} +} + +// adoptionTLS is the TLS the adopted Milvus keeps. +type adoptionTLS struct { + ExternalMode string // Disabled, TLS or mTLS + Internal bool + Server *tlsMaterial // nil when both client and internal TLS are off + ClientSecret string + IssuerKind string + IssuerName string +} + +// parseIssuerRef parses Kind/name; a bare name is an Issuer. +func parseIssuerRef(s string) (kind, name string, err error) { + kind, name = "Issuer", s + if k, n, ok := strings.Cut(s, "/"); ok { + kind, name = k, n + } + if name == "" || (kind != "Issuer" && kind != "ClusterIssuer") { + return "", "", fmt.Errorf("invalid issuer %q: want name, Issuer/name or ClusterIssuer/name", s) + } + return kind, name, nil +} + +// externalTLSMode reads common.security.tlsMode: 0 off, 1 TLS, 2 mTLS. +func externalTLSMode(cfg map[string]any) (string, error) { + switch v := lookupString(cfg, "common.security.tlsMode"); v { + case "", "0": + return "Disabled", nil + case "1": + return "TLS", nil + case "2": + return "mTLS", nil + default: + return "", fmt.Errorf("common.security.tlsMode is %q; Milvus knows 0, 1 and 2", v) + } +} + +// fileSource is one place a mounted file may come from. +type fileSource struct { + Kind string // Secret or ConfigMap + Name string + Key string +} + +// mountedFile lists where a file in the Milvus container comes from. Several +// sources are possible only for a projected volume without items. +func mountedFile(t core.PodTemplateSpec, file string) ([]fileSource, error) { + if !path.IsAbs(file) { + file = path.Join(milvusWorkDir, file) + } + c := milvusContainer(t) + if c == nil { + return nil, errors.New("no Milvus container") + } + var mount *core.VolumeMount + for i := range c.VolumeMounts { + m := &c.VolumeMounts[i] + mp := strings.TrimSuffix(m.MountPath, "/") + if file != mp && !strings.HasPrefix(file, mp+"/") { + continue + } + if mount == nil || len(mp) > len(strings.TrimSuffix(mount.MountPath, "/")) { + mount = m + } + } + if mount == nil { + return nil, fmt.Errorf("%s is not on a mounted volume", file) + } + rel := strings.TrimPrefix(strings.TrimPrefix(file, strings.TrimSuffix(mount.MountPath, "/")), "/") + rel = path.Join(mount.SubPath, rel) + if rel == "" || rel == "." { + return nil, fmt.Errorf("%s is a directory", file) + } + for _, v := range t.Spec.Volumes { + if v.Name == mount.Name { + return volumeFile(v, rel, file) + } + } + return nil, fmt.Errorf("volume %s of %s does not exist", mount.Name, file) +} + +func itemKey(items []core.KeyToPath, rel string) (string, bool) { + if len(items) == 0 { + return rel, true + } + for _, it := range items { + if it.Path == rel { + return it.Key, true + } + } + return "", false +} + +func volumeFile(v core.Volume, rel, file string) ([]fileSource, error) { + var out []fileSource + add := func(kind, name string, items []core.KeyToPath) { + if key, ok := itemKey(items, rel); ok { + out = append(out, fileSource{Kind: kind, Name: name, Key: key}) + } + } + switch { + case v.Secret != nil: + add("Secret", v.Secret.SecretName, v.Secret.Items) + case v.ConfigMap != nil: + add("ConfigMap", v.ConfigMap.Name, v.ConfigMap.Items) + case v.Projected != nil: + for _, s := range v.Projected.Sources { + if s.Secret != nil { + add("Secret", s.Secret.Name, s.Secret.Items) + } + if s.ConfigMap != nil { + add("ConfigMap", s.ConfigMap.Name, s.ConfigMap.Items) + } + } + default: + return nil, fmt.Errorf("%s does not come from a Secret or ConfigMap (volume %s), so it cannot be carried over", file, v.Name) + } + if len(out) == 0 { + return nil, fmt.Errorf("volume %s does not provide %s", v.Name, file) + } + return out, nil +} + +func (c *cluster) readSource(ctx context.Context, src fileSource) ([]byte, bool, error) { + if src.Kind == "ConfigMap" { + cm, err := c.kube.CoreV1().ConfigMaps(c.ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return nil, false, err + } + v, ok := cm.Data[src.Key] + return []byte(v), ok, nil + } + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, src.Name, metav1.GetOptions{}) + if err != nil { + return nil, false, err + } + v, ok := s.Data[src.Key] + return v, ok, nil +} + +func (c *cluster) readMounted(ctx context.Context, t core.PodTemplateSpec, file string) ([]byte, fileSource, error) { + sources, err := mountedFile(t, file) + if err != nil { + return nil, fileSource{}, err + } + for _, src := range sources { + data, ok, err := c.readSource(ctx, src) + if err != nil { + return nil, src, fmt.Errorf("failed to read %s %s: %w", src.Kind, src.Name, err) + } + if ok { + return data, src, nil + } + } + return nil, fileSource{}, fmt.Errorf("no mounted Secret or ConfigMap has %s", file) +} + +// readTLS reads the certificate, key and CA at the three config paths. +func (c *cluster) readTLS(ctx context.Context, t core.PodTemplateSpec, cfg map[string]any, certPath, keyPath, caPath string) (*tlsMaterial, error) { + m := &tlsMaterial{} + var srcs [3]fileSource + for i, p := range []string{certPath, keyPath, caPath} { + file := lookupString(cfg, p) + if file == "" { + return nil, fmt.Errorf("the config does not set %s", p) + } + data, src, err := c.readMounted(ctx, t, file) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + srcs[i] = src + switch i { + case 0: + m.Cert = data + case 1: + m.Key = data + case 2: + m.CA = data + } + } + name := srcs[0].Name + if srcs == [3]fileSource{{"Secret", name, tlsCertKey}, {"Secret", name, tlsKeyKey}, {"Secret", name, tlsCAKey}} { + m.Secret = name + } + return m, nil +} + +func parseLeaf(certPEM []byte) (*x509.Certificate, *x509.CertPool, error) { + var leaf *x509.Certificate + intermediates := x509.NewCertPool() + for rest := certPEM; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, err + } + if leaf == nil { + leaf = cert + } else { + intermediates.AddCert(cert) + } + } + if leaf == nil { + return nil, nil, errors.New("no certificate in the PEM data") + } + return leaf, intermediates, nil +} + +func verifyAgainstCA(certPEM, caPEM []byte, now time.Time) error { + leaf, intermediates, err := parseLeaf(certPEM) + if err != nil { + return err + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return errors.New("no certificate in the CA data") + } + _, err = leaf.Verify(x509.VerifyOptions{ + Roots: roots, Intermediates: intermediates, CurrentTime: now, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + }) + return err +} + +// checkCertificate reports a certificate that cannot serve this cluster. +func checkCertificate(label string, m *tlsMaterial, hosts []string, chain bool, now time.Time, fs *findings) { + if _, err := tls.X509KeyPair(m.Cert, m.Key); err != nil { + fs.errorf("%s: the certificate and key do not form a pair: %v", label, err) + return + } + leaf, _, err := parseLeaf(m.Cert) + if err != nil { + fs.errorf("%s: %v", label, err) + return + } + if chain { + if err := verifyAgainstCA(m.Cert, m.CA, now); err != nil { + fs.errorf("%s: the certificate does not verify against its CA: %v", label, err) + } + } + for _, h := range hosts { + if err := leaf.VerifyHostname(h); err != nil { + fs.errorf("%s: the certificate does not cover %s, which KubeDB connects to (it covers %s); reissue it with that name", label, h, strings.Join(leaf.DNSNames, ", ")) + } + } + if left := leaf.NotAfter.Sub(now); left > 0 && left < certExpiryWarn { + fs.warnf("%s: the certificate expires on %s", label, leaf.NotAfter.Format("2006-01-02")) + } +} + +// clientHosts are the names KubeDB and Milvus use to reach the proxies. +func clientHosts(name, ns string) []string { + return []string{name, name + "." + ns + ".svc", name + "." + ns + ".svc.cluster.local"} +} + +// planEtcdTLS carries the client certificate of an etcd that uses TLS. +func (c *cluster) planEtcdTLS(ctx context.Context, t core.PodTemplateSpec, cfg map[string]any, in *renderInput, fs *findings) { + if !etcdFromConfig(cfg).TLSEnabled { + return + } + m, err := c.readTLS(ctx, t, cfg, "etcd.ssl.tlsCert", "etcd.ssl.tlsKey", "etcd.ssl.tlsCACert") + if err != nil { + fs.errorf("etcd TLS: %v", err) + return + } + checkCertificate("etcd TLS", m, nil, false, time.Now(), fs) + in.EtcdTLS = m + fs.infof("etcd TLS: %s", materialNote(m)) +} + +func materialNote(m *tlsMaterial) string { + if m.Secret != "" { + return "Secret " + m.Secret + " is used as it is" + } + return "the certificate is copied to a new Secret, because its keys are not tls.crt, tls.key and ca.crt; renewing the copy is up to you" +} + +// planTLS keeps the TLS the running Milvus uses. What exists is reused, and +// KubeDB issues only a certificate that is missing. TLS that is off stays off: +// switching it on during a handover would split the old pods from the new ones. +func (c *cluster) planTLS(ctx context.Context, o planOptions, src *Source, in *renderInput, fs *findings) error { + cfg := src.Config + proxy := src.Templates[src.firstUnit(roleProxy).key()] + c.planEtcdTLS(ctx, proxy, cfg, in, fs) + mode, err := externalTLSMode(cfg) + if err != nil { + fs.errorf("%v", err) + return nil + } + internal := lookupBool(cfg, "common.security.internaltlsEnabled") + if mode == "Disabled" && !internal { + fs.infof("client and internal TLS are off and stay off") + return nil + } + out := &adoptionTLS{ExternalMode: mode, Internal: internal} + + var external, inner *tlsMaterial + if mode != "Disabled" { + if external, err = c.readTLS(ctx, proxy, cfg, "tls.serverPemPath", "tls.serverKeyPath", "tls.caPemPath"); err != nil { + fs.errorf("client TLS: %v", err) + return nil + } + out.Server = external + } + if internal { + if inner, err = c.readTLS(ctx, proxy, cfg, "internaltls.serverPemPath", "internaltls.serverKeyPath", "internaltls.caPemPath"); err != nil { + fs.errorf("internal TLS: %v", err) + return nil + } + out.Server = inner + } + if external != nil && inner != nil { + if !bytes.Equal(external.Cert, inner.Cert) { + fs.errorf("client and internal TLS use different certificates; KubeDB mounts one for both") + return nil + } + out.Server = external + } + checkCertificate("server certificate", out.Server, clientHosts(in.Name, c.ns), true, time.Now(), fs) + fs.infof("TLS: client %s, internal %v; %s", mode, internal, materialNote(out.Server)) + + if mode == "mTLS" { + if err := c.planClientCert(ctx, o, out, fs); err != nil { + return err + } + } else if o.TLSIssuer != "" || o.TLSClientSecret != "" { + fs.infof("no client certificate is needed without mTLS; --tls-issuer and --tls-client-secret are not used") + } + in.TLS = out + c.setRESTTLS(out, in.Name) + return nil +} + +// planClientCert finds the certificate KubeDB presents when Milvus asks clients +// for one: the user's, or else one from the issuer. +func (c *cluster) planClientCert(ctx context.Context, o planOptions, out *adoptionTLS, fs *findings) error { + switch { + case o.TLSClientSecret != "": + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, o.TLSClientSecret, metav1.GetOptions{}) + if err != nil { + fs.errorf("--tls-client-secret: %v", err) + return nil + } + for _, k := range []string{tlsCertKey, tlsKeyKey, tlsCAKey} { + if _, ok := s.Data[k]; !ok { + fs.errorf("--tls-client-secret: Secret %s has no key %s", s.Name, k) + return nil + } + } + checkCertificate("client certificate", &tlsMaterial{Cert: s.Data[tlsCertKey], Key: s.Data[tlsKeyKey], CA: out.Server.CA}, nil, true, time.Now(), fs) + if err := verifyAgainstCA(out.Server.Cert, s.Data[tlsCAKey], time.Now()); err != nil { + fs.errorf("--tls-client-secret: its ca.crt does not verify the server certificate: %v", err) + } + out.ClientSecret = s.Name + fs.infof("client certificate: Secret %s is used as it is", s.Name) + case o.TLSIssuer != "": + kind, name, err := parseIssuerRef(o.TLSIssuer) + if err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + out.IssuerKind, out.IssuerName = kind, name + return c.checkIssuer(ctx, kind, name, out.Server.CA, fs) + default: + fs.errorf("client mTLS is on, so KubeDB needs a client certificate that Milvus trusts; give it with --tls-client-secret, or give --tls-issuer to have one issued") + } + return nil +} + +// checkIssuer stops an issuer whose certificates Milvus would reject: in mTLS, +// Milvus trusts only the CA it runs with. +func (c *cluster) checkIssuer(ctx context.Context, kind, name string, serverCA []byte, fs *findings) error { + if kind == "ClusterIssuer" { + if _, err := c.dyn.Resource(certClusterIssuerGVR).Get(ctx, name, metav1.GetOptions{}); err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + fs.warnf("cannot confirm that ClusterIssuer %s signs with the CA Milvus trusts; client certificates from another CA are rejected", name) + return nil + } + obj, err := c.dyn.Resource(certIssuerGVR).Namespace(c.ns).Get(ctx, name, metav1.GetOptions{}) + if kerr.IsNotFound(err) { + fs.errorf("--tls-issuer: Issuer %s does not exist", name) + return nil + } + if err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + spec, _ := obj.Object["spec"].(map[string]any) + ca, _ := spec["ca"].(map[string]any) + secret, _ := ca["secretName"].(string) + if secret == "" { + fs.warnf("cannot confirm that Issuer %s signs with the CA Milvus trusts; client certificates from another CA are rejected", name) + return nil + } + s, err := c.kube.CoreV1().Secrets(c.ns).Get(ctx, secret, metav1.GetOptions{}) + if err != nil { + fs.errorf("--tls-issuer: %v", err) + return nil + } + if !bytes.Equal(bytes.TrimSpace(s.Data[tlsCertKey]), bytes.TrimSpace(serverCA)) { + fs.errorf("Issuer %s signs with a different CA than the one Milvus trusts, so Milvus would reject its client certificates", name) + } + return nil +} + +// setRESTTLS makes the credential check speak TLS to the running proxies. +func (c *cluster) setRESTTLS(t *adoptionTLS, name string) { + if t.ExternalMode == "Disabled" { + return + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(t.Server.CA) + c.restTLS = &tls.Config{RootCAs: roots, ServerName: name, MinVersion: tls.VersionTLS12} +} + +// tlsSpec is spec.tls of the adopted Milvus. +func tlsSpec(t *adoptionTLS, serverSecret string) map[string]any { + internal := "Disabled" + if t.Internal { + internal = "TLS" + } + existing := map[string]any{"server": map[string]any{"name": serverSecret}} + if t.ClientSecret != "" { + existing["client"] = map[string]any{"name": t.ClientSecret} + } + spec := map[string]any{ + "external": map[string]any{"mode": t.ExternalMode}, + "internal": map[string]any{"mode": internal}, + "existingSecrets": existing, + } + if t.IssuerName != "" { + spec["issuerRef"] = map[string]any{"apiGroup": certManagerGroup, "kind": t.IssuerKind, "name": t.IssuerName} + } + return spec +} diff --git a/pkg/adopt/milvus_tls_test.go b/pkg/adopt/milvus_tls_test.go new file mode 100644 index 000000000..b710741c3 --- /dev/null +++ b/pkg/adopt/milvus_tls_test.go @@ -0,0 +1,245 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package adopt + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "strings" + "testing" + "time" + + core "k8s.io/api/core/v1" +) + +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + pem []byte +} + +func newTestCA(t *testing.T) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * 365 * time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + cert, _ := x509.ParseCertificate(der) + return &testCA{cert: cert, key: key, pem: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})} +} + +func (ca *testCA) issue(t *testing.T, ttl time.Duration, dns ...string) *tlsMaterial { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "milvus"}, DNSNames: dns, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(ttl), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + return &tlsMaterial{ + Cert: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + Key: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + CA: ca.pem, + } +} + +func TestCheckCertificate(t *testing.T) { + ca := newTestCA(t) + hosts := clientHosts("mv", "demo") + year := 365 * 24 * time.Hour + now := time.Now() + + good := ca.issue(t, year, hosts...) + fs := &findings{} + checkCertificate("server", good, hosts, true, now, fs) + if fs.hasErrors() || len(*fs) != 0 { + t.Fatalf("a good certificate has findings: %s", findingText(*fs)) + } + + fs = &findings{} + checkCertificate("server", ca.issue(t, year, "other.example.com"), hosts, true, now, fs) + if !fs.hasErrors() || !strings.Contains(findingText(*fs), "does not cover mv") { + t.Fatalf("a wrong name must stop the plan: %s", findingText(*fs)) + } + + fs = &findings{} + foreign := ca.issue(t, year, hosts...) + foreign.CA = newTestCA(t).pem + checkCertificate("server", foreign, hosts, true, now, fs) + if !fs.hasErrors() { + t.Fatal("a certificate from another CA must stop the plan") + } + + fs = &findings{} + checkCertificate("server", ca.issue(t, 5*24*time.Hour, hosts...), hosts, true, now, fs) + if fs.hasErrors() || !strings.Contains(findingText(*fs), "expires") { + t.Fatalf("a certificate near expiry is a warning: %s", findingText(*fs)) + } + + fs = &findings{} + mismatched := ca.issue(t, year, hosts...) + mismatched.Key = ca.issue(t, year, hosts...).Key + checkCertificate("server", mismatched, hosts, true, now, fs) + if !fs.hasErrors() { + t.Fatal("a key of another certificate must stop the plan") + } +} + +func TestExternalTLSMode(t *testing.T) { + for v, want := range map[any]string{nil: "Disabled", 0: "Disabled", 1: "TLS", float64(2): "mTLS"} { + cfg := map[string]any{} + if v != nil { + cfg = map[string]any{"common": map[string]any{"security": map[string]any{"tlsMode": v}}} + } + if got, err := externalTLSMode(cfg); err != nil || got != want { + t.Fatalf("tlsMode %v: got %q %v, want %q", v, got, err, want) + } + } + if _, err := externalTLSMode(map[string]any{"common": map[string]any{"security": map[string]any{"tlsMode": 3}}}); err == nil { + t.Fatal("an unknown mode must fail") + } +} + +func TestMountedFile(t *testing.T) { + tmpl := core.PodTemplateSpec{Spec: core.PodSpec{ + Containers: []core.Container{{Name: "milvus", Image: "milvusdb/milvus", VolumeMounts: []core.VolumeMount{ + {Name: "certs", MountPath: "/certs"}, + {Name: "items", MountPath: "/items"}, + {Name: "proj", MountPath: "/proj"}, + {Name: "sub", MountPath: "/single/server.pem", SubPath: "tls.crt"}, + {Name: "disk", MountPath: "/disk"}, + {Name: "certs", MountPath: "/milvus/configs/cert"}, + }}}, + Volumes: []core.Volume{ + {Name: "certs", VolumeSource: core.VolumeSource{Secret: &core.SecretVolumeSource{SecretName: "s1"}}}, + {Name: "items", VolumeSource: core.VolumeSource{Secret: &core.SecretVolumeSource{ + SecretName: "s2", Items: []core.KeyToPath{{Key: "tls.crt", Path: "server.pem"}}, + }}}, + {Name: "proj", VolumeSource: core.VolumeSource{Projected: &core.ProjectedVolumeSource{Sources: []core.VolumeProjection{ + {Secret: &core.SecretProjection{LocalObjectReference: core.LocalObjectReference{Name: "s3"}, Items: []core.KeyToPath{{Key: "tls.key", Path: "server.key"}}}}, + {ConfigMap: &core.ConfigMapProjection{LocalObjectReference: core.LocalObjectReference{Name: "cm"}, Items: []core.KeyToPath{{Key: "ca", Path: "ca.pem"}}}}, + }}}}, + {Name: "sub", VolumeSource: core.VolumeSource{Secret: &core.SecretVolumeSource{SecretName: "s4"}}}, + {Name: "disk", VolumeSource: core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{}}}, + }, + }} + for file, want := range map[string]fileSource{ + "/certs/server.pem": {"Secret", "s1", "server.pem"}, + "/items/server.pem": {"Secret", "s2", "tls.crt"}, + "/proj/server.key": {"Secret", "s3", "tls.key"}, + "/proj/ca.pem": {"ConfigMap", "cm", "ca"}, + "/single/server.pem": {"Secret", "s4", "tls.crt"}, + "configs/cert/ca.pem": {"Secret", "s1", "ca.pem"}, // relative to the Milvus work dir + } { + got, err := mountedFile(tmpl, file) + if err != nil || len(got) != 1 || got[0] != want { + t.Fatalf("%s: got %+v %v, want %+v", file, got, err, want) + } + } + for _, file := range []string{"/nowhere/x.pem", "/disk/x.pem", "/items/other.pem"} { + if _, err := mountedFile(tmpl, file); err == nil { + t.Fatalf("%s must fail", file) + } + } +} + +func TestParseIssuerRef(t *testing.T) { + for in, want := range map[string][2]string{ + "ca": {"Issuer", "ca"}, "Issuer/ca": {"Issuer", "ca"}, "ClusterIssuer/ca": {"ClusterIssuer", "ca"}, + } { + if k, n, err := parseIssuerRef(in); err != nil || k != want[0] || n != want[1] { + t.Fatalf("%s: got %s %s %v", in, k, n, err) + } + } + for _, in := range []string{"Certificate/ca", "Issuer/", ""} { + if _, _, err := parseIssuerRef(in); err == nil { + t.Fatalf("%q must fail", in) + } + } +} + +func TestCarriedConfigDropsTLS(t *testing.T) { + raw, err := carriedConfig(map[string]any{ + "tls": map[string]any{"serverPemPath": "/old/server.pem"}, + "internaltls": map[string]any{"serverPemPath": "/old/server.pem"}, + "common": map[string]any{"security": map[string]any{ + "tlsMode": 1, "internaltlsEnabled": true, "authorizationEnabled": true, + }}, + }) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{"/old/", "tlsMode", "internaltlsEnabled"} { + if strings.Contains(raw, s) { + t.Fatalf("%s must not be carried, the operator renders it: %s", s, raw) + } + } + if !strings.Contains(raw, "authorizationEnabled") { + t.Fatalf("other security settings stay: %s", raw) + } +} + +func TestConfigMismatchesTLS(t *testing.T) { + sec := func(v map[string]any) map[string]any { + return map[string]any{"common": map[string]any{"security": v}, "etcd": map[string]any{"rootPath": "by-dev"}} + } + if d := configMismatches(sec(map[string]any{"tlsMode": 0, "internaltlsEnabled": false}), sec(map[string]any{})); len(d) != 0 { + t.Fatalf("off and not rendered are the same: %v", d) + } + if d := configMismatches(sec(map[string]any{"tlsMode": 1}), sec(map[string]any{})); len(d) != 1 { + t.Fatalf("TLS on and not rendered must be reported: %v", d) + } +} + +func TestTLSSpec(t *testing.T) { + got := tlsSpec(&adoptionTLS{ExternalMode: "mTLS", Internal: true, IssuerKind: "Issuer", IssuerName: "ca"}, "theirs") + if got["external"].(map[string]any)["mode"] != "mTLS" || got["internal"].(map[string]any)["mode"] != "TLS" { + t.Fatalf("modes: %v", got) + } + if got["existingSecrets"].(map[string]any)["server"].(map[string]any)["name"] != "theirs" { + t.Fatalf("server Secret: %v", got) + } + if got["issuerRef"].(map[string]any)["apiGroup"] != certManagerGroup { + t.Fatalf("issuer: %v", got) + } + got = tlsSpec(&adoptionTLS{ExternalMode: "TLS"}, "theirs") + if _, ok := got["issuerRef"]; ok { + t.Fatalf("nothing to issue, no issuerRef: %v", got) + } +} diff --git a/pkg/cmds/adopt.go b/pkg/cmds/adopt.go new file mode 100644 index 000000000..76fdd243e --- /dev/null +++ b/pkg/cmds/adopt.go @@ -0,0 +1,26 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmds + +import ( + "kubedb.dev/cli/pkg/adopt" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +// NewCmdAdopt returns the command group that adopts a running database in place. +func NewCmdAdopt(f cmdutil.Factory) *cobra.Command { + return adopt.NewCmdAdopt(f) +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 1ab74a0f1..d9c583920 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -114,6 +114,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command { NewCmdMSSQL(f), }, }, + { + Message: "Adopt a running database in place", + Commands: []*cobra.Command{ + NewCmdAdopt(f), + }, + }, { Message: "Cross Data Center DR Commands", Commands: []*cobra.Command{