From da61e9e55c51f82d87c6d50f0378feba2be5da61 Mon Sep 17 00:00:00 2001 From: Ryan Swanson Date: Fri, 12 Jul 2024 14:02:42 -0400 Subject: [PATCH] Detect how the vcluster was put to sleep and guide the user to the correct commands to wake it up Warns allows waking and adding a sleeping helm vcluster in a single command. Adds the ability to set chart dir for local dev when adding external cluster. Only errors if helm driver is specified, otherwise it falls back to a platform resume. Remove old check due to platform's auto conversion of helm slept clusters. Continue with adding the secret in either case and only add if they choose to wake the cluster. Co-authored-by: Russell Centanni --- cmd/vclusterctl/cmd/platform/add/cluster.go | 2 +- cmd/vclusterctl/cmd/resume.go | 12 ++++- config/config.go | 8 ++++ no-sleepmode.yaml | 0 pkg/cli/add_vcluster_helm.go | 51 ++++++++++++++++++++- pkg/cli/create_helm.go | 9 +--- pkg/cli/find/find.go | 40 ++++++++++++---- pkg/cli/pause_helm.go | 5 ++ pkg/cli/pause_platform.go | 8 +++- pkg/cli/resume_helm.go | 7 +++ pkg/cli/resume_platform.go | 10 +++- pkg/kube/meta.go | 30 ++++++++++-- pkg/lifecycle/lifecycle.go | 13 ++++-- pkg/platform/helper.go | 9 +++- pkg/platform/sleepmode/sleepmode.go | 16 +++++++ 15 files changed, 186 insertions(+), 34 deletions(-) create mode 100644 no-sleepmode.yaml create mode 100644 pkg/platform/sleepmode/sleepmode.go diff --git a/cmd/vclusterctl/cmd/platform/add/cluster.go b/cmd/vclusterctl/cmd/platform/add/cluster.go index 81a5e008f7..a5c7636978 100644 --- a/cmd/vclusterctl/cmd/platform/add/cluster.go +++ b/cmd/vclusterctl/cmd/platform/add/cluster.go @@ -143,7 +143,7 @@ func (cmd *ClusterCmd) Run(ctx context.Context, args []string) error { if os.Getenv("DEVELOPMENT") == "true" { helmArgs = []string{ - "upgrade", "--install", "loft", "./chart", + "upgrade", "--install", "loft", cmp.Or(os.Getenv("DEVELOPMENT_CHART_DIR"), "./chart"), "--create-namespace", "--namespace", namespace, "--set", "agentOnly=true", diff --git a/cmd/vclusterctl/cmd/resume.go b/cmd/vclusterctl/cmd/resume.go index a6428170b3..07508da133 100644 --- a/cmd/vclusterctl/cmd/resume.go +++ b/cmd/vclusterctl/cmd/resume.go @@ -3,6 +3,7 @@ package cmd import ( "cmp" "context" + "errors" "fmt" "github.com/loft-sh/log" @@ -73,5 +74,14 @@ func (cmd *ResumeCmd) Run(ctx context.Context, args []string) error { return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log) } - return cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log) + if err := cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log); err != nil { + // If they specified a driver, don't fall back to the platform automatically. + if cmd.Driver == "" && errors.Is(err, cli.ErrPlatformDriverRequired) { + return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log) + } + + return err + } + + return nil } diff --git a/config/config.go b/config/config.go index dc07106dd8..0cd4049230 100644 --- a/config/config.go +++ b/config/config.go @@ -169,6 +169,14 @@ func (c *Config) Distro() string { return K8SDistro } +func (c *Config) IsConfiguredForSleepMode() bool { + if c != nil && c.External["platform"] == nil { + return false + } + + return c.External["platform"]["autoSleep"] != nil || c.External["platform"]["autoDelete"] != nil +} + // ValidateChanges checks for disallowed config changes. // Currently only certain backingstore changes are allowed but no distro change. func ValidateChanges(oldCfg, newCfg *Config) error { diff --git a/no-sleepmode.yaml b/no-sleepmode.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pkg/cli/add_vcluster_helm.go b/pkg/cli/add_vcluster_helm.go index 5d38bed262..d1117a8215 100644 --- a/pkg/cli/add_vcluster_helm.go +++ b/pkg/cli/add_vcluster_helm.go @@ -3,12 +3,16 @@ package cli import ( "context" "fmt" + "time" "github.com/loft-sh/log" + "github.com/loft-sh/log/survey" "github.com/loft-sh/vcluster/pkg/cli/find" "github.com/loft-sh/vcluster/pkg/cli/flags" "github.com/loft-sh/vcluster/pkg/lifecycle" "github.com/loft-sh/vcluster/pkg/platform" + "github.com/loft-sh/vcluster/pkg/platform/clihelper" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) @@ -44,6 +48,46 @@ func AddVClusterHelm( return err } + snoozed := false + // If the vCluster was paused with the helm driver, adding it to the platform will only create the secret for registration + // which leads to confusing behavior for the user since they won't see the cluster in the platform UI until it is resumed. + if lifecycle.IsPaused(vCluster) { + log.Infof("vCluster %s is currently sleeping. It will not be added to the platform until it wakes again.", vCluster.Name) + + snoozeConfirmation := "No. Leave it sleeping. (It will be added automatically on next wakeup)" + answer, err := log.Question(&survey.QuestionOptions{ + Question: fmt.Sprintf("Would you like to wake vCluster %s now to add immediately?", vCluster.Name), + DefaultValue: snoozeConfirmation, + Options: []string{ + snoozeConfirmation, + "Yes. Wake and add now.", + }, + }) + + if err != nil { + return fmt.Errorf("failed to capture your response %w", err) + } + + if snoozed = answer == snoozeConfirmation; !snoozed { + if err = ResumeHelm(ctx, globalFlags, vClusterName, log); err != nil { + return fmt.Errorf("failed to wake up vCluster %s: %w", vClusterName, err) + } + + err = wait.PollUntilContextTimeout(ctx, time.Second, clihelper.Timeout(), false, func(ctx context.Context) (done bool, err error) { + vCluster, err = find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log) + if err != nil { + return false, err + } + + return !lifecycle.IsPaused(vCluster), nil + }) + + if err != nil { + return fmt.Errorf("error waiting for vCluster to wake up %w", err) + } + } + } + // apply platform secret err = platform.ApplyPlatformSecret( ctx, @@ -68,6 +112,11 @@ func AddVClusterHelm( } } - log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name) + if snoozed { + log.Infof("vCluster %s/%s will be added the next time it awakes", vCluster.Namespace, vCluster.Name) + log.Donef("Run 'vcluster wakeup --help' to learn how to wake up vCluster %s/%s to complete the add operation.", vCluster.Namespace, vCluster.Name) + } else { + log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name) + } return nil } diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index 4e0e7b8f14..ea43679b17 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -268,7 +268,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. cmd.Connect = false } - if isSleepModeConfigured(vClusterConfig) { + if vClusterConfig.IsConfiguredForSleepMode() { if agentDeployed, err := cmd.isLoftAgentDeployed(ctx); err != nil { return fmt.Errorf("is agent deployed: %w", err) } else if !agentDeployed { @@ -384,13 +384,6 @@ func (cmd *createHelm) isLoftAgentDeployed(ctx context.Context) (bool, error) { return len(podList.Items) > 0, nil } -func isSleepModeConfigured(vClusterConfig *config.Config) bool { - if vClusterConfig == nil || vClusterConfig.External == nil || vClusterConfig.External["platform"] == nil { - return false - } - return vClusterConfig.External["platform"]["autoSleep"] != nil || vClusterConfig.External["platform"]["autoDelete"] != nil -} - func isVClusterDeployed(release *helm.Release) bool { return release != nil && release.Chart != nil && diff --git a/pkg/cli/find/find.go b/pkg/cli/find/find.go index 357e9fd65b..f5af82fdc5 100644 --- a/pkg/cli/find/find.go +++ b/pkg/cli/find/find.go @@ -11,6 +11,7 @@ import ( "github.com/loft-sh/log/survey" "github.com/loft-sh/log/terminal" "github.com/loft-sh/vcluster/pkg/platform" + "github.com/loft-sh/vcluster/pkg/platform/sleepmode" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/loft-sh/vcluster/pkg/constants" @@ -27,9 +28,12 @@ import ( const VirtualClusterSelector = "app=vcluster" type VCluster struct { - Name string - Namespace string - + ClientFactory clientcmd.ClientConfig `json:"-"` + Created metav1.Time + Name string + Namespace string + Annotations map[string]string + Labels map[string]string Status Status Created metav1.Time Context string @@ -158,6 +162,20 @@ func GetVCluster(ctx context.Context, context, name, namespace string, log log.L return nil, fmt.Errorf("unexpected error searching for selected virtual cluster") } +func (v *VCluster) IsSleeping() bool { + return sleepmode.IsSleeping(v) +} + +// GetAnnotations implements Annotated +func (v *VCluster) GetAnnotations() map[string]string { + return v.Annotations +} + +// GetLabels implements Labeled +func (v *VCluster) GetLabels() map[string]string { + return v.Labels +} + func FormatOptions(format string, options [][]string) []string { if len(options) == 0 { return []string{} @@ -309,12 +327,7 @@ func findInContext(ctx context.Context, context, name, namespace string, timeout continue } - var paused string - - if p.Annotations != nil { - paused = p.Annotations[constants.PausedAnnotation] - } - if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && paused != "true" { + if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && !isPaused(&p) { // if the stateful set has been scaled down we'll ignore it -- this happens when // using devspace to do vcluster plugin dev for example, devspace scales down the // vcluster stateful set and re-creates a deployment for "dev mode" so we end up @@ -410,6 +423,8 @@ func getVCluster(ctx context.Context, object client.Object, context, release str return VCluster{ Name: release, Namespace: namespace, + Annotations: object.GetAnnotations(), + Labels: object.GetLabels(), Status: Status(status), Created: created, Context: context, @@ -558,3 +573,10 @@ func GetPodStatus(pod *corev1.Pod) string { } return reason } + +func isPaused(v client.Object) bool { + annotations := v.GetAnnotations() + labels := v.GetLabels() + + return annotations[constants.PausedAnnotation] == "true" || labels[sleepmode.Label] == "true" +} diff --git a/pkg/cli/pause_helm.go b/pkg/cli/pause_helm.go index 0826c68122..33f9db5c9b 100644 --- a/pkg/cli/pause_helm.go +++ b/pkg/cli/pause_helm.go @@ -30,6 +30,11 @@ func PauseHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName return err } + if vCluster.IsSleeping() { + log.Infof("vcluster %s/%s is already sleeping", globalFlags.Namespace, vClusterName) + return nil + } + err = lifecycle.PauseVCluster(ctx, kubeClient, vClusterName, globalFlags.Namespace, log) if err != nil { return err diff --git a/pkg/cli/pause_platform.go b/pkg/cli/pause_platform.go index e5bf548562..26869e9b06 100644 --- a/pkg/cli/pause_platform.go +++ b/pkg/cli/pause_platform.go @@ -22,11 +22,15 @@ func PausePlatform(ctx context.Context, options *PauseOptions, cfg *cliconfig.CL if err != nil { return err } + vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log) if err != nil { return err - } else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External { - return fmt.Errorf("cannot pause a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag") + } + + if vCluster.IsInstanceSleeping() { + log.Infof("vcluster %s/%s is already paused", vCluster.VirtualCluster.Namespace, vClusterName) + return nil } managementClient, err := platformClient.Management() diff --git a/pkg/cli/resume_helm.go b/pkg/cli/resume_helm.go index 64a6a6ab74..6d986da8d2 100644 --- a/pkg/cli/resume_helm.go +++ b/pkg/cli/resume_helm.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "github.com/loft-sh/log" @@ -17,12 +18,18 @@ type ResumeOptions struct { Project string } +var ErrPlatformDriverRequired = errors.New("cannot resume a virtual cluster that is paused by the platform, please run 'vcluster use driver platform' or use the '--driver platform' flag") + func ResumeHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName string, log log.Logger) error { vCluster, err := find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log) if err != nil { return err } + if vCluster.IsSleeping() { + return ErrPlatformDriverRequired + } + kubeClient, err := prepareResume(vCluster, globalFlags) if err != nil { return err diff --git a/pkg/cli/resume_platform.go b/pkg/cli/resume_platform.go index 0dc1c493a3..e8683fc998 100644 --- a/pkg/cli/resume_platform.go +++ b/pkg/cli/resume_platform.go @@ -18,8 +18,14 @@ func ResumePlatform(ctx context.Context, options *ResumeOptions, config *config. vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log) if err != nil { return err - } else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External { - return fmt.Errorf("cannot resume a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag") + } + + if !vCluster.IsInstanceSleeping() { + return fmt.Errorf( + "couldn't find a paused vcluster %s in namespace %s. Make sure the vcluster exists and was paused previously", + vCluster.VirtualCluster.Spec.ClusterRef.VirtualCluster, + vCluster.VirtualCluster.Spec.ClusterRef.Namespace, + ) } managementClient, err := platformClient.Management() diff --git a/pkg/kube/meta.go b/pkg/kube/meta.go index 3024829139..8b7bd45497 100644 --- a/pkg/kube/meta.go +++ b/pkg/kube/meta.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/loft-sh/log" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -17,7 +16,30 @@ const ( LoftCustomLinksDelimiter = "\n" ) -func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) { +type ( + // Annotated is an interface for objects that have annotations + Annotated interface { + GetAnnotations() map[string]string + } + // Annotatable is an interface for objects that have annotations and ` + Annotatable interface { + Annotated + SetAnnotations(map[string]string) + } + + // Labeled is an interface for objects that have labels + Labeled interface { + GetLabels() map[string]string + } + + // Labelable is an interface for objects that have labels and can set them + Labelable interface { + Labeled + SetLabels(map[string]string) + } +) + +func UpdateLabels(obj Labelable, labelList []string) (bool, error) { // parse strings to map labels, err := parseStringMap(labelList) if err != nil { @@ -44,7 +66,7 @@ func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) { return changed, nil } -func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) { +func UpdateAnnotations(obj Annotatable, annotationList []string) (bool, error) { // parse strings to map annotations, err := parseStringMap(annotationList) if err != nil { @@ -72,7 +94,7 @@ func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) // SetCustomLinksAnnotation sets the list of links for the UI to display next to the project member({space/virtualcluster}instance) // it handles unspecified links (empty) during create and update -func SetCustomLinksAnnotation(obj metav1.Object, links []string) bool { +func SetCustomLinksAnnotation(obj Annotatable, links []string) bool { var changed bool if obj == nil { log.GetInstance().Error("SetCustomLinksAnnotation called on nil object") diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index bb9cc90ef2..140e808978 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -7,6 +7,7 @@ import ( "github.com/loft-sh/log" "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/kube" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -117,7 +118,7 @@ func scaleDownDeployment(ctx context.Context, kubeClient kubernetes.Interface, l zero := int32(0) for _, item := range list.Items { - if item.Annotations != nil && item.Annotations[constants.PausedAnnotation] == "true" { + if IsPaused(&item) { log.Infof("vcluster %s/%s is already paused", namespace, item.Name) return true, nil } else if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { @@ -179,7 +180,7 @@ func scaleDownStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, zero := int32(0) for _, item := range list.Items { - if item.Annotations != nil && item.Annotations[constants.PausedAnnotation] == "true" { + if IsPaused(&item) { log.Infof("vcluster %s/%s is already paused", namespace, item.Name) return true, nil } else if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { @@ -277,7 +278,7 @@ func scaleUpDeployment(ctx context.Context, kubeClient kubernetes.Interface, lab } for _, item := range list.Items { - if item.Annotations == nil || item.Annotations[constants.PausedAnnotation] != "true" { + if !IsPaused(&item) { return false, nil } @@ -323,7 +324,7 @@ func scaleUpStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, la } for _, item := range list.Items { - if item.Annotations == nil || item.Annotations[constants.PausedAnnotation] != "true" { + if !IsPaused(&item) { return false, nil } @@ -359,3 +360,7 @@ func scaleUpStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, la return true, nil } + +func IsPaused(annotated kube.Annotated) bool { + return annotated != nil && annotated.GetAnnotations()[constants.PausedAnnotation] == "true" +} diff --git a/pkg/platform/helper.go b/pkg/platform/helper.go index 129a98a8ee..f4da8b491e 100644 --- a/pkg/platform/helper.go +++ b/pkg/platform/helper.go @@ -21,6 +21,7 @@ import ( "github.com/loft-sh/vcluster/pkg/platform/clihelper" "github.com/loft-sh/vcluster/pkg/platform/kube" "github.com/loft-sh/vcluster/pkg/platform/kubeconfig" + "github.com/loft-sh/vcluster/pkg/platform/sleepmode" "github.com/loft-sh/vcluster/pkg/projectutil" "github.com/loft-sh/vcluster/pkg/util" perrors "github.com/pkg/errors" @@ -545,6 +546,10 @@ type ProjectProjectSecret struct { Project string } +func (vci *VirtualClusterInstanceProject) IsInstanceSleeping() bool { + return vci != nil && vci.VirtualCluster != nil && sleepmode.IsInstanceSleeping(vci.VirtualCluster) +} + func GetProjectSecrets(ctx context.Context, managementClient kube.Interface, projectNames ...string) ([]*ProjectProjectSecret, error) { var projects []*managementv1.Project if len(projectNames) == 0 { @@ -1103,8 +1108,8 @@ func WaitForVirtualClusterInstance(ctx context.Context, managementClient kube.In } if virtualClusterInstance.Status.Phase == storagev1.InstanceSleeping { - log.Info("Wait until vcluster wakes up") - defer log.Donef("Successfully woken up vcluster %s", name) + log.Info("Wait until vcluster instance wakes up") + defer log.Donef("virtual cluster %s wakeup successful", name) err := wakeupVCluster(ctx, managementClient, virtualClusterInstance) if err != nil { return nil, fmt.Errorf("error waking up vcluster %s: %s", name, util.GetCause(err)) diff --git a/pkg/platform/sleepmode/sleepmode.go b/pkg/platform/sleepmode/sleepmode.go new file mode 100644 index 0000000000..3a6381cc78 --- /dev/null +++ b/pkg/platform/sleepmode/sleepmode.go @@ -0,0 +1,16 @@ +package sleepmode + +import "github.com/loft-sh/vcluster/pkg/kube" + +const ( + Label = "loft.sh/sleep-mode" + SleepingSinceAnnotation = "sleepmode.loft.sh/sleeping-since" +) + +func IsSleeping(labeled kube.Labeled) bool { + return labeled.GetLabels()[Label] == "true" +} + +func IsInstanceSleeping(annotated kube.Annotated) bool { + return annotated != nil && annotated.GetAnnotations()[SleepingSinceAnnotation] != "" +}