diff --git a/e2e/application/application_test.go b/e2e/application/application_test.go index 3e8ac098b..65dc20c60 100644 --- a/e2e/application/application_test.go +++ b/e2e/application/application_test.go @@ -199,7 +199,7 @@ var ApplicationDeleteWithWaitOptions = func(context string, appName string) bool cli := fmt.Sprintf("vela delete %s --wait -y", appName) output, err := e2e.ExecAndTerminate(cli) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(output).To(gomega.ContainSubstring("deleted")) + gomega.Expect(output).To(gomega.ContainSubstring("succeeded")) }) }) } diff --git a/e2e/commonContext.go b/e2e/commonContext.go index 3bc7e2760..e0bcea4b2 100644 --- a/e2e/commonContext.go +++ b/e2e/commonContext.go @@ -160,7 +160,7 @@ var ( cli := fmt.Sprintf("vela delete %s -y", applicationName) output, err := Exec(cli) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(output).To(gomega.ContainSubstring("deleted from namespace")) + gomega.Expect(output).To(gomega.ContainSubstring("succeeded")) }) }) } diff --git a/go.mod b/go.mod index 671b49ca8..42e71078a 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,6 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 - github.com/gosuri/uilive v0.0.4 github.com/gosuri/uitable v0.0.4 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/hcl/v2 v2.12.0 diff --git a/go.sum b/go.sum index bc3f577b6..55aa45a53 100644 --- a/go.sum +++ b/go.sum @@ -1060,8 +1060,6 @@ github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5/g github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= -github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= -github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go index ebe9565a3..5afaed033 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go @@ -75,9 +75,6 @@ const ( const ( // baseWorkflowBackoffWaitTime is the time to wait gc check baseGCBackoffWaitTime = 3000 * time.Millisecond - - // resourceTrackerFinalizer is to delete the resource tracker of the latest app revision. - resourceTrackerFinalizer = "app.oam.dev/resource-tracker-finalizer" ) var ( @@ -366,18 +363,18 @@ func (r *Reconciler) result(err error) *reconcileResult { // We must delete all resource trackers related to an application through finalizer logic. func (r *Reconciler) handleFinalizers(ctx monitorContext.Context, app *v1beta1.Application, handler *AppHandler) (bool, ctrl.Result, error) { if app.ObjectMeta.DeletionTimestamp.IsZero() { - if !meta.FinalizerExists(app, resourceTrackerFinalizer) { + if !meta.FinalizerExists(app, oam.FinalizerResourceTracker) { subCtx := ctx.Fork("handle-finalizers", monitorContext.DurationMetric(func(v float64) { metrics.HandleFinalizersDurationHistogram.WithLabelValues("application", "add").Observe(v) })) defer subCtx.Commit("finish add finalizers") - meta.AddFinalizer(app, resourceTrackerFinalizer) - subCtx.Info("Register new finalizer for application", "finalizer", resourceTrackerFinalizer) + meta.AddFinalizer(app, oam.FinalizerResourceTracker) + subCtx.Info("Register new finalizer for application", "finalizer", oam.FinalizerResourceTracker) endReconcile := !EnableReconcileLoopReduction return r.result(errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer)).end(endReconcile) } } else { - if slices.Contains(app.GetFinalizers(), resourceTrackerFinalizer) { + if slices.Contains(app.GetFinalizers(), oam.FinalizerResourceTracker) { subCtx := ctx.Fork("handle-finalizers", monitorContext.DurationMetric(func(v float64) { metrics.HandleFinalizersDurationHistogram.WithLabelValues("application", "remove").Observe(v) })) @@ -391,7 +388,7 @@ func (r *Reconciler) handleFinalizers(ctx monitorContext.Context, app *v1beta1.A return true, result, err } if rootRT == nil && currentRT == nil && len(historyRTs) == 0 && cvRT == nil { - meta.RemoveFinalizer(app, resourceTrackerFinalizer) + meta.RemoveFinalizer(app, oam.FinalizerResourceTracker) meta.RemoveFinalizer(app, oam.FinalizerOrphanResource) return r.result(errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer)).end(true) } diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go index 197ce497d..92d8045df 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" + "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/oam/testutil" . "github.com/onsi/ginkgo" @@ -117,7 +118,7 @@ var _ = Describe("Test application controller finalizer logic", func() { checkApp = new(v1beta1.Application) Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(1)) - Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(resourceTrackerFinalizer)) + Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(oam.FinalizerResourceTracker)) By("delete this cross workload app") Expect(k8sClient.Delete(ctx, checkApp)).Should(BeNil()) By("delete app will delete resourceTracker") @@ -148,7 +149,7 @@ var _ = Describe("Test application controller finalizer logic", func() { checkApp = new(v1beta1.Application) Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(1)) - Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(resourceTrackerFinalizer)) + Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(oam.FinalizerResourceTracker)) Expect(len(rt.Spec.ManagedResources)).Should(BeEquivalentTo(1)) By("Update the app, set type to normal-worker") checkApp.Spec.Components[0].Type = "normal-worker" diff --git a/pkg/oam/labels.go b/pkg/oam/labels.go index 50678c6b2..1118d6a58 100644 --- a/pkg/oam/labels.go +++ b/pkg/oam/labels.go @@ -179,9 +179,6 @@ const ( // AnnotationDefinitionRevisionName is used to specify the name of DefinitionRevision in component/trait definition AnnotationDefinitionRevisionName = "definitionrevision.oam.dev/name" - // AnnotationResourceTrackerLifeLong is used to identify life-long resourcetracker which should only be recycled when application is deleted - AnnotationResourceTrackerLifeLong = "resourcetracker.oam.dev/life-long" - // AnnotationAddonsName records the name of initializer stored in configMap AnnotationAddonsName = "addons.oam.dev/name" @@ -247,6 +244,10 @@ const ( ResourceTopologyFormatJSON = "json" ) -// FinalizerOrphanResource indicates that the gc process should orphan managed -// resources instead of deleting them -const FinalizerOrphanResource = "app.oam.dev/orphan-resource" +const ( + // FinalizerResourceTracker is the application finalizer for gc + FinalizerResourceTracker = "app.oam.dev/resource-tracker-finalizer" + // FinalizerOrphanResource indicates that the gc process should orphan managed + // resources instead of deleting them + FinalizerOrphanResource = "app.oam.dev/orphan-resource" +) diff --git a/pkg/resourcekeeper/cache.go b/pkg/resourcekeeper/cache.go index 6069c7fa5..35a2d92f4 100644 --- a/pkg/resourcekeeper/cache.go +++ b/pkg/resourcekeeper/cache.go @@ -112,13 +112,22 @@ func (cache *resourceCache) exists(manifest *unstructured.Unstructured) bool { if cache.app == nil { return true } - appKey, controlledBy := apply.GetAppKey(cache.app), apply.GetControlledBy(manifest) - if appKey == controlledBy || (manifest.GetResourceVersion() == "" && !hasOrphanFinalizer(cache.app)) { + return IsResourceManagedByApplication(manifest, cache.app) +} + +// IsResourceManagedByApplication check if resource is managed by application +// If the resource has no ResourceVersion, always return true. +// If the owner label of the resource equals the given app, return true. +// If the sharer label of the resource contains the given app, return true. +// Otherwise, return false. +func IsResourceManagedByApplication(manifest *unstructured.Unstructured, app *v1beta1.Application) bool { + appKey, controlledBy := apply.GetAppKey(app), apply.GetControlledBy(manifest) + if appKey == controlledBy || (manifest.GetResourceVersion() == "" && !hasOrphanFinalizer(app)) { return true } annotations := manifest.GetAnnotations() if annotations == nil || annotations[oam.AnnotationAppSharedBy] == "" { return false } - return apply.ContainsSharer(annotations[oam.AnnotationAppSharedBy], cache.app) + return apply.ContainsSharer(annotations[oam.AnnotationAppSharedBy], app) } diff --git a/pkg/resourcekeeper/gc.go b/pkg/resourcekeeper/gc.go index b25f884b0..5b8808f85 100644 --- a/pkg/resourcekeeper/gc.go +++ b/pkg/resourcekeeper/gc.go @@ -329,7 +329,8 @@ func (h *gcHandler) deleteIndependentComponent(ctx context.Context, mr v1beta1.M return nil } -func (h *gcHandler) deleteSharedManagedResource(ctx context.Context, manifest *unstructured.Unstructured, sharedBy string) error { +// UpdateSharedManagedResourceOwner update owner & sharer labels for managed resource +func UpdateSharedManagedResourceOwner(ctx context.Context, cli client.Client, manifest *unstructured.Unstructured, sharedBy string) error { parts := strings.Split(apply.FirstSharer(sharedBy), "/") appName, appNs := "", metav1.NamespaceDefault if len(parts) == 1 { @@ -342,7 +343,7 @@ func (h *gcHandler) deleteSharedManagedResource(ctx context.Context, manifest *u oam.LabelAppName: appName, oam.LabelAppNamespace: appNs, }) - return h.Client.Update(ctx, manifest) + return cli.Update(ctx, manifest) } func (h *gcHandler) deleteManagedResource(ctx context.Context, mr v1beta1.ManagedResource, rt *v1beta1.ResourceTracker) error { @@ -354,27 +355,33 @@ func (h *gcHandler) deleteManagedResource(ctx context.Context, mr v1beta1.Manage return entry.err } if entry.exists { - _ctx := multicluster.ContextWithClusterName(ctx, mr.Cluster) - if annotations := entry.obj.GetAnnotations(); annotations != nil && annotations[oam.AnnotationAppSharedBy] != "" { - sharedBy := apply.RemoveSharer(annotations[oam.AnnotationAppSharedBy], h.app) - if sharedBy != "" { - if err := h.deleteSharedManagedResource(_ctx, entry.obj, sharedBy); err != nil { - return errors.Wrapf(err, "failed to remove sharer from resource %s", mr.ResourceKey()) - } - return nil + return DeleteManagedResourceInApplication(ctx, h.Client, mr, entry.obj, h.app) + } + return nil +} + +// DeleteManagedResourceInApplication delete managed resource in application +func DeleteManagedResourceInApplication(ctx context.Context, cli client.Client, mr v1beta1.ManagedResource, obj *unstructured.Unstructured, app *v1beta1.Application) error { + _ctx := multicluster.ContextWithClusterName(ctx, mr.Cluster) + if annotations := obj.GetAnnotations(); annotations != nil && annotations[oam.AnnotationAppSharedBy] != "" { + sharedBy := apply.RemoveSharer(annotations[oam.AnnotationAppSharedBy], app) + if sharedBy != "" { + if err := UpdateSharedManagedResourceOwner(_ctx, cli, obj, sharedBy); err != nil { + return errors.Wrapf(err, "failed to remove sharer from resource %s", mr.ResourceKey()) } + return nil } - if mr.SkipGC || hasOrphanFinalizer(h.app) { - if labels := entry.obj.GetLabels(); labels != nil { - delete(labels, oam.LabelAppName) - delete(labels, oam.LabelAppNamespace) - entry.obj.SetLabels(labels) - } - return errors.Wrapf(h.Client.Update(_ctx, entry.obj), "failed to remove owner labels for resource while skipping gc") - } - if err := h.Client.Delete(_ctx, entry.obj); err != nil && !kerrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to delete resource %s", mr.ResourceKey()) + } + if mr.SkipGC || hasOrphanFinalizer(app) { + if labels := obj.GetLabels(); labels != nil { + delete(labels, oam.LabelAppName) + delete(labels, oam.LabelAppNamespace) + obj.SetLabels(labels) } + return errors.Wrapf(cli.Update(_ctx, obj), "failed to remove owner labels for resource while skipping gc") + } + if err := cli.Delete(_ctx, obj); err != nil && !kerrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to delete resource %s", mr.ResourceKey()) } return nil } diff --git a/references/cli/cli.go b/references/cli/cli.go index 9b1b44c5e..df616475b 100644 --- a/references/cli/cli.go +++ b/references/cli/cli.go @@ -96,7 +96,7 @@ func NewCommandWithIOStreams(ioStream util.IOStreams) *cobra.Command { NewTopCommand(commandArgs, "11", ioStream), NewListCommand(commandArgs, "10", ioStream), NewAppStatusCommand(commandArgs, "9", ioStream), - NewDeleteCommand(commandArgs, "7", ioStream), + NewDeleteCommand(f, "7"), NewExecCommand(commandArgs, "6", ioStream), NewPortForwardCommand(commandArgs, "5", ioStream), NewLogsCommand(commandArgs, "4", ioStream), diff --git a/references/cli/delete.go b/references/cli/delete.go index ea85c3e5e..84cb26127 100644 --- a/references/cli/delete.go +++ b/references/cli/delete.go @@ -17,79 +17,384 @@ limitations under the License. package cli import ( - "errors" + "context" "fmt" + "strings" + "time" + "github.com/AlecAivazis/survey/v2" + "github.com/crossplane/crossplane-runtime/pkg/meta" + "github.com/kubevela/pkg/multicluster" + "github.com/kubevela/pkg/util/slices" "github.com/spf13/cobra" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + apitypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/core.oam.dev/condition" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" - common2 "github.com/oam-dev/kubevela/pkg/utils/common" - cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" - "github.com/oam-dev/kubevela/references/common" + velacmd "github.com/oam-dev/kubevela/pkg/cmd" + cmdutil "github.com/oam-dev/kubevela/pkg/cmd/util" + "github.com/oam-dev/kubevela/pkg/oam" + "github.com/oam-dev/kubevela/pkg/resourcekeeper" + "github.com/oam-dev/kubevela/pkg/resourcetracker" + com "github.com/oam-dev/kubevela/references/common" +) + +// DeleteOptions options for vela delete command +type DeleteOptions struct { + AppNames []string + Namespace string + + All bool + Wait bool + Orphan bool + Force bool + Interactive bool + + AssumeYes bool +} + +// Complete . +func (opt *DeleteOptions) Complete(f velacmd.Factory, cmd *cobra.Command, args []string) error { + opt.AppNames = args + opt.Namespace = velacmd.GetNamespace(f, cmd) + opt.AssumeYes = assumeYes + if len(opt.AppNames) > 0 && opt.All { + return fmt.Errorf("application name and --all cannot be both set") + } + if opt.All { + apps := &v1beta1.ApplicationList{} + if err := f.Client().List(cmd.Context(), apps, client.InNamespace(opt.Namespace)); err != nil { + return fmt.Errorf("failed to load application in namespace %s: %w", opt.Namespace, err) + } + opt.AppNames = slices.Map(apps.Items, func(app v1beta1.Application) string { return app.Name }) + } + return nil +} + +// Validate validate if vela delete args are valid +func (opt *DeleteOptions) Validate() error { + switch { + case len(opt.AppNames) == 0 && !opt.All: + return fmt.Errorf("no application provided for deletion") + case len(opt.AppNames) == 0 && opt.All: + return fmt.Errorf("no application found in namespace %s for deletion", opt.Namespace) + case opt.Interactive && (opt.Force || opt.Orphan): + return fmt.Errorf("--interactive cannot be used together with --force and --orphan") + } + return nil +} + +func (opt *DeleteOptions) getDeletingStatus(ctx context.Context, f velacmd.Factory, appKey apitypes.NamespacedName) (done bool, msg string, err error) { + app := &v1beta1.Application{} + err = f.Client().Get(ctx, appKey, app) + switch { + case kerrors.IsNotFound(err): + return true, "", nil + case err != nil: + return false, "", err + case app.DeletionTimestamp == nil: + return false, "application deletion is not handled by apiserver yet", nil + case app.Status.Phase != common.ApplicationDeleting: + return false, "application deletion is not handled by controller yet", nil + default: + if cond := slices.Find(app.Status.Conditions, func(cond condition.Condition) bool { return cond.Reason == condition.ReasonDeleting }); cond != nil { + return false, cond.Message, nil + } + return false, "", nil + } +} + +// DeleteApp delete one application +func (opt *DeleteOptions) DeleteApp(f velacmd.Factory, cmd *cobra.Command, app *v1beta1.Application) error { + ctx := cmd.Context() + + // delete the application interactively + if opt.Interactive { + if err := opt.interactiveDelete(ctx, f, cmd, app); err != nil { + return err + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Exit interactive deletion mode. You can switch to normal mode and continue with automatic deletion.\n") + return nil + } + + if !opt.AssumeYes { + if !NewUserInput().AskBool(fmt.Sprintf("Are you sure to delete the application %s/%s", app.Namespace, app.Name), &UserInputOptions{opt.AssumeYes}) { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "skip deleting appplication %s/%s\n", app.Namespace, app.Name) + return nil + } + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Start deleting appplication %s/%s\n", app.Namespace, app.Name) + + // orphan app + if opt.Orphan { + if err := opt.orphan(ctx, f, app); err != nil { + return err + } + } + + // delete app + if app.DeletionTimestamp == nil { + if err := opt.delete(ctx, f, app); err != nil { + return err + } + } + + // force delete the application + if opt.Force { + if err := com.PrepareToForceDeleteTerraformComponents(ctx, f.Client(), app.Namespace, app.Name); err != nil { + return err + } + if err := opt.forceDelete(ctx, f, app); err != nil { + return err + } + } + + // wait for deletion finished + if opt.Wait { + if err := opt.wait(ctx, f, app); err != nil { + return err + } + } + + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Delete appplication %s/%s succeeded\n", app.Namespace, app.Name) + return nil +} + +func (opt *DeleteOptions) orphan(ctx context.Context, f velacmd.Factory, app *v1beta1.Application) error { + if !slices.Contains(app.GetFinalizers(), oam.FinalizerOrphanResource) { + meta.AddFinalizer(app, oam.FinalizerOrphanResource) + if err := f.Client().Update(ctx, app); err != nil { + return fmt.Errorf("failed to set orphan resource finalizer to application %s/%s: %w", app.Namespace, app.Name, err) + } + } + return nil +} + +func (opt *DeleteOptions) forceDelete(ctx context.Context, f velacmd.Factory, app *v1beta1.Application) error { + return wait.PollImmediate(3*time.Second, 1*time.Minute, func() (done bool, err error) { + err = f.Client().Get(ctx, client.ObjectKeyFromObject(app), app) + if kerrors.IsNotFound(err) { + return true, nil + } + rk, err := resourcekeeper.NewResourceKeeper(ctx, f.Client(), app) + if err != nil { + return false, fmt.Errorf("failed to create resource keeper to run garbage collection: %w", err) + } + if done, _, err = rk.GarbageCollect(ctx); err != nil && !kerrors.IsConflict(err) { + return false, fmt.Errorf("failed to run garbage collect: %w", err) + } + if done { + meta.RemoveFinalizer(app, oam.FinalizerResourceTracker) + meta.RemoveFinalizer(app, oam.FinalizerOrphanResource) + if err = f.Client().Update(ctx, app); err != nil && !kerrors.IsConflict(err) && !kerrors.IsNotFound(err) { + return false, fmt.Errorf("failed to update app finalizer: %w", err) + } + } + return false, nil + }) +} + +func (opt *DeleteOptions) deleteResource(ctx context.Context, f velacmd.Factory, mr v1beta1.ManagedResource, app *v1beta1.Application) error { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(mr.GroupVersionKind()) + if err := f.Client().Get(multicluster.WithCluster(ctx, mr.Cluster), mr.NamespacedName(), obj); err != nil { + return client.IgnoreNotFound(err) + } + if !resourcekeeper.IsResourceManagedByApplication(obj, app) { + return nil + } + return resourcekeeper.DeleteManagedResourceInApplication(ctx, f.Client(), mr, obj, app) +} + +func _getManagedResourceSource(mr v1beta1.ManagedResource) string { + src := "in cluster local" + if mr.Cluster != "" { + src = fmt.Sprintf("in cluster %s", mr.Cluster) + } + if mr.Namespace != "" { + src += fmt.Sprintf(", namespace %s", mr.Namespace) + } + groups := strings.Split(mr.APIVersion, "/") + group := "." + groups[0] + if len(groups) == 0 { + group = "" + } + return fmt.Sprintf("%s%s %s %s", strings.ToLower(mr.Kind), group, mr.Name, src) +} + +func (opt *DeleteOptions) interactiveDelete(ctx context.Context, f velacmd.Factory, cmd *cobra.Command, app *v1beta1.Application) error { + for { + rootRT, currentRT, historyRTs, _, err := resourcetracker.ListApplicationResourceTrackers(ctx, f.Client(), app) + if err != nil { + return fmt.Errorf("failed to get ResourceTrackers for application %s/%s: %w", app.Namespace, app.Name, err) + } + rts := slices.Filter(append(historyRTs, currentRT, rootRT), func(rt *v1beta1.ResourceTracker) bool { return rt != nil }) + rs := map[string]v1beta1.ManagedResource{} + for _, rt := range rts { + for _, mr := range rt.Spec.ManagedResources { + rs[_getManagedResourceSource(mr)] = mr + } + } + var opts []string + for k := range rs { + opts = append(opts, k) + } + if len(opts) == 0 { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "No resources found for application %s/%s\n", app.Namespace, app.Name) + return nil + } + prompt := &survey.Select{ + Message: "Please choose which resource to delete", + Options: append(opts, "exit"), + } + var choice string + if err = survey.AskOne(prompt, &choice); err != nil { + return fmt.Errorf("exit on error: %w", err) + } + if choice == "exit" { + break + } + mr := rs[choice] + if err = opt.deleteResource(ctx, f, mr, app); err != nil { + if !NewUserInput().AskBool(fmt.Sprintf("Error encountered while recycling %s: %s.\nDo you want to skip this error?", choice, err.Error()), &UserInputOptions{AssumeYes: opt.AssumeYes}) { + return fmt.Errorf("deletion aborted") + } + } else { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Successfully recycled resource %s\n", choice) + } + for _, rt := range rts { + if slices.Index(rt.Spec.ManagedResources, func(r v1beta1.ManagedResource) bool { return r.ResourceKey() == mr.ResourceKey() }) >= 0 { + rt.Spec.ManagedResources = slices.Filter(rt.Spec.ManagedResources, func(r v1beta1.ManagedResource) bool { return r.ResourceKey() != mr.ResourceKey() }) + if err = f.Client().Update(ctx, rt); err != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Error encountered when updating ResourceTracker %s: %s\n", rt.Name, err.Error()) + } + } + } + } + return nil +} + +func (opt *DeleteOptions) delete(ctx context.Context, f velacmd.Factory, app *v1beta1.Application) error { + if err := f.Client().Delete(ctx, app); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete application %s/%s: %w", app.Namespace, app.Name, err) + } + return nil +} + +func (opt *DeleteOptions) wait(ctx context.Context, f velacmd.Factory, app *v1beta1.Application) error { + spinner := newTrackingSpinnerWithDelay(fmt.Sprintf("deleting application %s/%s", app.Namespace, app.Name), time.Second) + spinner.Start() + defer spinner.Stop() + return wait.PollImmediate(2*time.Second, 5*time.Minute, func() (done bool, err error) { + var msg string + done, msg, err = opt.getDeletingStatus(ctx, f, client.ObjectKeyFromObject(app)) + applySpinnerNewSuffix(spinner, msg) + return done, err + }) +} + +// Run vela delete +func (opt *DeleteOptions) Run(f velacmd.Factory, cmd *cobra.Command) error { + for _, appName := range opt.AppNames { + app := &v1beta1.Application{} + if err := f.Client().Get(cmd.Context(), apitypes.NamespacedName{Namespace: opt.Namespace, Name: appName}, app); err != nil { + if kerrors.IsNotFound(err) { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "application %s/%s already deleted", opt.Namespace, appName) + return nil + } + return fmt.Errorf("failed to get application %s/%s: %w", opt.Namespace, appName, err) + } + if err := opt.DeleteApp(f, cmd, app); err != nil { + return err + } + } + return nil +} + +var ( + deleteLong = templates.LongDesc(i18n.T(` + Delete applications + + Delete KubeVela applications. KubeVela application deletion is associated + with the recycle of underlying resources. By default, the resources created + by the KubeVela application will be deleted once it is not in use or the + application is deleted. There is garbage-collect policy in KubeVela application + that you can use to configure customized recycle rules. + + This command supports delete application in various modes. + Natively, you can use it like "kubectl delete app ". + In the cases you only want to delete the application but leave the + resources there, you can use the --orphan parameter. + In the cases the server-side controller is uninstalled, or you want to + manually skip some errors in the deletion process (like lack privileges or + handle cluster disconnection), you can use the --force parameter. + `)) + + deleteExample = templates.Examples(i18n.T(` + # Delete an application + vela delete my-app + + # Delete multiple applications in a namespace + vela delete app-1 app-2 -n example + + # Delete all applications in one namespace + vela delete -n example --all + + # Delete application without waiting to be deleted + vela delete my-app --wait=false + + # Delete application without confirmation + vela delete my-app -y + + # Force delete application at client-side + vela delete my-app -f + + # Delete application by orphaning resources and skip recycling them + vela delete my-app --orphan + + # Delete application interactively + vela delete my-app -i + `)) ) // NewDeleteCommand Delete App -func NewDeleteCommand(c common2.Args, order string, ioStreams cmdutil.IOStreams) *cobra.Command { +func NewDeleteCommand(f velacmd.Factory, order string) *cobra.Command { + o := &DeleteOptions{ + Wait: true, + } cmd := &cobra.Command{ - Use: "delete APP_NAME1 [APP_NAME2 APP_NAME3...]", + Use: "delete", DisableFlagsInUseLine: true, - Short: "Delete an application", - Long: "Delete an application.", + Short: i18n.T("Delete an application"), + Long: deleteLong, + Example: deleteExample, Annotations: map[string]string{ types.TagCommandOrder: order, types.TagCommandType: types.TypeApp, }, - Example: "vela delete frontend", - } - cmd.SetOut(ioStreams.Out) - - o := &common.DeleteOptions{ - C: c, - } - cmd.RunE = func(cmd *cobra.Command, args []string) error { - namespace, err := GetFlagNamespaceOrEnv(cmd, c) - if err != nil { - return err - } - o.Namespace = namespace - newClient, err := c.GetClient() - if err != nil { - return err - } - o.Client = newClient - - if len(args) < 1 { - return errors.New("must specify name for the app") - } - if o.Wait, err = cmd.Flags().GetBool("wait"); err != nil { - return err - } - if o.ForceDelete, err = cmd.Flags().GetBool("force"); err != nil { - return err - } - if o.Orphan, err = cmd.Flags().GetBool("orphan"); err != nil { - return err - } - for _, app := range args { - o.AppName = app - userInput := NewUserInput() - if !assumeYes { - userConfirmation := userInput.AskBool(fmt.Sprintf("Do you want to delete the application %s from namespace %s", o.AppName, o.Namespace), &UserInputOptions{assumeYes}) - if !userConfirmation { - return fmt.Errorf("stopping Deleting") - } - } - if err = o.DeleteApp(ioStreams); err != nil { - return err - } - ioStreams.Info(green.Sprintf("app \"%s\" deleted from namespace \"%s\"", o.AppName, o.Namespace)) - } - return nil + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(f, cmd, args)) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run(f, cmd)) + }, } - cmd.PersistentFlags().BoolVarP(&o.Wait, "wait", "w", false, "wait util the application is deleted completely") - cmd.PersistentFlags().BoolVarP(&o.ForceDelete, "force", "f", false, "force to delete the application") - cmd.PersistentFlags().BoolVarP(&o.Orphan, "orphan", "o", false, "delete the application and orphan managed resources") - addNamespaceAndEnvArg(cmd) - return cmd + cmd.PersistentFlags().BoolVarP(&o.Wait, "wait", "w", o.Wait, "wait util the application is deleted completely") + cmd.PersistentFlags().BoolVarP(&o.All, "all", "", o.All, "delete all the application under the given namespace") + cmd.PersistentFlags().BoolVarP(&o.Orphan, "orphan", "o", o.Orphan, "delete the application and orphan managed resources") + cmd.PersistentFlags().BoolVarP(&o.Force, "force", "f", o.Force, "force delete the application") + cmd.PersistentFlags().BoolVarP(&o.Interactive, "interactive", "i", o.Interactive, "delete the application interactively") + + return velacmd.NewCommandBuilder(f, cmd). + WithNamespaceFlag(). + WithResponsiveWriter(). + Build() } diff --git a/references/common/application.go b/references/common/application.go index ecaf5c268..a78b4f02f 100644 --- a/references/common/application.go +++ b/references/common/application.go @@ -21,26 +21,18 @@ import ( "context" j "encoding/json" "fmt" - "time" - "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/fatih/color" - "github.com/gosuri/uilive" terraformapi "github.com/oam-dev/terraform-controller/api/v1beta2" - "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/serializer/json" apitypes "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/strings/slices" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" - corev1alpha2 "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" corev1beta1 "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/oam" - "github.com/oam-dev/kubevela/pkg/resourcekeeper" "github.com/oam-dev/kubevela/pkg/utils" "github.com/oam-dev/kubevela/pkg/utils/apply" "github.com/oam-dev/kubevela/pkg/utils/common" @@ -50,13 +42,6 @@ import ( "github.com/oam-dev/kubevela/references/appfile/template" ) -const ( - resourceTrackerFinalizer = "app.oam.dev/resource-tracker-finalizer" - // legacyOnlyRevisionFinalizer is to delete all resource trackers of app revisions which may be used - // out of the domain of app controller, e.g., AppRollout controller. - legacyOnlyRevisionFinalizer = "app.oam.dev/only-revision-finalizer" -) - // AppfileOptions is some configuration that modify options for an Appfile type AppfileOptions struct { Kubecli client.Client @@ -72,171 +57,8 @@ type BuildResult struct { scopes []oam.Object } -// Option is option work with dashboard api server -type Option struct { - // Optional filter, if specified, only components in such app will be listed - AppName string - - Namespace string -} - -// DeleteOptions is options for delete -type DeleteOptions struct { - Namespace string - AppName string - CompName string - Client client.Client - C common.Args - - Wait bool - ForceDelete bool - Orphan bool -} - -// DeleteApp will delete app including server side -func (o *DeleteOptions) DeleteApp(io cmdutil.IOStreams) error { - if o.Orphan { - if err := o.OrphanApp(); err != nil { - return err - } - } - if o.ForceDelete { - return o.ForceDeleteApp(io) - } - if o.Wait { - return o.WaitUntilDeleteApp(io) - } - return o.DeleteAppWithoutDoubleCheck(io) -} - -// OrphanApp set orphan finalizer to app -func (o *DeleteOptions) OrphanApp() error { - app, ctx := &corev1beta1.Application{}, context.Background() - if err := o.Client.Get(ctx, client.ObjectKey{Name: o.AppName, Namespace: o.Namespace}, app); err != nil { - return err - } - if !slices.Contains(app.GetFinalizers(), oam.FinalizerOrphanResource) { - meta.AddFinalizer(app, oam.FinalizerOrphanResource) - return o.Client.Update(ctx, app) - } - return nil -} - -// ForceDeleteApp force delete the application -func (o *DeleteOptions) ForceDeleteApp(io cmdutil.IOStreams) error { - ctx := context.Background() - err := o.DeleteAppWithoutDoubleCheck(io) - if err != nil { - return err - } - app := new(corev1beta1.Application) - err = o.Client.Get(ctx, client.ObjectKey{Name: o.AppName, Namespace: o.Namespace}, app) - if err != nil { - return client.IgnoreNotFound(err) - } - io.Info("force deleted the resources created by application") - err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (done bool, err error) { - err = o.Client.Get(ctx, client.ObjectKeyFromObject(app), app) - if apierrors.IsNotFound(err) { - return true, nil - } - rk, err := resourcekeeper.NewResourceKeeper(ctx, o.Client, app) - if err != nil { - return false, errors.Wrapf(err, "failed to create resource keeper to run garbage collection") - } - if done, _, err = rk.GarbageCollect(ctx); err != nil && !apierrors.IsConflict(err) { - return false, errors.Wrapf(err, "failed to run garbage collect") - } - if done { - meta.RemoveFinalizer(app, resourceTrackerFinalizer) - meta.RemoveFinalizer(app, legacyOnlyRevisionFinalizer) - if err = o.Client.Update(ctx, app); err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { - return false, errors.Wrapf(err, "failed to update app finalizer") - } - } - return false, nil - }) - if err != nil { - io.Info("successfully cleanup the resources created by application, but fail to delete the application") - return err - } - return nil -} - -// WaitUntilDeleteApp will wait until the application is completely deleted -func (o *DeleteOptions) WaitUntilDeleteApp(io cmdutil.IOStreams) error { - tryCnt, startTime := 0, time.Now() - writer := uilive.New() - writer.Start() - defer writer.Stop() - - io.Infof(color.New(color.FgYellow).Sprintf("waiting for delete the application \"%s\"...\n", o.AppName)) - err := wait.PollImmediate(2*time.Second, 5*time.Minute, func() (done bool, err error) { - tryCnt++ - fmt.Fprintf(writer, "try to delete the application for the %d time, wait a total of %f s\n", tryCnt, time.Since(startTime).Seconds()) - err = o.DeleteAppWithoutDoubleCheck(io) - if err != nil { - fmt.Printf("Failed delete Application \"%s\": %s\n", o.AppName, err.Error()) - return false, nil - } - app := new(corev1beta1.Application) - err = o.Client.Get(context.Background(), client.ObjectKey{Name: o.AppName, Namespace: o.Namespace}, app) - if apierrors.IsNotFound(err) { - return true, nil - } - return false, nil - }) - if err != nil { - io.Info("waiting for the application to be deleted timed out, please try again") - return err - } - return nil -} - -// DeleteAppWithoutDoubleCheck delete application without double check -func (o *DeleteOptions) DeleteAppWithoutDoubleCheck(io cmdutil.IOStreams) error { - ctx := context.Background() - - if o.ForceDelete { - if err := prepareToForceDeleteTerraformComponents(ctx, o.Client, o.Namespace, o.AppName); err != nil { - return err - } - } - - var app = new(corev1beta1.Application) - err := o.Client.Get(ctx, client.ObjectKey{Name: o.AppName, Namespace: o.Namespace}, app) - if err != nil { - if apierrors.IsNotFound(err) { - return fmt.Errorf("app %s not exist or deleted in namespace %s", o.AppName, o.Namespace) - } - return fmt.Errorf("delete application err: %w", err) - } - - err = o.Client.Delete(ctx, app) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("delete application err: %w", err) - } - - for _, cmp := range app.Spec.Components { - healthScopeName, ok := cmp.Scopes[api.DefaultHealthScopeKey] - if ok { - var healthScope corev1alpha2.HealthScope - if err := o.Client.Get(ctx, client.ObjectKey{Namespace: o.Namespace, Name: healthScopeName}, &healthScope); err != nil { - if apierrors.IsNotFound(err) { - continue - } - return fmt.Errorf("delete health scope %s err: %w", healthScopeName, err) - } - if err = o.Client.Delete(ctx, &healthScope); err != nil { - return fmt.Errorf("delete health scope %s err: %w", healthScopeName, err) - } - } - } - return nil -} - -// prepareToForceDeleteTerraformComponents sets Terraform typed Component to force-delete mode -func prepareToForceDeleteTerraformComponents(ctx context.Context, k8sClient client.Client, namespace, name string) error { +// PrepareToForceDeleteTerraformComponents sets Terraform typed Component to force-delete mode +func PrepareToForceDeleteTerraformComponents(ctx context.Context, k8sClient client.Client, namespace, name string) error { var ( app = new(corev1beta1.Application) forceDelete = true @@ -272,37 +94,6 @@ func prepareToForceDeleteTerraformComponents(ctx context.Context, k8sClient clie return nil } -// DeleteComponent will delete one component including server side. -func (o *DeleteOptions) DeleteComponent(io cmdutil.IOStreams) error { - var err error - if o.AppName == "" { - return errors.New("app name is required") - } - app, err := appfile.LoadApplication(o.Namespace, o.AppName, o.C) - if err != nil { - return err - } - - if len(appfile.GetComponents(app)) <= 1 { - return o.DeleteApp(io) - } - - // Remove component from local appfile - if err := appfile.RemoveComponent(app, o.CompName); err != nil { - return err - } - - // Remove component from appConfig in k8s cluster - ctx := context.Background() - - if err := o.Client.Update(ctx, app); err != nil { - return err - } - - // It's the server responsibility to GC component - return nil -} - // LoadAppFile will load vela appfile from remote URL or local file system. func LoadAppFile(pathOrURL string) (*api.AppFile, error) { body, err := utils.ReadRemoteOrLocalPath(pathOrURL, false) diff --git a/references/common/application_test.go b/references/common/application_test.go index f2e384031..e8da691d5 100644 --- a/references/common/application_test.go +++ b/references/common/application_test.go @@ -167,7 +167,7 @@ func TestPrepareToForceDeleteTerraformComponents(t *testing.T) { for name, tc := range testcases { t.Run(name, func(t *testing.T) { - err := prepareToForceDeleteTerraformComponents(ctx, tc.args.k8sClient, tc.args.namespace, tc.args.name) + err := PrepareToForceDeleteTerraformComponents(ctx, tc.args.k8sClient, tc.args.namespace, tc.args.name) if err != nil { assert.NotEmpty(t, tc.want.errMsg) assert.Contains(t, err.Error(), tc.want.errMsg)