Feat: helm chart keep fluxcd addon while delete (#3404)

* keep fluxcd addon will delete

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

Makefile

@@ -132,4 +132,5 @@ def-install:
 
 helm-doc-gen: helmdoc
 	readme-generator -v charts/vela-core/values.yaml -r charts/vela-core/README.md
+	cat charts/vela-core/README.md
 	readme-generator -v charts/vela-minimal/values.yaml -r charts/vela-minimal/README.md

charts/vela-core/README.md

@@ -125,3 +125,38 @@ helm install --create-namespace -n vela-system kubevela kubevela/vela-core --wai
 | `kubeClient.burst`           | The burst for reconcile clients, default is 100                                                                            | `100`   |
 
 
+## Uninstalling the Chart
+
+To uninstall/delete the KubeVela helm release
+
+```shell
+$ helm uninstall -n vela-system kubevela
+```
+
+The command removes all the Kubernetes components associated with kubevela and deletes the release.
+
+**Notice**: If you enable fluxcd addon  when install the chart by set `enableFluxcdAddon=true` .Uninstall wouldn't disable the fluxcd addon ,and it will be kept in the cluster.Please guarantee there is no application in cluster use this addon and disable it firstly before uninstall the helm chart. 
+You can use this script to disable all addons.
+```shell
+#! /bin/sh
+addon=$(vela addon list|grep enabled|awk {'print $1'})
+
+fluxcd=false
+for var in ${addon[*]}
+do
+  if [ $var == "fluxcd" ]; then
+      fluxcd=true
+      continue
+      else
+        vela addon disable $var
+  fi
+done
+if [ $fluxcd ]; then
+    vela addon disable fluxcd
+fi
+```
+
+
+
+
+

charts/vela-core/templates/NOTES.txt

@@ -27,3 +27,9 @@ Welcome to use the KubeVela! Enjoy your shipping application journey!
       | . \| |_| || |_) ||  __/ \ V /|  __/| || (_| |
       |_|\_\\__,_||_.__/  \___|  \_/  \___||_| \__,_|
 
+** Please note before uninstalling **
+
+If you enable fluxcd addon when install the chart by set `enableFluxcdAddon=true` .
+Uninstall wouldn't disable the fluxcd addon ,and it will be kept in the cluster.
+Please guarantee there is no application in cluster using this addon and disable it firstly before uninstall the helm chart.
+And you can find the script of one-short disable all addons from the uninstalling section of https://github.com/oam-dev/kubevela/blob/master/charts/vela-core/README.md.

charts/vela-core/templates/addon/fluxcd-def.yaml

@@ -6,6 +6,8 @@ metadata:
     addons.oam.dev/name: fluxcd-def
   name: addon-fluxcd-def
   namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
 spec:
   components:
     - name: fluxc-def-resources

charts/vela-core/templates/addon/fluxcd.yaml

@@ -7,6 +7,8 @@ metadata:
     addons.oam.dev/registry: KubeVela
   name: addon-fluxcd
   namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
 spec:
   components:
     - name: flux-system-namespace

references/cli/uninstall.go

@@ -19,16 +19,20 @@ package cli
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	apierror "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apitypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/apis/types"
+	pkgaddon "github.com/oam-dev/kubevela/pkg/addon"
+	"github.com/oam-dev/kubevela/pkg/oam"
 	"github.com/oam-dev/kubevela/pkg/utils/common"
 	"github.com/oam-dev/kubevela/pkg/utils/helm"
 	"github.com/oam-dev/kubevela/pkg/utils/util"
@@ -41,6 +45,7 @@ type UnInstallArgs struct {
 	Args       common.Args
 	Namespace  string
 	Detail     bool
+	force      bool
 }
 
 // NewUnInstallCommand creates `uninstall` command to uninstall vela core
@@ -61,9 +66,26 @@ func NewUnInstallCommand(c common.Args, order string, ioStreams util.IOStreams)
 			if err != nil {
 				return errors.Wrapf(err, "failed to get kube client")
 			}
+
+			if !unInstallArgs.force {
+				// if use --force flag will skip checking the addon
+				addons, err := checkInstallAddon(kubeClient)
+				if err != nil {
+					return errors.Wrapf(err, "cannot check installed addon")
+				}
+				if len(addons) != 0 {
+					return fmt.Errorf("these addons have been eanbled :%v, please guarantee there is no application using these addons and use `vela unintall -f` unintall include addon ", addons)
+				}
+			}
+
+			// filter out addon related app, these app will be delete by force uninstall
+			// ignore the error, this error cannot be not nil
+			labels, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{{Key: oam.LabelAddonName, Operator: metav1.LabelSelectorOpDoesNotExist}}})
 			var apps v1beta1.ApplicationList
 			err = kubeClient.List(context.Background(), &apps, &client.ListOptions{
-				Namespace: "",
+				Namespace:     "",
+				LabelSelector: labels,
 			})
 			if err != nil {
 				return errors.Wrapf(err, "failed to check app in cluster")
@@ -79,14 +101,21 @@ func NewUnInstallCommand(c common.Args, order string, ioStreams util.IOStreams)
 			if err != nil {
 				return errors.Wrapf(err, "failed to get kube config, You can set KUBECONFIG env or make file ~/.kube/config")
 			}
-			if err := unInstallArgs.helmHelper.UninstallRelease(kubeVelaReleaseName, unInstallArgs.Namespace, restConfig, unInstallArgs.Detail, ioStreams); err != nil {
-				return err
-			}
-			// Clean up vela-system namespace
 			kubeClient, err := c.GetClient()
 			if err != nil {
 				return errors.Wrapf(err, "failed to get kube client")
 			}
+			if unInstallArgs.force {
+				// if use --force disable all addons
+				err := forceDisableAddon(context.Background(), kubeClient)
+				if err != nil {
+					return errors.Wrapf(err, "cannot force disabe all addons")
+				}
+			}
+			if err := unInstallArgs.helmHelper.UninstallRelease(kubeVelaReleaseName, unInstallArgs.Namespace, restConfig, unInstallArgs.Detail, ioStreams); err != nil {
+				return err
+			}
+			// Clean up vela-system namespace
 			if err := deleteNamespace(kubeClient, unInstallArgs.Namespace); err != nil {
 				return err
 			}
@@ -119,6 +148,7 @@ func NewUnInstallCommand(c common.Args, order string, ioStreams util.IOStreams)
 
 	cmd.Flags().StringVarP(&unInstallArgs.Namespace, "namespace", "n", "vela-system", "namespace scope for installing KubeVela Core")
 	cmd.Flags().BoolVarP(&unInstallArgs.Detail, "detail", "d", true, "show detail log of installation")
+	cmd.Flags().BoolVarP(&unInstallArgs.force, "force", "f", false, "force uninstall whole vela include all addons")
 	return cmd
 }
 
@@ -127,3 +157,56 @@ func deleteNamespace(kubeClient client.Client, namespace string) error {
 	ns.Name = namespace
 	return kubeClient.Delete(context.Background(), &ns)
 }
+
+func checkInstallAddon(kubeClient client.Client) ([]string, error) {
+	apps := &v1beta1.ApplicationList{}
+	if err := kubeClient.List(context.Background(), apps, client.InNamespace(types.DefaultKubeVelaNS), client.HasLabels{oam.LabelAddonName}); err != nil {
+		return nil, err
+	}
+	var res []string
+	for _, application := range apps.Items {
+		res = append(res, application.Labels[oam.LabelAddonName])
+	}
+	return res, nil
+}
+
+// forceDisableAddon force delete all enabled addons, fluxcd must be the last one to be deleted
+func forceDisableAddon(ctx context.Context, kubeClient client.Client) error {
+	addons, err := checkInstallAddon(kubeClient)
+	if err != nil {
+		return errors.Wrapf(err, "cannot check the installed addon")
+	}
+	// fluxcd addon should be deleted lastly
+	fluxcdFlag := false
+	for _, addon := range addons {
+		if addon == "fluxcd" {
+			fluxcdFlag = true
+			continue
+		}
+		if err := pkgaddon.DisableAddon(ctx, kubeClient, addon); err != nil {
+			return err
+		}
+	}
+	if fluxcdFlag {
+		timeConsumed := time.Now()
+		var addons []string
+		for {
+			// block 5 minute until other addons have been deleted
+			if time.Now().After(timeConsumed.Add(5 * time.Minute)) {
+				return fmt.Errorf("timeout disable addon, please disable theis addons: %v", addons)
+			}
+			addons, err = checkInstallAddon(kubeClient)
+			if err != nil {
+				return err
+			}
+			if len(addons) == 1 && addons[0] == "fluxcd" {
+				break
+			}
+			time.Sleep(5 * time.Second)
+		}
+		if err := pkgaddon.DisableAddon(ctx, kubeClient, "fluxcd"); err != nil {
+			return err
+		}
+	}
+	return nil
+}

references/cli/uninstall_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/yaml"
+
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
+	"github.com/oam-dev/kubevela/pkg/oam/util"
+)
+
+var _ = Describe("Test Install Command", func() {
+	BeforeEach(func() {
+		fluxcd := v1beta1.Application{}
+		err := yaml.Unmarshal([]byte(fluxcdYaml), &fluxcd)
+		Expect(err).Should(BeNil())
+		Expect(k8sClient.Create(context.Background(), &fluxcd)).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{}))
+		rollout := v1beta1.Application{}
+		err = yaml.Unmarshal([]byte(rolloutYaml), &rollout)
+		Expect(err).Should(BeNil())
+		Expect(k8sClient.Create(context.Background(), &rollout)).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{}))
+	})
+
+	It("Test check addon enabled", func() {
+		addons, err := checkInstallAddon(k8sClient)
+		Expect(err).Should(BeNil())
+		Expect(len(addons)).Should(BeEquivalentTo(2))
+	})
+
+	It("Test disable all addons", func() {
+		err := forceDisableAddon(context.Background(), k8sClient)
+		Expect(err).Should(BeNil())
+		Eventually(func() error {
+			addons, err := checkInstallAddon(k8sClient)
+			if err != nil {
+				return err
+			}
+			if len(addons) != 0 {
+				return fmt.Errorf("%v still exist", addons)
+			}
+			return nil
+		}, 1*time.Minute, 5*time.Second).Should(BeNil())
+	})
+})
+
+var fluxcdYaml = `
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: addon-fluxcd
+  namespace: vela-system
+  labels:
+     addons.oam.dev/name: fluxcd
+spec:
+  components:
+  - name: ns-flux-system
+    properties:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: flux-system
+`
+
+var rolloutYaml = `
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: addon-rollout
+  namespace: vela-system
+  labels:
+     addons.oam.dev/name: rollout
+spec:
+  components:
+  - name: test-ns
+    properties:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: test-ns
+`