add e2e for deleting klusterlet when the managed cluster was destroyed (#339)

Signed-off-by: zhujian <jiazhu@redhat.com>

pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_cleanup_controller.go

@@ -211,7 +211,8 @@ func (r *klusterletCleanupController) checkConnectivity(ctx context.Context,
 	// if the managed cluster is destroyed, the returned err is TCP timeout or TCP no such host,
 	// the k8s.io/apimachinery/pkg/api/errors.IsTimeout,IsServerTimeout can not match this error
 	if isTCPTimeOutError(err) || isTCPNoSuchHostError(err) {
-		klog.Infof("Check the connectivity, err: %v", err)
+		klog.V(4).Infof("Check the connectivity for klusterlet %s, annotation: %s, err: %v",
+			klusterlet.Name, klusterlet.Annotations, err)
 		if klusterlet.Annotations == nil {
 			klusterlet.Annotations = make(map[string]string, 0)
 		}
@@ -223,13 +224,14 @@ func (r *klusterletCleanupController) checkConnectivity(ctx context.Context,
 		}
 		evictionTime, perr := time.Parse(time.RFC3339, evictionTimeStr)
 		if perr != nil {
-			klog.Infof("Parse eviction time %v error %s", evictionTimeStr, perr)
+			klog.Infof("Parse eviction time %v for klusterlet %s error %s", evictionTimeStr, klusterlet.Name, perr)
 			klusterlet.Annotations[managedResourcesEvictionTimestampAnno] = time.Now().Format(time.RFC3339)
 			return true, err
 		}
 
 		if evictionTime.Add(5 * time.Minute).Before(time.Now()) {
-			klog.Infof("Try to connect managed cluster timed out for 5 minutes, ignore the resources")
+			klog.Infof("Try to connect managed cluster timed out for 5 minutes, klusterlet %s, ignore the resources",
+				klusterlet.Name)
 			// ignore the resources on the managed cluster, return false here
 			return false, nil
 		}

test/e2e/common.go

@@ -453,6 +453,9 @@ func (t *Tester) cleanKlusterletResources(klusterletName, clusterName string) er
 	// clean the klusterlets
 	err := t.OperatorClient.OperatorV1().Klusterlets().Delete(context.TODO(), klusterletName, metav1.DeleteOptions{})
 	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
 		return err
 	}
 
@@ -471,6 +474,9 @@ func (t *Tester) cleanKlusterletResources(klusterletName, clusterName string) er
 	// clean the managed clusters
 	err = t.ClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{})
 	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
 		return err
 	}
 
@@ -725,3 +731,82 @@ func (t *Tester) CheckManagedClusterAddOnStatus(managedClusterNamespace, addOnNa
 
 	return nil
 }
+
+func (t *Tester) DeleteExternalKubeconfigSecret(klusterlet *operatorapiv1.Klusterlet) error {
+	agentNamespace := helpers.AgentNamespace(klusterlet)
+	err := t.KubeClient.CoreV1().Secrets(agentNamespace).Delete(context.TODO(),
+		helpers.ExternalManagedKubeConfig, metav1.DeleteOptions{})
+	if err != nil {
+		klog.Errorf("failed to delete external managed secret in ns %v. %v", agentNamespace, err)
+		return err
+	}
+
+	return nil
+}
+
+func (t *Tester) CreateFakeExternalKubeconfigSecret(klusterlet *operatorapiv1.Klusterlet) error {
+	agentNamespace := helpers.AgentNamespace(klusterlet)
+	klog.Infof("klusterlet: %s/%s, \t, \t agent namespace: %s",
+		klusterlet.Name, klusterlet.Namespace, agentNamespace)
+
+	bsSecret, err := t.KubeClient.CoreV1().Secrets(agentNamespace).Get(context.TODO(),
+		t.bootstrapHubSecret.Name, metav1.GetOptions{})
+	if err != nil {
+		klog.Errorf("failed to get bootstrap secret %v in ns %v. %v", bsSecret, agentNamespace, err)
+		return err
+	}
+
+	// create external-managed-kubeconfig, will use the same cluster to simulate the Hosted mode.
+	secret, err := changeHostOfKubeconfigSecret(*bsSecret, "https://kube-apiserver.i-am-a-fake-server:6443")
+	if err != nil {
+		klog.Errorf("failed to change host of the kubeconfig secret in. %v", err)
+		return err
+	}
+	secret.Namespace = agentNamespace
+	secret.Name = helpers.ExternalManagedKubeConfig
+	secret.ResourceVersion = ""
+
+	_, err = t.KubeClient.CoreV1().Secrets(agentNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
+	if err != nil {
+		klog.Errorf("failed to create external managed secret %v in ns %v. %v", bsSecret, agentNamespace, err)
+		return err
+	}
+
+	return nil
+}
+
+func changeHostOfKubeconfigSecret(secret corev1.Secret, apiServerURL string) (*corev1.Secret, error) {
+	kubeconfigData, ok := secret.Data["kubeconfig"]
+	if !ok {
+		return nil, fmt.Errorf("kubeconfig not found")
+	}
+
+	if kubeconfigData == nil {
+		return nil, fmt.Errorf("failed to get kubeconfig from secret: %s", secret.GetName())
+	}
+
+	kubeconfig, err := clientcmd.Load(kubeconfigData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load kubeconfig from secret: %s", secret.GetName())
+	}
+
+	if len(kubeconfig.Clusters) == 0 {
+		return nil, fmt.Errorf("there is no cluster in kubeconfig from secret: %s", secret.GetName())
+	}
+
+	for k := range kubeconfig.Clusters {
+		kubeconfig.Clusters[k].Server = apiServerURL
+	}
+
+	newKubeconfig, err := clientcmd.Write(*kubeconfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write new kubeconfig to secret: %s", secret.GetName())
+	}
+
+	secret.Data = map[string][]byte{
+		"kubeconfig": newKubeconfig,
+	}
+
+	klog.Info("Set the cluster server URL in %v secret", "apiServerURL", secret.Name, apiServerURL)
+	return &secret, nil
+}

test/e2e/klusterlet_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -194,10 +195,12 @@ var _ = Describe("Create klusterlet CR", func() {
 var _ = Describe("Delete klusterlet CR", func() {
 	var klusterletName string
 	var clusterName string
+	var klusterletNamespace string
 
 	BeforeEach(func() {
 		klusterletName = fmt.Sprintf("e2e-klusterlet-%s", rand.String(6))
 		clusterName = fmt.Sprintf("e2e-managedcluster-%s", rand.String(6))
+		klusterletNamespace = fmt.Sprintf("open-cluster-management-agent-%s", rand.String(6))
 	})
 
 	It("Delete klusterlet CR in Hosted mode without external managed kubeconfig", func() {
@@ -236,4 +239,111 @@ var _ = Describe("Delete klusterlet CR", func() {
 			return fmt.Errorf("klusterlet namespace still exists")
 		}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
 	})
+
+	It("Delete klusterlet CR in Hosted mode when the managed cluster was destroyed", func() {
+		By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
+		klusterlet, err := t.CreateKlusterlet(klusterletName, clusterName, klusterletNamespace, operatorapiv1.InstallModeHosted)
+		Expect(err).ToNot(HaveOccurred())
+
+		By(fmt.Sprintf("waiting for the managed cluster %v to be created", clusterName))
+		Eventually(func() error {
+			_, err := t.GetCreatedManagedCluster(clusterName)
+			return err
+		}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
+
+		By(fmt.Sprintf("check klusterlet %s status", klusterletName))
+		Eventually(func() error {
+			err := t.checkKlusterletStatus(klusterletName, "HubConnectionDegraded",
+				"BootstrapSecretFunctional,HubKubeConfigSecretMissing", metav1.ConditionTrue)
+			return err
+		}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
+
+		By(fmt.Sprintf("approve the created managed cluster %v", clusterName))
+		Eventually(func() error {
+			return t.ApproveCSR(clusterName)
+		}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
+
+		By(fmt.Sprintf("accept the created managed cluster %v", clusterName))
+		Eventually(func() error {
+			return t.AcceptsClient(clusterName)
+		}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
+
+		By(fmt.Sprintf("waiting for the managed cluster %v to be ready", clusterName))
+		Eventually(func() error {
+			return t.CheckManagedClusterStatus(clusterName)
+		}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
+
+		By(fmt.Sprintf("check klusterlet %s status", klusterletName))
+		Eventually(func() error {
+			err := t.checkKlusterletStatus(klusterletName, "HubConnectionDegraded",
+				"HubConnectionFunctional", metav1.ConditionFalse)
+			return err
+		}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
+
+		// change the kubeconfig host of external managed kubeconfig secret to a wrong value
+		// to simulate the managed cluster was destroyed
+		By("Delete external managed kubeconfig", func() {
+			err = t.DeleteExternalKubeconfigSecret(klusterlet)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("Delete managed cluster", func() {
+			// clean the managed clusters
+			err = t.ClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(),
+				clusterName, metav1.DeleteOptions{})
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("Delete klusterlet", func() {
+			// clean the klusterlets
+			err = t.OperatorClient.OperatorV1().Klusterlets().Delete(context.TODO(),
+				klusterletName, metav1.DeleteOptions{})
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("Create a fake external managed kubeconfig", func() {
+			err = t.CreateFakeExternalKubeconfigSecret(klusterlet)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		// in the future, if the eviction can be configured, we can set a short timeout period and
+		// remove the wait and update parts
+		evictionTimestampAnno := "operator.open-cluster-management.io/managed-resources-eviction-timestamp"
+		By("Wait for the eviction timestamp annotation", func() {
+			Eventually(func() error {
+				k, err := t.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(),
+					klusterletName, metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+				_, ok := k.Annotations[evictionTimestampAnno]
+				if !ok {
+					return fmt.Errorf("expected annotation %s does not exist", evictionTimestampAnno)
+				}
+				return nil
+			}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
+		})
+
+		time.Sleep(3 * time.Second) // after the eviction timestamp exists, wait 3 seconds for cache syncing
+		By("Update the eviction timestamp annotation", func() {
+			Eventually(func() error {
+				k, err := t.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(),
+					klusterletName, metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+
+				ta := time.Now().Add(-6 * time.Minute).Format(time.RFC3339)
+				By(fmt.Sprintf("add time %v anno for klusterlet %s", ta, klusterletName))
+				k.Annotations[evictionTimestampAnno] = ta
+				_, err = t.OperatorClient.OperatorV1().Klusterlets().Update(context.TODO(),
+					k, metav1.UpdateOptions{})
+				return err
+			}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
+		})
+
+		By("Check manged cluster and klusterlet can be deleted", func() {
+			Expect(t.cleanKlusterletResources(klusterletName, clusterName)).To(BeNil())
+		})
+	})
 })