From c4d3d9e90a879f7ea6643711fbb4de66e57b8ae0 Mon Sep 17 00:00:00 2001 From: Jian Zhu <36154065+zhujian7@users.noreply.github.com> Date: Fri, 31 Mar 2023 16:41:18 +0800 Subject: [PATCH] add e2e for deleting klusterlet when the managed cluster was destroyed (#339) Signed-off-by: zhujian --- .../klusterlet_cleanup_controller.go | 8 +- test/e2e/common.go | 85 ++++++++++++++ test/e2e/klusterlet_test.go | 110 ++++++++++++++++++ 3 files changed, 200 insertions(+), 3 deletions(-) diff --git a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_cleanup_controller.go b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_cleanup_controller.go index 5e8749547..809e005e8 100644 --- a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_cleanup_controller.go +++ b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_cleanup_controller.go @@ -211,7 +211,8 @@ func (r *klusterletCleanupController) checkConnectivity(ctx context.Context, // if the managed cluster is destroyed, the returned err is TCP timeout or TCP no such host, // the k8s.io/apimachinery/pkg/api/errors.IsTimeout,IsServerTimeout can not match this error if isTCPTimeOutError(err) || isTCPNoSuchHostError(err) { - klog.Infof("Check the connectivity, err: %v", err) + klog.V(4).Infof("Check the connectivity for klusterlet %s, annotation: %s, err: %v", + klusterlet.Name, klusterlet.Annotations, err) if klusterlet.Annotations == nil { klusterlet.Annotations = make(map[string]string, 0) } @@ -223,13 +224,14 @@ func (r *klusterletCleanupController) checkConnectivity(ctx context.Context, } evictionTime, perr := time.Parse(time.RFC3339, evictionTimeStr) if perr != nil { - klog.Infof("Parse eviction time %v error %s", evictionTimeStr, perr) + klog.Infof("Parse eviction time %v for klusterlet %s error %s", evictionTimeStr, klusterlet.Name, perr) klusterlet.Annotations[managedResourcesEvictionTimestampAnno] = time.Now().Format(time.RFC3339) return true, err } if evictionTime.Add(5 * time.Minute).Before(time.Now()) { - klog.Infof("Try to connect managed cluster timed out for 5 minutes, ignore the resources") + klog.Infof("Try to connect managed cluster timed out for 5 minutes, klusterlet %s, ignore the resources", + klusterlet.Name) // ignore the resources on the managed cluster, return false here return false, nil } diff --git a/test/e2e/common.go b/test/e2e/common.go index cfb301b20..60874ee5f 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -453,6 +453,9 @@ func (t *Tester) cleanKlusterletResources(klusterletName, clusterName string) er // clean the klusterlets err := t.OperatorClient.OperatorV1().Klusterlets().Delete(context.TODO(), klusterletName, metav1.DeleteOptions{}) if err != nil { + if errors.IsNotFound(err) { + return nil + } return err } @@ -471,6 +474,9 @@ func (t *Tester) cleanKlusterletResources(klusterletName, clusterName string) er // clean the managed clusters err = t.ClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) if err != nil { + if errors.IsNotFound(err) { + return nil + } return err } @@ -725,3 +731,82 @@ func (t *Tester) CheckManagedClusterAddOnStatus(managedClusterNamespace, addOnNa return nil } + +func (t *Tester) DeleteExternalKubeconfigSecret(klusterlet *operatorapiv1.Klusterlet) error { + agentNamespace := helpers.AgentNamespace(klusterlet) + err := t.KubeClient.CoreV1().Secrets(agentNamespace).Delete(context.TODO(), + helpers.ExternalManagedKubeConfig, metav1.DeleteOptions{}) + if err != nil { + klog.Errorf("failed to delete external managed secret in ns %v. %v", agentNamespace, err) + return err + } + + return nil +} + +func (t *Tester) CreateFakeExternalKubeconfigSecret(klusterlet *operatorapiv1.Klusterlet) error { + agentNamespace := helpers.AgentNamespace(klusterlet) + klog.Infof("klusterlet: %s/%s, \t, \t agent namespace: %s", + klusterlet.Name, klusterlet.Namespace, agentNamespace) + + bsSecret, err := t.KubeClient.CoreV1().Secrets(agentNamespace).Get(context.TODO(), + t.bootstrapHubSecret.Name, metav1.GetOptions{}) + if err != nil { + klog.Errorf("failed to get bootstrap secret %v in ns %v. %v", bsSecret, agentNamespace, err) + return err + } + + // create external-managed-kubeconfig, will use the same cluster to simulate the Hosted mode. + secret, err := changeHostOfKubeconfigSecret(*bsSecret, "https://kube-apiserver.i-am-a-fake-server:6443") + if err != nil { + klog.Errorf("failed to change host of the kubeconfig secret in. %v", err) + return err + } + secret.Namespace = agentNamespace + secret.Name = helpers.ExternalManagedKubeConfig + secret.ResourceVersion = "" + + _, err = t.KubeClient.CoreV1().Secrets(agentNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + klog.Errorf("failed to create external managed secret %v in ns %v. %v", bsSecret, agentNamespace, err) + return err + } + + return nil +} + +func changeHostOfKubeconfigSecret(secret corev1.Secret, apiServerURL string) (*corev1.Secret, error) { + kubeconfigData, ok := secret.Data["kubeconfig"] + if !ok { + return nil, fmt.Errorf("kubeconfig not found") + } + + if kubeconfigData == nil { + return nil, fmt.Errorf("failed to get kubeconfig from secret: %s", secret.GetName()) + } + + kubeconfig, err := clientcmd.Load(kubeconfigData) + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig from secret: %s", secret.GetName()) + } + + if len(kubeconfig.Clusters) == 0 { + return nil, fmt.Errorf("there is no cluster in kubeconfig from secret: %s", secret.GetName()) + } + + for k := range kubeconfig.Clusters { + kubeconfig.Clusters[k].Server = apiServerURL + } + + newKubeconfig, err := clientcmd.Write(*kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to write new kubeconfig to secret: %s", secret.GetName()) + } + + secret.Data = map[string][]byte{ + "kubeconfig": newKubeconfig, + } + + klog.Info("Set the cluster server URL in %v secret", "apiServerURL", secret.Name, apiServerURL) + return &secret, nil +} diff --git a/test/e2e/klusterlet_test.go b/test/e2e/klusterlet_test.go index 8f2f6f534..602846d7f 100644 --- a/test/e2e/klusterlet_test.go +++ b/test/e2e/klusterlet_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -194,10 +195,12 @@ var _ = Describe("Create klusterlet CR", func() { var _ = Describe("Delete klusterlet CR", func() { var klusterletName string var clusterName string + var klusterletNamespace string BeforeEach(func() { klusterletName = fmt.Sprintf("e2e-klusterlet-%s", rand.String(6)) clusterName = fmt.Sprintf("e2e-managedcluster-%s", rand.String(6)) + klusterletNamespace = fmt.Sprintf("open-cluster-management-agent-%s", rand.String(6)) }) It("Delete klusterlet CR in Hosted mode without external managed kubeconfig", func() { @@ -236,4 +239,111 @@ var _ = Describe("Delete klusterlet CR", func() { return fmt.Errorf("klusterlet namespace still exists") }, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed()) }) + + It("Delete klusterlet CR in Hosted mode when the managed cluster was destroyed", func() { + By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName)) + klusterlet, err := t.CreateKlusterlet(klusterletName, clusterName, klusterletNamespace, operatorapiv1.InstallModeHosted) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("waiting for the managed cluster %v to be created", clusterName)) + Eventually(func() error { + _, err := t.GetCreatedManagedCluster(clusterName) + return err + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed()) + + By(fmt.Sprintf("check klusterlet %s status", klusterletName)) + Eventually(func() error { + err := t.checkKlusterletStatus(klusterletName, "HubConnectionDegraded", + "BootstrapSecretFunctional,HubKubeConfigSecretMissing", metav1.ConditionTrue) + return err + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed()) + + By(fmt.Sprintf("approve the created managed cluster %v", clusterName)) + Eventually(func() error { + return t.ApproveCSR(clusterName) + }, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed()) + + By(fmt.Sprintf("accept the created managed cluster %v", clusterName)) + Eventually(func() error { + return t.AcceptsClient(clusterName) + }, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed()) + + By(fmt.Sprintf("waiting for the managed cluster %v to be ready", clusterName)) + Eventually(func() error { + return t.CheckManagedClusterStatus(clusterName) + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed()) + + By(fmt.Sprintf("check klusterlet %s status", klusterletName)) + Eventually(func() error { + err := t.checkKlusterletStatus(klusterletName, "HubConnectionDegraded", + "HubConnectionFunctional", metav1.ConditionFalse) + return err + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed()) + + // change the kubeconfig host of external managed kubeconfig secret to a wrong value + // to simulate the managed cluster was destroyed + By("Delete external managed kubeconfig", func() { + err = t.DeleteExternalKubeconfigSecret(klusterlet) + Expect(err).ToNot(HaveOccurred()) + }) + + By("Delete managed cluster", func() { + // clean the managed clusters + err = t.ClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), + clusterName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + By("Delete klusterlet", func() { + // clean the klusterlets + err = t.OperatorClient.OperatorV1().Klusterlets().Delete(context.TODO(), + klusterletName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + By("Create a fake external managed kubeconfig", func() { + err = t.CreateFakeExternalKubeconfigSecret(klusterlet) + Expect(err).ToNot(HaveOccurred()) + }) + + // in the future, if the eviction can be configured, we can set a short timeout period and + // remove the wait and update parts + evictionTimestampAnno := "operator.open-cluster-management.io/managed-resources-eviction-timestamp" + By("Wait for the eviction timestamp annotation", func() { + Eventually(func() error { + k, err := t.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(), + klusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + _, ok := k.Annotations[evictionTimestampAnno] + if !ok { + return fmt.Errorf("expected annotation %s does not exist", evictionTimestampAnno) + } + return nil + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed()) + }) + + time.Sleep(3 * time.Second) // after the eviction timestamp exists, wait 3 seconds for cache syncing + By("Update the eviction timestamp annotation", func() { + Eventually(func() error { + k, err := t.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(), + klusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + ta := time.Now().Add(-6 * time.Minute).Format(time.RFC3339) + By(fmt.Sprintf("add time %v anno for klusterlet %s", ta, klusterletName)) + k.Annotations[evictionTimestampAnno] = ta + _, err = t.OperatorClient.OperatorV1().Klusterlets().Update(context.TODO(), + k, metav1.UpdateOptions{}) + return err + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed()) + }) + + By("Check manged cluster and klusterlet can be deleted", func() { + Expect(t.cleanKlusterletResources(klusterletName, clusterName)).To(BeNil()) + }) + }) })