diff --git a/test/e2e/addon_lease_test.go b/test/e2e/addon_lease_test.go index 890500fbb..1dbed5d3e 100644 --- a/test/e2e/addon_lease_test.go +++ b/test/e2e/addon_lease_test.go @@ -15,6 +15,7 @@ import ( certificatesv1 "k8s.io/api/certificates/v1" coordv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -58,6 +59,18 @@ var _ = ginkgo.Describe("Addon Health Check", func() { gomega.Expect(err).ToNot(gomega.HaveOccurred()) }) + ginkgo.AfterEach(func() { + clusterName := fmt.Sprintf("loopback-e2e-%v", suffix) + ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName)) + err = cleanupManagedCluster(clusterName, suffix) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + addOnName := fmt.Sprintf("addon-%s", suffix) + ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName)) + err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + ginkgo.It("Should keep addon status to available", func() { ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOn.Name, addOn.Name)) _, err := hubClient.CoordinationV1().Leases(addOn.Name).Create(context.TODO(), &coordv1.Lease{ @@ -210,6 +223,18 @@ var _ = ginkgo.Describe("Addon Health Check", func() { gomega.Expect(err).ToNot(gomega.HaveOccurred()) }) + ginkgo.AfterEach(func() { + clusterName := fmt.Sprintf("loopback-e2e-%v", suffix) + ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName)) + err = cleanupManagedCluster(clusterName, suffix) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + addOnName := fmt.Sprintf("addon-%s", suffix) + ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName)) + err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + ginkgo.It("Should update addon status to unknow if managed cluster stops to update its lease", func() { ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOn.Name, addOn.Name)) _, err := hubClient.CoordinationV1().Leases(addOn.Name).Create(context.TODO(), &coordv1.Lease{ @@ -627,3 +652,105 @@ func createManagedClusterAddOn(managedCluster *clusterv1.ManagedCluster, addOnNa } return hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedCluster.Name).Create(context.TODO(), addOn, metav1.CreateOptions{}) } + +// cleanupManagedCluster deletes the managed cluster related resources, including the namespace +func cleanupManagedCluster(clusterName, suffix string) error { + nsName := fmt.Sprintf("loopback-spoke-%v", suffix) + + if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), nsName, metav1.DeleteOptions{}) + if err != nil { + return false, err + } + + return true, nil + }); err != nil { + return fmt.Errorf("delete ns %q failed: %v", nsName, err) + } + + if err := deleteManageClusterAndRelatedNamespace(clusterName); err != nil { + return fmt.Errorf("delete manage cluster and related namespace for cluster %q failed: %v", clusterName, err) + } + + var ( + err error + cr *unstructured.Unstructured + crResource = schema.GroupVersionResource{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Resource: "clusterroles", + } + ) + cr, err = spokeCR(suffix) + if err != nil { + return err + } + if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + err = hubDynamicClient.Resource(crResource).Delete(context.TODO(), cr.GetName(), metav1.DeleteOptions{}) + if err != nil { + return false, err + } + + return true, nil + }); err != nil { + return fmt.Errorf("delete cr %q failed: %v", cr.GetName(), err) + } + + var ( + crb *unstructured.Unstructured + crbResource = schema.GroupVersionResource{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Resource: "clusterrolebindings", + } + ) + crb, err = spokeCRB(nsName, suffix) + if err != nil { + return err + } + if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + err = hubDynamicClient.Resource(crbResource).Delete(context.TODO(), crb.GetName(), metav1.DeleteOptions{}) + if err != nil { + return false, err + } + + return true, nil + }); err != nil { + return fmt.Errorf("delete crb %q failed: %v", crb.GetName(), err) + } + + return nil +} + +func deleteManageClusterAndRelatedNamespace(clusterName string) error { + if err := wait.Poll(1*time.Second, 90*time.Second, func() (bool, error) { + var err error + err = clusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return false, err + } + return true, nil + }); err != nil { + return fmt.Errorf("delete managed cluster %q failed: %v", clusterName, err) + } + + // delete namespace created by hub automaticly + if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) + // some managed cluster just created, but the csr is not approved, + // so there is not a related namespace + if err != nil && !errors.IsNotFound(err) { + return false, err + } + + return true, nil + }); err != nil { + return fmt.Errorf("delete related namespace %q failed: %v", clusterName, err) + } + + return nil +} diff --git a/test/e2e/loopback_test.go b/test/e2e/loopback_test.go index 565a91653..523368026 100644 --- a/test/e2e/loopback_test.go +++ b/test/e2e/loopback_test.go @@ -544,6 +544,18 @@ var _ = ginkgo.Describe("Loopback registration [development]", func() { return false }, 90*time.Second, 1*time.Second).Should(gomega.BeTrue()) + + ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName)) + err = cleanupManagedCluster(clusterName, suffix) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName)) + err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("Cleaning managed cluster spoken namespace %q", nsName)) + err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), nsName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) }) }) diff --git a/test/e2e/webhook_test.go b/test/e2e/webhook_test.go index 1ce0c09b1..4b40fc29d 100644 --- a/test/e2e/webhook_test.go +++ b/test/e2e/webhook_test.go @@ -78,6 +78,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { managedCluster, err := clusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Expect(managedCluster.Spec.LeaseDurationSeconds).To(gomega.Equal(int32(60))) + + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should respond bad request when creating a managed cluster with invalid external server URLs", func() { @@ -94,6 +96,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { admissionName, invalidURL, ))) + + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should forbid the request when creating an accepted managed cluster by unauthorized user", func() { @@ -124,6 +128,9 @@ var _ = ginkgo.Describe("Admission webhook", func() { saNamespace, sa, ))) + + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should accept the request when creating an accepted managed cluster by authorized user", func() { @@ -149,6 +156,9 @@ var _ = ginkgo.Describe("Admission webhook", func() { managedCluster := newManagedCluster(clusterName, true, validURL) _, err = authorizedClient.ClusterV1().ManagedClusters().Create(context.TODO(), managedCluster, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should accept the request when creating a managed cluster with clusterset specified by authorized user", func() { @@ -189,6 +199,9 @@ var _ = ginkgo.Describe("Admission webhook", func() { } _, err = authorizedClient.ClusterV1().ManagedClusters().Create(context.TODO(), managedCluster, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should forbid the request when creating a managed cluster with clusterset specified by unauthorized user", func() { @@ -234,19 +247,27 @@ var _ = ginkgo.Describe("Admission webhook", func() { sa, clusterSetName, ))) + + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) }) ginkgo.Context("Updating a managed cluster", func() { var clusterName string - ginkgo.BeforeEach(func() { + ginkgo.By(fmt.Sprintf("Creating managed cluster %q", clusterName)) clusterName = fmt.Sprintf("webhook-spoke-%s", rand.String(6)) managedCluster := newManagedCluster(clusterName, false, validURL) _, err := clusterClient.ClusterV1().ManagedClusters().Create(context.TODO(), managedCluster, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName)) + gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred()) + }) + ginkgo.It("Should not update the LeaseDurationSeconds to zero", func() { ginkgo.By(fmt.Sprintf("try to update managed cluster %q LeaseDurationSeconds to zero", clusterName)) err := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -315,6 +336,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { saNamespace, sa, ))) + + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should accept the request when updating the clusterset of a managed cluster by authorized user", func() { @@ -358,6 +381,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { return err }) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("Should forbid the request when updating the clusterset of a managed cluster by unauthorized user", func() { @@ -406,6 +431,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { sa, clusterSetName, ))) + + gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred()) }) }) }) @@ -479,6 +506,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { managedClusterSetBinding := newManagedClusterSetBinding(namespace, clusterSetName, clusterSetName) _, err = authorizedClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Create(context.TODO(), managedClusterSetBinding, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Expect(cleanupClusterClient(namespace, sa)).ToNot(gomega.HaveOccurred()) }) ginkgo.It("should forbid the request when creating a ManagedClusterSetBinding by unauthorized user", func() { @@ -507,6 +536,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { sa, clusterSetName, ))) + + gomega.Expect(cleanupClusterClient(namespace, sa)).ToNot(gomega.HaveOccurred()) }) }) @@ -554,6 +585,8 @@ var _ = ginkgo.Describe("Admission webhook", func() { } _, err = unauthorizedClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Update(context.TODO(), managedClusterSetBinding, metav1.UpdateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Expect(cleanupClusterClient(namespace, sa)).ToNot(gomega.HaveOccurred()) }) }) }) @@ -715,3 +748,27 @@ func buildClusterClient(saNamespace, saName string, clusterPolicyRules, policyRu }) return unauthorizedClusterClient, err } + +// cleanupClusterClient delete cluster-scope resource created by func "buildClusterClient", +// the namespace-scope resources should be deleted by an additional namespace deleting func. +// It is recommended be invoked as a pair with the func "buildClusterClient" +func cleanupClusterClient(saNamespace, saName string) error { + err := hubClient.CoreV1().ServiceAccounts(saNamespace).Delete(context.TODO(), saName, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("delete sa %q/%q failed: %v", saNamespace, saName, err) + } + + // delete cluster role and cluster role binding if exists + clusterRoleName := fmt.Sprintf("%s-clusterrole", saName) + err = hubClient.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRoleName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("delete cluster role %q failed: %v", clusterRoleName, err) + } + clusterRoleBindingName := fmt.Sprintf("%s-clusterrolebinding", saName) + err = hubClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBindingName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("delete cluster role binding %q failed: %v", clusterRoleBindingName, err) + } + + return nil +}