test: clean up remains of the e2e tests

Signed-off-by: zhujian7 <zjfqxa@gmail.com>

test/e2e/addon_lease_test.go

@@ -15,6 +15,7 @@ import (
 	certificatesv1 "k8s.io/api/certificates/v1"
 	coordv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -58,6 +59,18 @@ var _ = ginkgo.Describe("Addon Health Check", func() {
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())
 		})
 
+		ginkgo.AfterEach(func() {
+			clusterName := fmt.Sprintf("loopback-e2e-%v", suffix)
+			ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName))
+			err = cleanupManagedCluster(clusterName, suffix)
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			addOnName := fmt.Sprintf("addon-%s", suffix)
+			ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName))
+			err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		})
+
 		ginkgo.It("Should keep addon status to available", func() {
 			ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOn.Name, addOn.Name))
 			_, err := hubClient.CoordinationV1().Leases(addOn.Name).Create(context.TODO(), &coordv1.Lease{
@@ -210,6 +223,18 @@ var _ = ginkgo.Describe("Addon Health Check", func() {
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())
 		})
 
+		ginkgo.AfterEach(func() {
+			clusterName := fmt.Sprintf("loopback-e2e-%v", suffix)
+			ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName))
+			err = cleanupManagedCluster(clusterName, suffix)
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			addOnName := fmt.Sprintf("addon-%s", suffix)
+			ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName))
+			err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		})
+
 		ginkgo.It("Should update addon status to unknow if managed cluster stops to update its lease", func() {
 			ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOn.Name, addOn.Name))
 			_, err := hubClient.CoordinationV1().Leases(addOn.Name).Create(context.TODO(), &coordv1.Lease{
@@ -627,3 +652,105 @@ func createManagedClusterAddOn(managedCluster *clusterv1.ManagedCluster, addOnNa
 	}
 	return hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedCluster.Name).Create(context.TODO(), addOn, metav1.CreateOptions{})
 }
+
+// cleanupManagedCluster deletes the managed cluster related resources, including the namespace
+func cleanupManagedCluster(clusterName, suffix string) error {
+	nsName := fmt.Sprintf("loopback-spoke-%v", suffix)
+
+	if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) {
+		var err error
+		err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), nsName, metav1.DeleteOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("delete ns %q failed: %v", nsName, err)
+	}
+
+	if err := deleteManageClusterAndRelatedNamespace(clusterName); err != nil {
+		return fmt.Errorf("delete manage cluster and related namespace for cluster %q failed: %v", clusterName, err)
+	}
+
+	var (
+		err        error
+		cr         *unstructured.Unstructured
+		crResource = schema.GroupVersionResource{
+			Group:    "rbac.authorization.k8s.io",
+			Version:  "v1",
+			Resource: "clusterroles",
+		}
+	)
+	cr, err = spokeCR(suffix)
+	if err != nil {
+		return err
+	}
+	if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) {
+		var err error
+		err = hubDynamicClient.Resource(crResource).Delete(context.TODO(), cr.GetName(), metav1.DeleteOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("delete cr %q failed: %v", cr.GetName(), err)
+	}
+
+	var (
+		crb         *unstructured.Unstructured
+		crbResource = schema.GroupVersionResource{
+			Group:    "rbac.authorization.k8s.io",
+			Version:  "v1",
+			Resource: "clusterrolebindings",
+		}
+	)
+	crb, err = spokeCRB(nsName, suffix)
+	if err != nil {
+		return err
+	}
+	if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) {
+		var err error
+		err = hubDynamicClient.Resource(crbResource).Delete(context.TODO(), crb.GetName(), metav1.DeleteOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("delete crb %q failed: %v", crb.GetName(), err)
+	}
+
+	return nil
+}
+
+func deleteManageClusterAndRelatedNamespace(clusterName string) error {
+	if err := wait.Poll(1*time.Second, 90*time.Second, func() (bool, error) {
+		var err error
+		err = clusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{})
+		if err != nil && !errors.IsNotFound(err) {
+			return false, err
+		}
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("delete managed cluster %q failed: %v", clusterName, err)
+	}
+
+	// delete namespace created by hub automaticly
+	if err := wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) {
+		var err error
+		err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), clusterName, metav1.DeleteOptions{})
+		// some managed cluster just created, but the csr is not approved,
+		// so there is not a related namespace
+		if err != nil && !errors.IsNotFound(err) {
+			return false, err
+		}
+
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("delete related namespace %q failed: %v", clusterName, err)
+	}
+
+	return nil
+}

test/e2e/loopback_test.go

@@ -544,6 +544,18 @@ var _ = ginkgo.Describe("Loopback registration [development]", func() {
 
 			return false
 		}, 90*time.Second, 1*time.Second).Should(gomega.BeTrue())
+
+		ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName))
+		err = cleanupManagedCluster(clusterName, suffix)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName))
+		err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		ginkgo.By(fmt.Sprintf("Cleaning managed cluster spoken namespace %q", nsName))
+		err = hubClient.CoreV1().Namespaces().Delete(context.TODO(), nsName, metav1.DeleteOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	})
 })
 

test/e2e/webhook_test.go

@@ -78,6 +78,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 				managedCluster, err := clusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), clusterName, metav1.GetOptions{})
 				gomega.Expect(err).ToNot(gomega.HaveOccurred())
 				gomega.Expect(managedCluster.Spec.LeaseDurationSeconds).To(gomega.Equal(int32(60)))
+
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should respond bad request when creating a managed cluster with invalid external server URLs", func() {
@@ -94,6 +96,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					admissionName,
 					invalidURL,
 				)))
+
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should forbid the request when creating an accepted managed cluster by unauthorized user", func() {
@@ -124,6 +128,9 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					saNamespace,
 					sa,
 				)))
+
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should accept the request when creating an accepted managed cluster by authorized user", func() {
@@ -149,6 +156,9 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 				managedCluster := newManagedCluster(clusterName, true, validURL)
 				_, err = authorizedClient.ClusterV1().ManagedClusters().Create(context.TODO(), managedCluster, metav1.CreateOptions{})
 				gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should accept the request when creating a managed cluster with clusterset specified by authorized user", func() {
@@ -189,6 +199,9 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 				}
 				_, err = authorizedClient.ClusterV1().ManagedClusters().Create(context.TODO(), managedCluster, metav1.CreateOptions{})
 				gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should forbid the request when creating a managed cluster with clusterset specified by unauthorized user", func() {
@@ -234,19 +247,27 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					sa,
 					clusterSetName,
 				)))
+
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 		})
 
 		ginkgo.Context("Updating a managed cluster", func() {
 			var clusterName string
-
 			ginkgo.BeforeEach(func() {
+				ginkgo.By(fmt.Sprintf("Creating managed cluster %q", clusterName))
 				clusterName = fmt.Sprintf("webhook-spoke-%s", rand.String(6))
 				managedCluster := newManagedCluster(clusterName, false, validURL)
 				_, err := clusterClient.ClusterV1().ManagedClusters().Create(context.TODO(), managedCluster, metav1.CreateOptions{})
 				gomega.Expect(err).NotTo(gomega.HaveOccurred())
 			})
 
+			ginkgo.AfterEach(func() {
+				ginkgo.By(fmt.Sprintf("Cleaning managed cluster %q", clusterName))
+				gomega.Expect(deleteManageClusterAndRelatedNamespace(clusterName)).ToNot(gomega.HaveOccurred())
+			})
+
 			ginkgo.It("Should not update the LeaseDurationSeconds to zero", func() {
 				ginkgo.By(fmt.Sprintf("try to update managed cluster %q LeaseDurationSeconds to zero", clusterName))
 				err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
@@ -315,6 +336,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					saNamespace,
 					sa,
 				)))
+
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should accept the request when updating the clusterset of a managed cluster by authorized user", func() {
@@ -358,6 +381,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					return err
 				})
 				gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("Should forbid the request when updating the clusterset of a managed cluster by unauthorized user", func() {
@@ -406,6 +431,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					sa,
 					clusterSetName,
 				)))
+
+				gomega.Expect(cleanupClusterClient(saNamespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 		})
 	})
@@ -479,6 +506,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 				managedClusterSetBinding := newManagedClusterSetBinding(namespace, clusterSetName, clusterSetName)
 				_, err = authorizedClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Create(context.TODO(), managedClusterSetBinding, metav1.CreateOptions{})
 				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				gomega.Expect(cleanupClusterClient(namespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 
 			ginkgo.It("should forbid the request when creating a ManagedClusterSetBinding by unauthorized user", func() {
@@ -507,6 +536,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 					sa,
 					clusterSetName,
 				)))
+
+				gomega.Expect(cleanupClusterClient(namespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 		})
 
@@ -554,6 +585,8 @@ var _ = ginkgo.Describe("Admission webhook", func() {
 				}
 				_, err = unauthorizedClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Update(context.TODO(), managedClusterSetBinding, metav1.UpdateOptions{})
 				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				gomega.Expect(cleanupClusterClient(namespace, sa)).ToNot(gomega.HaveOccurred())
 			})
 		})
 	})
@@ -715,3 +748,27 @@ func buildClusterClient(saNamespace, saName string, clusterPolicyRules, policyRu
 	})
 	return unauthorizedClusterClient, err
 }
+
+// cleanupClusterClient delete cluster-scope resource created by func "buildClusterClient",
+// the namespace-scope resources should be deleted by an additional namespace deleting func.
+// It is recommended be invoked as a pair with the func "buildClusterClient"
+func cleanupClusterClient(saNamespace, saName string) error {
+	err := hubClient.CoreV1().ServiceAccounts(saNamespace).Delete(context.TODO(), saName, metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("delete sa %q/%q failed: %v", saNamespace, saName, err)
+	}
+
+	// delete cluster role and cluster role binding if exists
+	clusterRoleName := fmt.Sprintf("%s-clusterrole", saName)
+	err = hubClient.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRoleName, metav1.DeleteOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		return fmt.Errorf("delete cluster role %q failed: %v", clusterRoleName, err)
+	}
+	clusterRoleBindingName := fmt.Sprintf("%s-clusterrolebinding", saName)
+	err = hubClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBindingName, metav1.DeleteOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		return fmt.Errorf("delete cluster role binding %q failed: %v", clusterRoleBindingName, err)
+	}
+
+	return nil
+}