make e2e test cases ran as canary check (#105)

Signed-off-by: Yang Le <yangle@redhat.com>

README.md

@@ -152,6 +152,30 @@ Undeploy placement controller from the cluster.
 make undeploy-hub
 ```
 
+### Run e2e test cases as sanity check on an existing environment
+
+In order to verify the `Placement` API on an existing environment with placement controller installed and well configured, you are able to run the e2e test cases as sanity check by following the steps below.
+
+Build the binary of the e2e test cases
+```
+make build-e2e
+```
+
+And then run the e2e test cases against an existing environment.
+```
+./e2e.test --ginkgo.v --ginkgo.label-filter=sanity-check -hub-kubeconfig=/path/to/file
+```
+
+In an environment that has already had the `global` clusterset created, you can skip the creation of the `global` clusterset during testing.
+```
+./e2e.test --ginkgo.v --ginkgo.label-filter=sanity-check -hub-kubeconfig=/path/to/file -create-global-clusterset=false
+```
+
+Since the e2e test cases create fake ManagedClusters (without agent installed) during testing, in a full featured OCM environment (with registration controller running on the hub), a taint `cluster.open-cluster-management.io/unreachable` will be added to those fake ManagedClusters automatically. You have to tolerate this taint when running e2e test cases in such an environment.
+```
+./e2e.test --ginkgo.v --ginkgo.label-filter=sanity-check -hub-kubeconfig=/path/to/file -tolerate-unreachable-taint
+```
+
 <!--
 ## XXX References
 

test/e2e/placement_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/rand"
 
 	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
@@ -20,10 +21,15 @@ import (
 const (
 	clusterSetLabel          = "cluster.open-cluster-management.io/clusterset"
 	placementLabel           = "cluster.open-cluster-management.io/placement"
+	e2eTestLabel             = "created-by"
+	e2eTestLabelValue        = "placement-e2e-test"
 	maxNumOfClusterDecisions = 100
 )
 
-var _ = ginkgo.Describe("Placement", func() {
+// Test cases with lable "sanity-check" could be ran as sanity check on an existing enviroment with
+// placement controller installed and well configured . Resource leftovers should be cleaned up on
+// the hub cluster.
+var _ = ginkgo.Describe("Placement", ginkgo.Label("sanity-check"), func() {
 	var namespace string
 	var placementName string
 	var clusterSet1Name string
@@ -42,6 +48,9 @@ var _ = ginkgo.Describe("Placement", func() {
 		ns := &corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: namespace,
+				Labels: map[string]string{
+					e2eTestLabel: e2eTestLabelValue,
+				},
 			},
 		}
 		_, err := kubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
@@ -49,17 +58,30 @@ var _ = ginkgo.Describe("Placement", func() {
 	})
 
 	ginkgo.AfterEach(func() {
-		ginkgo.By("Delete managedclusterset")
-		clusterClient.ClusterV1beta2().ManagedClusterSets().Delete(context.Background(), clusterSet1Name, metav1.DeleteOptions{})
-		clusterClient.ClusterV1beta2().ManagedClusterSets().Delete(context.Background(), clusterSetGlobal, metav1.DeleteOptions{})
+		var errs []error
+		ginkgo.By("Delete managedclustersets")
+		err := clusterClient.ClusterV1beta2().ManagedClusterSets().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
+			LabelSelector: e2eTestLabel + "=" + e2eTestLabelValue,
+		})
+		if err != nil {
+			errs = append(errs, err)
+		}
 
 		ginkgo.By("Delete managedclusters")
-		clusterClient.ClusterV1().ManagedClusters().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
-			LabelSelector: clusterSetLabel + "=" + clusterSet1Name,
+		err = clusterClient.ClusterV1().ManagedClusters().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
+			LabelSelector: e2eTestLabel + "=" + e2eTestLabelValue,
 		})
+		if err != nil {
+			errs = append(errs, err)
+		}
+
+		ginkgo.By("Delete namespace")
+		err = kubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
+		if err != nil {
+			errs = append(errs, err)
+		}
+		gomega.Expect(utilerrors.NewAggregate(errs)).ToNot(gomega.HaveOccurred())
 
-		err := kubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
-		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	})
 
 	assertPlacementDecisionCreated := func(placement *clusterapiv1beta1.Placement) {
@@ -142,6 +164,9 @@ var _ = ginkgo.Describe("Placement", func() {
 		clusterset := &clusterapiv1beta2.ManagedClusterSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: clusterSetName,
+				Labels: map[string]string{
+					e2eTestLabel: e2eTestLabelValue,
+				},
 			},
 		}
 		if matchLabel != nil {
@@ -162,6 +187,9 @@ var _ = ginkgo.Describe("Placement", func() {
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace: namespace,
 				Name:      clusterSetName,
+				Labels: map[string]string{
+					e2eTestLabel: e2eTestLabelValue,
+				},
 			},
 			Spec: clusterapiv1beta2.ManagedClusterSetBindingSpec{
 				ClusterSet: clusterSetName,
@@ -180,12 +208,16 @@ var _ = ginkgo.Describe("Placement", func() {
 	assertCreatingClusters := func(clusterSetName string, num int) {
 		ginkgo.By(fmt.Sprintf("Create %d clusters", num))
 		for i := 0; i < num; i++ {
+			labels := map[string]string{
+				e2eTestLabel: e2eTestLabelValue,
+			}
+			if len(clusterSetName) > 0 {
+				labels[clusterSetLabel] = clusterSetName
+			}
 			cluster := &clusterapiv1.ManagedCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					GenerateName: "cluster-",
-					Labels: map[string]string{
-						clusterSetLabel: clusterSetName,
-					},
+					Labels:       labels,
 				},
 			}
 			_, err = clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
@@ -199,11 +231,23 @@ var _ = ginkgo.Describe("Placement", func() {
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace: namespace,
 				Name:      name,
+				Labels: map[string]string{
+					e2eTestLabel: e2eTestLabelValue,
+				},
 			},
 			Spec: clusterapiv1beta1.PlacementSpec{
 				NumberOfClusters: noc,
 			},
 		}
+
+		if tolerateUnreachableTaint {
+			placement.Spec.Tolerations = []clusterapiv1beta1.Toleration{
+				{
+					Key: clusterapiv1.ManagedClusterTaintUnreachable,
+				},
+			}
+		}
+
 		placement, err = clusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
@@ -234,8 +278,12 @@ var _ = ginkgo.Describe("Placement", func() {
 		assertNumberOfDecisions(placementName, 5)
 		assertPlacementStatus(placementName, 5, false)
 
-		// create global clusterset
-		assertBindingClusterSet(clusterSetGlobal, map[string]string{})
+		// create global clusterset if necessary
+		if createGlobalClusterSet {
+			assertCreatingClusterSet(clusterSetGlobal, map[string]string{})
+		}
+		assertCreatingClusterSetBinding(clusterSetGlobal)
+
 		// create 2 more clusters belong to global clusterset
 		assertCreatingClusters("", 2)
 		assertNumberOfDecisions(placementName, 6)
@@ -302,6 +350,15 @@ var _ = ginkgo.Describe("Placement", func() {
 				Name:      "test",
 			},
 		}
+
+		if tolerateUnreachableTaint {
+			placement.Spec.Tolerations = []clusterapiv1beta1.Toleration{
+				{
+					Key: clusterapiv1.ManagedClusterTaintUnreachable,
+				},
+			}
+		}
+
 		_, err = clusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	})

test/e2e/suite_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"flag"
 	"os"
 	"testing"
 
@@ -22,23 +23,38 @@ const (
 	eventuallyInterval = 1  // seconds
 )
 
-func TestE2E(t *testing.T) {
-	gomega.RegisterFailHandler(ginkgo.Fail)
-	ginkgo.RunSpecs(t, "E2E Suite")
+var (
+	hubKubeConfig            string
+	createGlobalClusterSet   bool
+	tolerateUnreachableTaint bool
+	kubeClient               kubernetes.Interface
+	clusterClient            clusterclient.Interface
+	restConfig               *rest.Config
+)
+
+func init() {
+	flag.StringVar(&hubKubeConfig, "hub-kubeconfig", "", "The kubeconfig of the hub cluster")
+	flag.BoolVar(&createGlobalClusterSet, "create-global-clusterset", true, "Whether create global clusterset or not (default true, global clusterset will be created)")
+	flag.BoolVar(&tolerateUnreachableTaint, "tolerate-unreachable-taint", false, "Whether tolerate the cluster.open-cluster-management.io/unreachable taint or not (default not, placements created by the test cases will not tolerate this taint)")
 }
 
-var (
-	kubeClient    kubernetes.Interface
-	clusterClient clusterclient.Interface
-	restConfig    *rest.Config
-)
+func TestE2E(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "Placement E2E Suite")
+}
 
 var _ = ginkgo.BeforeSuite(func() {
 	logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true)))
-	kubeconfig := os.Getenv("KUBECONFIG")
+
+	// pick up hubKubeconfig from argument first,
+	// and then fall back to environment variable
+	if len(hubKubeConfig) == 0 {
+		hubKubeConfig = os.Getenv("KUBECONFIG")
+	}
+	gomega.Expect(hubKubeConfig).ToNot(gomega.BeEmpty(), "hubKubeConfig is not specified.")
 
 	var err error
-	restConfig, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	restConfig, err = clientcmd.BuildConfigFromFlags("", hubKubeConfig)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	kubeClient, err = kubernetes.NewForConfig(restConfig)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())