add scalability test for placement create/update (#44)

Signed-off-by: suigh <suigh@cn.ibm.com>

test/scalability/placement_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -18,19 +19,20 @@ import (
 )
 
 const (
-	clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
-	placementLabel  = "cluster.open-cluster-management.io/placement"
+	clusterSetLabel   = "cluster.open-cluster-management.io/clusterset"
+	placementLabel    = "cluster.open-cluster-management.io/placement"
+	placementSetLabel = "cluster.open-cluster-management.io/placementset"
 )
 
 var _ = ginkgo.Describe("Placement scalability test", func() {
 	var namespace string
-	var placementName string
-	var clusterSet1Name string
+	var placementSetName string
+	var clusterSetName string
 	var suffix string
+	var sampleCount = 10
 	var err error
 
 	assertPlacementDecisionCreated := func(placement *clusterapiv1alpha1.Placement) error {
-		ginkgo.By("Check if placementdecision is created")
 		var localerr error
 		gomega.Eventually(func() bool {
 			localerr = nil
@@ -57,13 +59,12 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 		return localerr
 	}
 
-	assertNumberOfDecisions := func(placementName string, desiredNOD int) error {
-		ginkgo.By("Check the number of decisions in placementdecisions")
+	assertNumberOfDecisions := func(placement *clusterapiv1alpha1.Placement, desiredNOD int) error {
 		var localerr error
 		gomega.Eventually(func() bool {
 			localerr = nil
 			pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
-				LabelSelector: placementLabel + "=" + placementName,
+				LabelSelector: placementLabel + "=" + placement.Name,
 			})
 			if err != nil {
 				localerr = err
@@ -82,12 +83,11 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 		return localerr
 	}
 
-	assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) error {
-		ginkgo.By("Check the status of placement")
+	assertPlacementStatus := func(placement *clusterapiv1alpha1.Placement, numOfSelectedClusters int, satisfied bool) error {
 		var localerr error
 		gomega.Eventually(func() bool {
 			localerr = nil
-			placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
+			placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placement.Name, metav1.GetOptions{})
 			if err != nil {
 				localerr = err
 				return false
@@ -114,7 +114,7 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 		return localerr
 	}
 
-	assertBindingClusterSet := func(clusterSetName string) error {
+	assertBindingClusterSet := func() error {
 		ginkgo.By("Create clusterset/clustersetbinding")
 		clusterset := &clusterapiv1beta1.ManagedClusterSet{
 			ObjectMeta: metav1.ObjectMeta{
@@ -139,7 +139,7 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 		return err
 	}
 
-	assertCreatingClusters := func(clusterSetName string, num int) error {
+	assertCreatingClusters := func(num int) error {
 		ginkgo.By(fmt.Sprintf("Create %d clusters", num))
 		for i := 0; i < num; i++ {
 			cluster := &clusterapiv1.ManagedCluster{
@@ -156,39 +156,83 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 		return err
 	}
 
-	assertCreatingPlacement := func(name string, noc *int32, nod int) error {
-		ginkgo.By("Create placement")
+	assertCreatingPlacement := func(noc *int32, nod int) error {
 		placement := &clusterapiv1alpha1.Placement{
 			ObjectMeta: metav1.ObjectMeta{
-				Namespace: namespace,
-				Name:      name,
+				Namespace:    namespace,
+				GenerateName: "placement-",
+				Labels: map[string]string{
+					placementSetLabel: placementSetName,
+				},
 			},
 			Spec: clusterapiv1alpha1.PlacementSpec{
 				NumberOfClusters: noc,
 			},
 		}
-		placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
+		pl, err := clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
-		err = assertPlacementDecisionCreated(placement)
+		err = assertPlacementDecisionCreated(pl)
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
-		err = assertNumberOfDecisions(placementName, nod)
+		err = assertNumberOfDecisions(pl, nod)
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		if noc != nil {
-			err = assertPlacementStatus(placementName, nod, nod == int(*noc))
+			err = assertPlacementStatus(pl, nod, nod == int(*noc))
 		}
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		return err
 	}
 
+	assertCreatingPlacements := func(num int, noc *int32, nod int) error {
+		ginkgo.By(fmt.Sprintf("Create %d placements", num))
+		for i := 0; i < num; i++ {
+			err := assertCreatingPlacement(noc, nod)
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+			time.Sleep(time.Duration(1) * time.Second) //sleep 1 second in case API server is too busy
+		}
+		return err
+	}
+
+	assertUpdatingPlacements := func(num int, nod int) error {
+		ginkgo.By(fmt.Sprintf("Check %v updated placements", num))
+
+		var localerr error
+
+		pls, err := clusterClient.ClusterV1alpha1().Placements(namespace).List(context.Background(), metav1.ListOptions{
+			LabelSelector: placementSetLabel + "=" + placementSetName,
+		})
+		if err != nil {
+			localerr = err
+			return localerr
+		}
+
+		sampleCap := num / sampleCount
+		targetSample := 0
+		currentSample := 0
+
+		for _, pl := range pls.Items {
+			currentSample++
+			if currentSample > targetSample {
+				targetSample += sampleCap
+				err := assertNumberOfDecisions(&pl, nod)
+				if err != nil {
+					localerr = err
+					break
+				}
+			}
+		}
+
+		return err
+	}
+
 	ginkgo.BeforeEach(func() {
 		suffix = rand.String(5)
 		namespace = fmt.Sprintf("ns-%s", suffix)
-		placementName = fmt.Sprintf("placement-%s", suffix)
-		clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix)
+		placementSetName = fmt.Sprintf("placementset-%s", suffix)
+		clusterSetName = fmt.Sprintf("clusterset-%s", suffix)
 
 		// create testing namespace
 		ns := &corev1.Namespace{
@@ -199,20 +243,22 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 		_, err = kubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
-		assertBindingClusterSet(clusterSet1Name)
+		assertBindingClusterSet()
 	})
 
 	ginkgo.AfterEach(func() {
 		ginkgo.By("Delete placement")
-		err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.TODO(), placementName, metav1.DeleteOptions{})
+		err = clusterClient.ClusterV1alpha1().Placements(namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
+			LabelSelector: placementSetLabel + "=" + placementSetName,
+		})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		ginkgo.By("Delete managedclusterset")
-		clusterClient.ClusterV1beta1().ManagedClusterSets().Delete(context.Background(), clusterSet1Name, metav1.DeleteOptions{})
+		clusterClient.ClusterV1beta1().ManagedClusterSets().Delete(context.Background(), clusterSetName, metav1.DeleteOptions{})
 
 		ginkgo.By("Delete managedclusters")
 		clusterClient.ClusterV1().ManagedClusters().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
-			LabelSelector: clusterSetLabel + "=" + clusterSet1Name,
+			LabelSelector: clusterSetLabel + "=" + clusterSetName,
 		})
 
 		err = kubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
@@ -220,46 +266,121 @@ var _ = ginkgo.Describe("Placement scalability test", func() {
 	})
 
 	/* we create N managedclusters here, and create a placement whose NumberOfClusters is N-1 to ensure the placement logic will
-	do comparison to select N-1 managedclusters from N candidates */
+	 * do comparison to select N-1 managedclusters from N candidates
+	 * N will be 100, 1000, 2000.
+	 */
 
-	totalClusters_1 := 100
-	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_1), func(b ginkgo.Benchmarker) {
-		err = assertCreatingClusters(clusterSet1Name, totalClusters_1)
+	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently"), func(b ginkgo.Benchmarker) {
+		totalClusters := 100
+		ginkgo.By(fmt.Sprintf("Create 1 placement with %v managedclusters", totalClusters))
+		err = assertCreatingClusters(totalClusters)
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		runtime := b.Time("runtime", func() {
-			err = assertCreatingPlacement(placementName, noc(totalClusters_1-1), totalClusters_1-1)
+			err = assertCreatingPlacement(noc(totalClusters-1), totalClusters-1)
 		})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 5), "Something during creating placement take too long.")
 	}, 1)
 
-	totalClusters_2 := 1000
-	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_2), func(b ginkgo.Benchmarker) {
-		err = assertCreatingClusters(clusterSet1Name, totalClusters_2)
+	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently"), func(b ginkgo.Benchmarker) {
+		totalClusters := 1000
+		ginkgo.By(fmt.Sprintf("Create 1 placement with %v managedclusters", totalClusters))
+		err = assertCreatingClusters(totalClusters)
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		runtime := b.Time("runtime", func() {
-			err = assertCreatingPlacement(placementName, noc(totalClusters_2-1), totalClusters_2-1)
+			err = assertCreatingPlacement(noc(totalClusters-1), totalClusters-1)
 		})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 10), "Something during creating placement take too long.")
 	}, 1)
 
-	totalClusters_3 := 2000
-	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_3), func(b ginkgo.Benchmarker) {
-		err = assertCreatingClusters(clusterSet1Name, totalClusters_3)
+	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently"), func(b ginkgo.Benchmarker) {
+		totalClusters := 2000
+		ginkgo.By(fmt.Sprintf("Create 1 placement with %v managedclusters", totalClusters))
+		err = assertCreatingClusters(totalClusters)
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		runtime := b.Time("runtime", func() {
-			err = assertCreatingPlacement(placementName, noc(totalClusters_3-1), totalClusters_3-1)
+			err = assertCreatingPlacement(noc(totalClusters-1), totalClusters-1)
 		})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 		gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 20), "Something during creating placement take too long.")
 	}, 1)
+
+	/* To check the scalability of placement creating/updating, we will
+	 * 1. create N-2 managedclusters, and create M placements whose NumberOfClusters is N-1, then select several placements to ensure each one has N-2 decisions, this is for creating.
+	 * 2. create 2 managedclusters, then select several placements to ensure each one has N-1 decisions, this is for updating.
+	 * M will be 10, 100, 300
+	 * N will be 100
+	 */
+
+	ginkgo.Measure(fmt.Sprintf("Should create/update placement efficiently"), func(b ginkgo.Benchmarker) {
+		totalPlacements := 10
+		totalClusters := 100
+		ginkgo.By(fmt.Sprintf("Create %v placement with %v managedclusters", totalPlacements, totalClusters))
+		err = assertCreatingClusters(totalClusters - 2)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		createtime := b.Time("createtime", func() {
+			err = assertCreatingPlacements(totalPlacements, noc(totalClusters-1), totalClusters-2)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Ω(createtime.Seconds()).Should(gomega.BeNumerically("<", 50), "Something during creating placement take too long.")
+
+		err = assertCreatingClusters(2)
+		updatetime := b.Time("updatetime", func() {
+			err = assertUpdatingPlacements(totalPlacements, totalClusters-1)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Ω(updatetime.Seconds()).Should(gomega.BeNumerically("<", 20), "Something during updating placement take too long.")
+	}, 1)
+
+	ginkgo.Measure(fmt.Sprintf("Should create/update placement efficiently"), func(b ginkgo.Benchmarker) {
+		totalPlacements := 100
+		totalClusters := 100
+		ginkgo.By(fmt.Sprintf("Create %v placement with %v managedclusters", totalPlacements, totalClusters))
+		err = assertCreatingClusters(totalClusters - 2)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		createtime := b.Time("createtime", func() {
+			err = assertCreatingPlacements(totalPlacements, noc(totalClusters-1), totalClusters-2)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Ω(createtime.Seconds()).Should(gomega.BeNumerically("<", 400), "Something during creating placement take too long.")
+
+		err = assertCreatingClusters(2)
+		updatetime := b.Time("updatetime", func() {
+			err = assertUpdatingPlacements(totalPlacements, totalClusters-1)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Ω(updatetime.Seconds()).Should(gomega.BeNumerically("<", 60), "Something during updating placement take too long.")
+	}, 1)
+
+	ginkgo.Measure(fmt.Sprintf("Should create/update placement efficiently"), func(b ginkgo.Benchmarker) {
+		totalPlacements := 300
+		totalClusters := 100
+		ginkgo.By(fmt.Sprintf("Create %v placement with %v managedclusters", totalPlacements, totalClusters))
+		err = assertCreatingClusters(totalClusters - 2)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		createtime := b.Time("createtime", func() {
+			err = assertCreatingPlacements(totalPlacements, noc(totalClusters-1), totalClusters-2)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Ω(createtime.Seconds()).Should(gomega.BeNumerically("<", 1200), "Something during creating placement take too long.")
+
+		err = assertCreatingClusters(2)
+		updatetime := b.Time("updatetime", func() {
+			err = assertUpdatingPlacements(totalPlacements, totalClusters-1)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Ω(updatetime.Seconds()).Should(gomega.BeNumerically("<", 200), "Something during updating placement take too long.")
+	}, 1)
 })
 
 func noc(n int) *int32 {

test/scalability/suite_test.go

@@ -18,8 +18,8 @@ import (
 )
 
 const (
-	eventuallyTimeout  = 30 // seconds
-	eventuallyInterval = 1  // seconds
+	eventuallyTimeout  = 120 // seconds
+	eventuallyInterval = 3   // seconds
 )
 
 func TestScalability(t *testing.T) {