From 7db4f6194a4945853b0212fffd18d0b00428d582 Mon Sep 17 00:00:00 2001 From: suigh Date: Mon, 1 Nov 2021 11:04:45 +0800 Subject: [PATCH] add scalability test for placement create/update (#44) Signed-off-by: suigh --- test/scalability/placement_test.go | 201 +++++++++++++++++++++++------ test/scalability/suite_test.go | 4 +- 2 files changed, 163 insertions(+), 42 deletions(-) diff --git a/test/scalability/placement_test.go b/test/scalability/placement_test.go index 2b0e60e88..10dc3af7f 100644 --- a/test/scalability/placement_test.go +++ b/test/scalability/placement_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -18,19 +19,20 @@ import ( ) const ( - clusterSetLabel = "cluster.open-cluster-management.io/clusterset" - placementLabel = "cluster.open-cluster-management.io/placement" + clusterSetLabel = "cluster.open-cluster-management.io/clusterset" + placementLabel = "cluster.open-cluster-management.io/placement" + placementSetLabel = "cluster.open-cluster-management.io/placementset" ) var _ = ginkgo.Describe("Placement scalability test", func() { var namespace string - var placementName string - var clusterSet1Name string + var placementSetName string + var clusterSetName string var suffix string + var sampleCount = 10 var err error assertPlacementDecisionCreated := func(placement *clusterapiv1alpha1.Placement) error { - ginkgo.By("Check if placementdecision is created") var localerr error gomega.Eventually(func() bool { localerr = nil @@ -57,13 +59,12 @@ var _ = ginkgo.Describe("Placement scalability test", func() { return localerr } - assertNumberOfDecisions := func(placementName string, desiredNOD int) error { - ginkgo.By("Check the number of decisions in placementdecisions") + assertNumberOfDecisions := func(placement *clusterapiv1alpha1.Placement, desiredNOD int) error { var localerr error gomega.Eventually(func() bool { localerr = nil pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: placementLabel + "=" + placementName, + LabelSelector: placementLabel + "=" + placement.Name, }) if err != nil { localerr = err @@ -82,12 +83,11 @@ var _ = ginkgo.Describe("Placement scalability test", func() { return localerr } - assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) error { - ginkgo.By("Check the status of placement") + assertPlacementStatus := func(placement *clusterapiv1alpha1.Placement, numOfSelectedClusters int, satisfied bool) error { var localerr error gomega.Eventually(func() bool { localerr = nil - placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) + placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placement.Name, metav1.GetOptions{}) if err != nil { localerr = err return false @@ -114,7 +114,7 @@ var _ = ginkgo.Describe("Placement scalability test", func() { return localerr } - assertBindingClusterSet := func(clusterSetName string) error { + assertBindingClusterSet := func() error { ginkgo.By("Create clusterset/clustersetbinding") clusterset := &clusterapiv1beta1.ManagedClusterSet{ ObjectMeta: metav1.ObjectMeta{ @@ -139,7 +139,7 @@ var _ = ginkgo.Describe("Placement scalability test", func() { return err } - assertCreatingClusters := func(clusterSetName string, num int) error { + assertCreatingClusters := func(num int) error { ginkgo.By(fmt.Sprintf("Create %d clusters", num)) for i := 0; i < num; i++ { cluster := &clusterapiv1.ManagedCluster{ @@ -156,39 +156,83 @@ var _ = ginkgo.Describe("Placement scalability test", func() { return err } - assertCreatingPlacement := func(name string, noc *int32, nod int) error { - ginkgo.By("Create placement") + assertCreatingPlacement := func(noc *int32, nod int) error { placement := &clusterapiv1alpha1.Placement{ ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, + Namespace: namespace, + GenerateName: "placement-", + Labels: map[string]string{ + placementSetLabel: placementSetName, + }, }, Spec: clusterapiv1alpha1.PlacementSpec{ NumberOfClusters: noc, }, } - placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) + pl, err := clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - err = assertPlacementDecisionCreated(placement) + err = assertPlacementDecisionCreated(pl) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - err = assertNumberOfDecisions(placementName, nod) + err = assertNumberOfDecisions(pl, nod) gomega.Expect(err).ToNot(gomega.HaveOccurred()) if noc != nil { - err = assertPlacementStatus(placementName, nod, nod == int(*noc)) + err = assertPlacementStatus(pl, nod, nod == int(*noc)) } gomega.Expect(err).ToNot(gomega.HaveOccurred()) return err } + assertCreatingPlacements := func(num int, noc *int32, nod int) error { + ginkgo.By(fmt.Sprintf("Create %d placements", num)) + for i := 0; i < num; i++ { + err := assertCreatingPlacement(noc, nod) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + time.Sleep(time.Duration(1) * time.Second) //sleep 1 second in case API server is too busy + } + return err + } + + assertUpdatingPlacements := func(num int, nod int) error { + ginkgo.By(fmt.Sprintf("Check %v updated placements", num)) + + var localerr error + + pls, err := clusterClient.ClusterV1alpha1().Placements(namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: placementSetLabel + "=" + placementSetName, + }) + if err != nil { + localerr = err + return localerr + } + + sampleCap := num / sampleCount + targetSample := 0 + currentSample := 0 + + for _, pl := range pls.Items { + currentSample++ + if currentSample > targetSample { + targetSample += sampleCap + err := assertNumberOfDecisions(&pl, nod) + if err != nil { + localerr = err + break + } + } + } + + return err + } + ginkgo.BeforeEach(func() { suffix = rand.String(5) namespace = fmt.Sprintf("ns-%s", suffix) - placementName = fmt.Sprintf("placement-%s", suffix) - clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix) + placementSetName = fmt.Sprintf("placementset-%s", suffix) + clusterSetName = fmt.Sprintf("clusterset-%s", suffix) // create testing namespace ns := &corev1.Namespace{ @@ -199,20 +243,22 @@ var _ = ginkgo.Describe("Placement scalability test", func() { _, err = kubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - assertBindingClusterSet(clusterSet1Name) + assertBindingClusterSet() }) ginkgo.AfterEach(func() { ginkgo.By("Delete placement") - err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.TODO(), placementName, metav1.DeleteOptions{}) + err = clusterClient.ClusterV1alpha1().Placements(namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{ + LabelSelector: placementSetLabel + "=" + placementSetName, + }) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Delete managedclusterset") - clusterClient.ClusterV1beta1().ManagedClusterSets().Delete(context.Background(), clusterSet1Name, metav1.DeleteOptions{}) + clusterClient.ClusterV1beta1().ManagedClusterSets().Delete(context.Background(), clusterSetName, metav1.DeleteOptions{}) ginkgo.By("Delete managedclusters") clusterClient.ClusterV1().ManagedClusters().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: clusterSetLabel + "=" + clusterSet1Name, + LabelSelector: clusterSetLabel + "=" + clusterSetName, }) err = kubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{}) @@ -220,46 +266,121 @@ var _ = ginkgo.Describe("Placement scalability test", func() { }) /* we create N managedclusters here, and create a placement whose NumberOfClusters is N-1 to ensure the placement logic will - do comparison to select N-1 managedclusters from N candidates */ + * do comparison to select N-1 managedclusters from N candidates + * N will be 100, 1000, 2000. + */ - totalClusters_1 := 100 - ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_1), func(b ginkgo.Benchmarker) { - err = assertCreatingClusters(clusterSet1Name, totalClusters_1) + ginkgo.Measure(fmt.Sprintf("Should create placement efficiently"), func(b ginkgo.Benchmarker) { + totalClusters := 100 + ginkgo.By(fmt.Sprintf("Create 1 placement with %v managedclusters", totalClusters)) + err = assertCreatingClusters(totalClusters) gomega.Expect(err).ToNot(gomega.HaveOccurred()) runtime := b.Time("runtime", func() { - err = assertCreatingPlacement(placementName, noc(totalClusters_1-1), totalClusters_1-1) + err = assertCreatingPlacement(noc(totalClusters-1), totalClusters-1) }) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 5), "Something during creating placement take too long.") }, 1) - totalClusters_2 := 1000 - ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_2), func(b ginkgo.Benchmarker) { - err = assertCreatingClusters(clusterSet1Name, totalClusters_2) + ginkgo.Measure(fmt.Sprintf("Should create placement efficiently"), func(b ginkgo.Benchmarker) { + totalClusters := 1000 + ginkgo.By(fmt.Sprintf("Create 1 placement with %v managedclusters", totalClusters)) + err = assertCreatingClusters(totalClusters) gomega.Expect(err).ToNot(gomega.HaveOccurred()) runtime := b.Time("runtime", func() { - err = assertCreatingPlacement(placementName, noc(totalClusters_2-1), totalClusters_2-1) + err = assertCreatingPlacement(noc(totalClusters-1), totalClusters-1) }) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 10), "Something during creating placement take too long.") }, 1) - totalClusters_3 := 2000 - ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_3), func(b ginkgo.Benchmarker) { - err = assertCreatingClusters(clusterSet1Name, totalClusters_3) + ginkgo.Measure(fmt.Sprintf("Should create placement efficiently"), func(b ginkgo.Benchmarker) { + totalClusters := 2000 + ginkgo.By(fmt.Sprintf("Create 1 placement with %v managedclusters", totalClusters)) + err = assertCreatingClusters(totalClusters) gomega.Expect(err).ToNot(gomega.HaveOccurred()) runtime := b.Time("runtime", func() { - err = assertCreatingPlacement(placementName, noc(totalClusters_3-1), totalClusters_3-1) + err = assertCreatingPlacement(noc(totalClusters-1), totalClusters-1) }) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 20), "Something during creating placement take too long.") }, 1) + + /* To check the scalability of placement creating/updating, we will + * 1. create N-2 managedclusters, and create M placements whose NumberOfClusters is N-1, then select several placements to ensure each one has N-2 decisions, this is for creating. + * 2. create 2 managedclusters, then select several placements to ensure each one has N-1 decisions, this is for updating. + * M will be 10, 100, 300 + * N will be 100 + */ + + ginkgo.Measure(fmt.Sprintf("Should create/update placement efficiently"), func(b ginkgo.Benchmarker) { + totalPlacements := 10 + totalClusters := 100 + ginkgo.By(fmt.Sprintf("Create %v placement with %v managedclusters", totalPlacements, totalClusters)) + err = assertCreatingClusters(totalClusters - 2) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + createtime := b.Time("createtime", func() { + err = assertCreatingPlacements(totalPlacements, noc(totalClusters-1), totalClusters-2) + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Ω(createtime.Seconds()).Should(gomega.BeNumerically("<", 50), "Something during creating placement take too long.") + + err = assertCreatingClusters(2) + updatetime := b.Time("updatetime", func() { + err = assertUpdatingPlacements(totalPlacements, totalClusters-1) + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Ω(updatetime.Seconds()).Should(gomega.BeNumerically("<", 20), "Something during updating placement take too long.") + }, 1) + + ginkgo.Measure(fmt.Sprintf("Should create/update placement efficiently"), func(b ginkgo.Benchmarker) { + totalPlacements := 100 + totalClusters := 100 + ginkgo.By(fmt.Sprintf("Create %v placement with %v managedclusters", totalPlacements, totalClusters)) + err = assertCreatingClusters(totalClusters - 2) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + createtime := b.Time("createtime", func() { + err = assertCreatingPlacements(totalPlacements, noc(totalClusters-1), totalClusters-2) + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Ω(createtime.Seconds()).Should(gomega.BeNumerically("<", 400), "Something during creating placement take too long.") + + err = assertCreatingClusters(2) + updatetime := b.Time("updatetime", func() { + err = assertUpdatingPlacements(totalPlacements, totalClusters-1) + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Ω(updatetime.Seconds()).Should(gomega.BeNumerically("<", 60), "Something during updating placement take too long.") + }, 1) + + ginkgo.Measure(fmt.Sprintf("Should create/update placement efficiently"), func(b ginkgo.Benchmarker) { + totalPlacements := 300 + totalClusters := 100 + ginkgo.By(fmt.Sprintf("Create %v placement with %v managedclusters", totalPlacements, totalClusters)) + err = assertCreatingClusters(totalClusters - 2) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + createtime := b.Time("createtime", func() { + err = assertCreatingPlacements(totalPlacements, noc(totalClusters-1), totalClusters-2) + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Ω(createtime.Seconds()).Should(gomega.BeNumerically("<", 1200), "Something during creating placement take too long.") + + err = assertCreatingClusters(2) + updatetime := b.Time("updatetime", func() { + err = assertUpdatingPlacements(totalPlacements, totalClusters-1) + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Ω(updatetime.Seconds()).Should(gomega.BeNumerically("<", 200), "Something during updating placement take too long.") + }, 1) }) func noc(n int) *int32 { diff --git a/test/scalability/suite_test.go b/test/scalability/suite_test.go index 68c4558b5..109d1512b 100644 --- a/test/scalability/suite_test.go +++ b/test/scalability/suite_test.go @@ -18,8 +18,8 @@ import ( ) const ( - eventuallyTimeout = 30 // seconds - eventuallyInterval = 1 // seconds + eventuallyTimeout = 120 // seconds + eventuallyInterval = 3 // seconds ) func TestScalability(t *testing.T) {