add scalability test (#41)

Signed-off-by: suigh <suigh@cn.ibm.com>
2026-05-21 16:43:48 +00:00 · 2021-10-26 16:00:26 +08:00
parent a0e134ebee
commit d08b95abe2
3 changed files with 324 additions and 0 deletions
--- a/9
+++ b/9
@@ -55,6 +55,15 @@ test-e2e: build-e2e ensure-kustomize deploy-hub
 clean-e2e:
 	$(RM) ./e2e.test

+build-scalability:
+	go test -c ./test/scalability -mod=vendor
+
+test-scalability: build-scalability ensure-kustomize deploy-hub
+	./scalability.test -test.v -ginkgo.v
+
+clean-scalability:
+	$(RM) ./scalability.test
+
 ensure-kustomize:
 ifeq "" "$(wildcard $(KUSTOMIZE))"
 	$(info Installing kustomize into '$(KUSTOMIZE)')
--- a/test/scalability/placement_test.go
+++ b/test/scalability/placement_test.go
@@ -0,0 +1,268 @@
+package scalability
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+
+	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
+	clusterapiv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1"
+	clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
+	"open-cluster-management.io/placement/test/integration/util"
+)
+
+const (
+	clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
+	placementLabel  = "cluster.open-cluster-management.io/placement"
+)
+
+var _ = ginkgo.Describe("Placement scalability test", func() {
+	var namespace string
+	var placementName string
+	var clusterSet1Name string
+	var suffix string
+	var err error
+
+	assertPlacementDecisionCreated := func(placement *clusterapiv1alpha1.Placement) error {
+		ginkgo.By("Check if placementdecision is created")
+		var localerr error
+		gomega.Eventually(func() bool {
+			localerr = nil
+			pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
+				LabelSelector: placementLabel + "=" + placement.Name,
+			})
+			if err != nil {
+				localerr = err
+				return false
+			}
+			if len(pdl.Items) == 0 {
+				localerr = errors.New("No placementdecision found")
+				return false
+			}
+			for _, pd := range pdl.Items {
+				if controlled := metav1.IsControlledBy(&pd.ObjectMeta, placement); !controlled {
+					localerr = errors.New("No controllerRef found for a placement")
+					return false
+				}
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		return localerr
+	}
+
+	assertNumberOfDecisions := func(placementName string, desiredNOD int) error {
+		ginkgo.By("Check the number of decisions in placementdecisions")
+		var localerr error
+		gomega.Eventually(func() bool {
+			localerr = nil
+			pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
+				LabelSelector: placementLabel + "=" + placementName,
+			})
+			if err != nil {
+				localerr = err
+				return false
+			}
+			actualNOD := 0
+			for _, pd := range pdl.Items {
+				actualNOD += len(pd.Status.Decisions)
+			}
+			if actualNOD != desiredNOD {
+				localerr = errors.New(fmt.Sprintf("Mismatch value %v:%v", actualNOD, desiredNOD))
+			}
+			return actualNOD == desiredNOD
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		return localerr
+	}
+
+	assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) error {
+		ginkgo.By("Check the status of placement")
+		var localerr error
+		gomega.Eventually(func() bool {
+			localerr = nil
+			placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
+			if err != nil {
+				localerr = err
+				return false
+			}
+			status := metav1.ConditionFalse
+			if satisfied {
+				status = metav1.ConditionTrue
+			}
+			if !util.HasCondition(
+				placement.Status.Conditions,
+				clusterapiv1alpha1.PlacementConditionSatisfied,
+				"",
+				status,
+			) {
+				localerr = errors.New("Contition check failed")
+				return false
+			}
+			if placement.Status.NumberOfSelectedClusters != int32(numOfSelectedClusters) {
+				localerr = errors.New(fmt.Sprintf("Mismatch value %v:%v", placement.Status.NumberOfSelectedClusters, int32(numOfSelectedClusters)))
+			}
+			return placement.Status.NumberOfSelectedClusters == int32(numOfSelectedClusters)
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		return localerr
+	}
+
+	assertBindingClusterSet := func(clusterSetName string) error {
+		ginkgo.By("Create clusterset/clustersetbinding")
+		clusterset := &clusterapiv1beta1.ManagedClusterSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: clusterSetName,
+			},
+		}
+		_, err = clusterClient.ClusterV1beta1().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		csb := &clusterapiv1beta1.ManagedClusterSetBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      clusterSetName,
+			},
+			Spec: clusterapiv1beta1.ManagedClusterSetBindingSpec{
+				ClusterSet: clusterSetName,
+			},
+		}
+		_, err = clusterClient.ClusterV1beta1().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		return err
+	}
+
+	assertCreatingClusters := func(clusterSetName string, num int) error {
+		ginkgo.By(fmt.Sprintf("Create %d clusters", num))
+		for i := 0; i < num; i++ {
+			cluster := &clusterapiv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "cluster-",
+					Labels: map[string]string{
+						clusterSetLabel: clusterSetName,
+					},
+				},
+			}
+			_, err = clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		}
+		return err
+	}
+
+	assertCreatingPlacement := func(name string, noc *int32, nod int) error {
+		ginkgo.By("Create placement")
+		placement := &clusterapiv1alpha1.Placement{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      name,
+			},
+			Spec: clusterapiv1alpha1.PlacementSpec{
+				NumberOfClusters: noc,
+			},
+		}
+		placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		err = assertPlacementDecisionCreated(placement)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		err = assertNumberOfDecisions(placementName, nod)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		if noc != nil {
+			err = assertPlacementStatus(placementName, nod, nod == int(*noc))
+		}
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		return err
+	}
+
+	ginkgo.BeforeEach(func() {
+		suffix = rand.String(5)
+		namespace = fmt.Sprintf("ns-%s", suffix)
+		placementName = fmt.Sprintf("placement-%s", suffix)
+		clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix)
+
+		// create testing namespace
+		ns := &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: namespace,
+			},
+		}
+		_, err = kubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		assertBindingClusterSet(clusterSet1Name)
+	})
+
+	ginkgo.AfterEach(func() {
+		ginkgo.By("Delete placement")
+		err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.TODO(), placementName, metav1.DeleteOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		ginkgo.By("Delete managedclusterset")
+		clusterClient.ClusterV1beta1().ManagedClusterSets().Delete(context.Background(), clusterSet1Name, metav1.DeleteOptions{})
+
+		ginkgo.By("Delete managedclusters")
+		clusterClient.ClusterV1().ManagedClusters().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
+			LabelSelector: clusterSetLabel + "=" + clusterSet1Name,
+		})
+
+		err = kubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	})
+
+	/* we create N managedclusters here, and create a placement whose NumberOfClusters is N-1 to ensure the placement logic will
+	do comparison to select N-1 managedclusters from N candidates */
+
+	totalClusters_1 := 100
+	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_1), func(b ginkgo.Benchmarker) {
+		err = assertCreatingClusters(clusterSet1Name, totalClusters_1)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		runtime := b.Time("runtime", func() {
+			err = assertCreatingPlacement(placementName, noc(totalClusters_1-1), totalClusters_1-1)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 5), "Something during creating placement take too long.")
+	}, 1)
+
+	totalClusters_2 := 1000
+	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_2), func(b ginkgo.Benchmarker) {
+		err = assertCreatingClusters(clusterSet1Name, totalClusters_2)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		runtime := b.Time("runtime", func() {
+			err = assertCreatingPlacement(placementName, noc(totalClusters_2-1), totalClusters_2-1)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 10), "Something during creating placement take too long.")
+	}, 1)
+
+	totalClusters_3 := 2000
+	ginkgo.Measure(fmt.Sprintf("Should create placement efficiently with %d managedclusters", totalClusters_3), func(b ginkgo.Benchmarker) {
+		err = assertCreatingClusters(clusterSet1Name, totalClusters_3)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		runtime := b.Time("runtime", func() {
+			err = assertCreatingPlacement(placementName, noc(totalClusters_3-1), totalClusters_3-1)
+		})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		gomega.Ω(runtime.Seconds()).Should(gomega.BeNumerically("<", 20), "Something during creating placement take too long.")
+	}, 1)
+})
+
+func noc(n int) *int32 {
+	noc := int32(n)
+	return &noc
+}
--- a/test/scalability/suite_test.go
+++ b/test/scalability/suite_test.go
@@ -0,0 +1,47 @@
+package scalability
+
+import (
+	"os"
+	"testing"
+
+	ginkgo "github.com/onsi/ginkgo"
+	gomega "github.com/onsi/gomega"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	clusterclient "open-cluster-management.io/api/client/cluster/clientset/versioned"
+)
+
+const (
+	eventuallyTimeout  = 30 // seconds
+	eventuallyInterval = 1  // seconds
+)
+
+func TestScalability(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "Scalability Suite")
+}
+
+var (
+	kubeClient    kubernetes.Interface
+	clusterClient clusterclient.Interface
+	restConfig    *rest.Config
+)
+
+var _ = ginkgo.BeforeSuite(func() {
+	logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true)))
+	kubeconfig := os.Getenv("KUBECONFIG")
+
+	var err error
+	restConfig, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	kubeClient, err = kubernetes.NewForConfig(restConfig)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	clusterClient, err = clusterclient.NewForConfig(restConfig)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+})