From 3d64c80b752e316737514f8b4e0e3a9644cac8e2 Mon Sep 17 00:00:00 2001 From: Yang Le Date: Fri, 7 May 2021 18:32:50 +0800 Subject: [PATCH] add scheduling controller Signed-off-by: Yang Le --- pkg/controllers/manager.go | 17 +- pkg/controllers/placement/controller.go | 66 ----- pkg/controllers/placement/controller_test.go | 44 ---- .../placementdecision/creating_controller.go | 132 ---------- .../creating_controller_test.go | 140 ----------- pkg/controllers/scheduling/predicate.go | 94 ++++++++ pkg/controllers/scheduling/predicate_test.go | 131 ++++++++++ pkg/controllers/scheduling/schedule.go | 148 ++++++++++++ pkg/controllers/scheduling/schedule_test.go | 185 ++++++++++++++ .../scheduling/scheduling_controller.go | 226 ++++++++++++++++++ .../scheduling/scheduling_controller_test.go | 210 ++++++++++++++++ pkg/helpers/testing/builders.go | 151 +++++++++++- pkg/helpers/testing/helpers.go | 23 ++ pkg/helpers/testing/informer.go | 37 +++ test/integration/placement_test.go | 220 +++++++++++++++-- test/integration/util/util.go | 24 +- 16 files changed, 1428 insertions(+), 420 deletions(-) delete mode 100644 pkg/controllers/placement/controller.go delete mode 100644 pkg/controllers/placement/controller_test.go delete mode 100644 pkg/controllers/placementdecision/creating_controller.go delete mode 100644 pkg/controllers/placementdecision/creating_controller_test.go create mode 100644 pkg/controllers/scheduling/predicate.go create mode 100644 pkg/controllers/scheduling/predicate_test.go create mode 100644 pkg/controllers/scheduling/schedule.go create mode 100644 pkg/controllers/scheduling/schedule_test.go create mode 100644 pkg/controllers/scheduling/scheduling_controller.go create mode 100644 pkg/controllers/scheduling/scheduling_controller_test.go create mode 100644 pkg/helpers/testing/informer.go diff --git a/pkg/controllers/manager.go b/pkg/controllers/manager.go index 72a45213f..add6d62ad 100644 --- a/pkg/controllers/manager.go +++ b/pkg/controllers/manager.go @@ -8,8 +8,7 @@ import ( clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned" clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions" - placement "github.com/open-cluster-management/placement/pkg/controllers/placement" - placementdecision "github.com/open-cluster-management/placement/pkg/controllers/placementdecision" + scheduling "github.com/open-cluster-management/placement/pkg/controllers/scheduling" ) // RunControllerManager starts the controllers on hub to make placement decisions. @@ -20,14 +19,11 @@ func RunControllerManager(ctx context.Context, controllerContext *controllercmd. } clusterInformers := clusterinformers.NewSharedInformerFactory(clusterClient, 10*time.Minute) - placementController := placement.NewPlacementController( - clusterInformers.Cluster().V1().ManagedClusters(), - clusterInformers.Cluster().V1alpha1().ManagedClusterSets(), - controllerContext.EventRecorder, - ) - - placementDecisionCreatingController := placementdecision.NewPlacementDecisionCreatingController( + schedulingController := scheduling.NewSchedulingController( clusterClient, + clusterInformers.Cluster().V1().ManagedClusters().Lister(), + clusterInformers.Cluster().V1alpha1().ManagedClusterSets().Lister(), + clusterInformers.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(), clusterInformers.Cluster().V1alpha1().Placements(), clusterInformers.Cluster().V1alpha1().PlacementDecisions(), controllerContext.EventRecorder, @@ -35,8 +31,7 @@ func RunControllerManager(ctx context.Context, controllerContext *controllercmd. go clusterInformers.Start(ctx.Done()) - go placementController.Run(ctx, 1) - go placementDecisionCreatingController.Run(ctx, 1) + go schedulingController.Run(ctx, 1) <-ctx.Done() return nil diff --git a/pkg/controllers/placement/controller.go b/pkg/controllers/placement/controller.go deleted file mode 100644 index 6f5eec922..000000000 --- a/pkg/controllers/placement/controller.go +++ /dev/null @@ -1,66 +0,0 @@ -package placement - -import ( - "context" - "fmt" - - "github.com/openshift/library-go/pkg/controller/factory" - "github.com/openshift/library-go/pkg/operator/events" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - - clusterinformerv1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1" - clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1" - clusterlisterv1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1" - clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1" -) - -const ( - clusterSetLabel = "cluster.open-cluster-management.io/clusterset" -) - -// placementController makes placement decisions for Placements -type placementController struct { - clusterLister clusterlisterv1.ManagedClusterLister - clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister -} - -// NewPlacementController return an instance of placementController -func NewPlacementController( - clusterInformer clusterinformerv1.ManagedClusterInformer, - clusterSetInformer clusterinformerv1alpha1.ManagedClusterSetInformer, - recorder events.Recorder, -) factory.Controller { - c := placementController{ - clusterLister: clusterInformer.Lister(), - clusterSetLister: clusterSetInformer.Lister(), - } - - return factory.New(). - WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string { - accessor, _ := meta.Accessor(obj) - return fmt.Sprintf("cluster:%s", accessor.GetName()) - }, func(obj interface{}) bool { - accessor, err := meta.Accessor(obj) - if err != nil { - return false - } - - // ignore cluster belongs to no clusterset - labels := accessor.GetLabels() - clusterSetName, ok := labels[clusterSetLabel] - if !ok { - return false - } - - // ignore cluster if its clusterset does not exist - _, err = c.clusterSetLister.Get(clusterSetName) - return err == nil - }, clusterInformer.Informer()). - WithSync(c.sync). - ToController("PlacementController", recorder) -} - -func (c *placementController) sync(ctx context.Context, syncCtx factory.SyncContext) error { - return nil -} diff --git a/pkg/controllers/placement/controller_test.go b/pkg/controllers/placement/controller_test.go deleted file mode 100644 index 6dab065fa..000000000 --- a/pkg/controllers/placement/controller_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package placement - -import ( - "context" - "testing" - "time" - - "k8s.io/apimachinery/pkg/runtime" - clienttesting "k8s.io/client-go/testing" - - clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake" - clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions" - testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing" -) - -func TestSync(t *testing.T) { - cases := []struct { - name string - queueKey string - initObjs []runtime.Object - validateActions func(t *testing.T, hubActions, agentActions []clienttesting.Action) - }{} - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - clusterClient := clusterfake.NewSimpleClientset(c.initObjs...) - clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10) - clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore() - for _, cluster := range c.initObjs { - clusterStore.Add(cluster) - } - - ctrl := placementController{ - clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(), - } - syncErr := ctrl.sync(context.TODO(), testinghelpers.NewFakeSyncContext(t, c.queueKey)) - if syncErr != nil { - t.Errorf("unexpected err: %v", syncErr) - } - - //c.validateActions(t, nil) - }) - } -} diff --git a/pkg/controllers/placementdecision/creating_controller.go b/pkg/controllers/placementdecision/creating_controller.go deleted file mode 100644 index 0ed01557c..000000000 --- a/pkg/controllers/placementdecision/creating_controller.go +++ /dev/null @@ -1,132 +0,0 @@ -package placementdecision - -import ( - "context" - "fmt" - - "github.com/openshift/library-go/pkg/controller/factory" - "github.com/openshift/library-go/pkg/operator/events" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" - cache "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - - clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned" - clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1" - clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1" - clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" -) - -const ( - placementLabel = "cluster.open-cluster-management.io/placement" -) - -// placementDecisionCreatingController creates PlacementDecisions for Placements -type placementDecisionCreatingController struct { - clusterClient clusterclient.Interface - placementLister clusterlisterv1alpha1.PlacementLister - placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister -} - -// NewPlacementDecisionCreatingController return an instance of placementDecisionCreatingController -func NewPlacementDecisionCreatingController( - clusterClient clusterclient.Interface, - placementInformer clusterinformerv1alpha1.PlacementInformer, - placementDecisionInformer clusterinformerv1alpha1.PlacementDecisionInformer, - recorder events.Recorder, -) factory.Controller { - c := placementDecisionCreatingController{ - clusterClient: clusterClient, - placementLister: placementInformer.Lister(), - placementDecisionLister: placementDecisionInformer.Lister(), - } - - return factory.New(). - WithInformersQueueKeyFunc(func(obj runtime.Object) string { - key, _ := cache.MetaNamespaceKeyFunc(obj) - return key - }, placementInformer.Informer()). - WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string { - accessor, _ := meta.Accessor(obj) - labels := accessor.GetLabels() - placementName := labels[placementLabel] - return fmt.Sprintf("%s/%s", accessor.GetNamespace(), placementName) - }, func(obj interface{}) bool { - accessor, err := meta.Accessor(obj) - if err != nil { - return false - } - labels := accessor.GetLabels() - if _, ok := labels[placementLabel]; ok { - return true - } - return false - }, placementDecisionInformer.Informer()). - WithSync(c.sync). - ToController("PlacementDecisionCreatingController", recorder) -} - -func (c *placementDecisionCreatingController) sync(ctx context.Context, syncCtx factory.SyncContext) error { - queueKey := syncCtx.QueueKey() - namespace, name, err := cache.SplitMetaNamespaceKey(queueKey) - if err != nil { - // ignore placement whose key is not in format: namespace/name - return nil - } - - klog.V(4).Infof("Reconciling placement %q", queueKey) - placement, err := c.placementLister.Placements(namespace).Get(name) - if errors.IsNotFound(err) { - // no work if placement is deleted - return nil - } - if err != nil { - return err - } - - // no work if placement is deleting - if !placement.DeletionTimestamp.IsZero() { - return nil - } - - // query placementdecisions with label selector - requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name}) - if err != nil { - return err - } - labelSelector := labels.NewSelector().Add(*requirement) - placementDecisions, err := c.placementDecisionLister.PlacementDecisions(namespace).List(labelSelector) - if err != nil { - return err - } - - // no work if PlacementDecision has been created - if len(placementDecisions) > 0 { - return nil - } - - // otherwise create placementdecision - return c.createPlacementDecision(ctx, placement) -} - -// createPlacementDecision creates PlacementDecision for the Placement -func (c *placementDecisionCreatingController) createPlacementDecision(ctx context.Context, placement *clusterapiv1alpha1.Placement) error { - owner := metav1.NewControllerRef(placement, clusterapiv1alpha1.GroupVersion.WithKind("Placement")) - placementDecision := &clusterapiv1alpha1.PlacementDecision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("%s-", placement.Name), - Namespace: placement.Namespace, - Labels: map[string]string{ - placementLabel: placement.Name, - }, - OwnerReferences: []metav1.OwnerReference{*owner}, - }, - } - - _, err := c.clusterClient.ClusterV1alpha1().PlacementDecisions(placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{}) - return err -} diff --git a/pkg/controllers/placementdecision/creating_controller_test.go b/pkg/controllers/placementdecision/creating_controller_test.go deleted file mode 100644 index b7a96ee7b..000000000 --- a/pkg/controllers/placementdecision/creating_controller_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package placementdecision - -import ( - "context" - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/rand" - clienttesting "k8s.io/client-go/testing" - - clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake" - clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions" - clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" - testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing" -) - -func assertValidPlacementDecision(t *testing.T, placementNamespace, placementName string, placementDecision *clusterapiv1alpha1.PlacementDecision) { - // check namespace - if placementDecision.Namespace != placementNamespace { - t.Errorf("expected PlacementDecision is created under namespace %q, but got %q", - placementNamespace, placementDecision.Namespace) - } - - // check label - if _, ok := placementDecision.Labels[placementLabel]; !ok { - t.Errorf("expected Placement label on Placement decision") - } - - // check owner reference - owner := metav1.GetControllerOf(&placementDecision.ObjectMeta) - if owner == nil { - t.Errorf("expected PlacementDecision with ownerreference") - } - - if owner.Kind != "Placement" { - t.Errorf("expected ownerreference with kind %q, but got %q", "Placement", owner.Kind) - } - - if owner.Name != placementName { - t.Errorf("expected ownerreference with name %q, but got %q", placementName, owner.Name) - } -} - -func TestPlacementDecisionCreatingControllerSync(t *testing.T) { - placementNamespace := "ns1" - placementName := "placement1" - queueKey := placementNamespace + "/" + placementName - placementUID := rand.String(16) - - cases := []struct { - name string - queueKey string - initObjs []runtime.Object - validateActions func(t *testing.T, actions []clienttesting.Action) - }{ - { - name: "placement not found", - queueKey: queueKey, - validateActions: func(t *testing.T, actions []clienttesting.Action) { - if len(actions) != 0 { - t.Errorf("expected no action but got: %v ", actions) - } - }, - }, - { - name: "placement is deleting", - queueKey: queueKey, - initObjs: []runtime.Object{ - testinghelpers.NewPlacement(placementNamespace, placementName).WithDeletionTimestamp().Build(), - }, - validateActions: func(t *testing.T, actions []clienttesting.Action) { - if len(actions) != 0 { - t.Errorf("expected no action but got: %v ", actions) - } - }, - }, - { - name: "new placement", - queueKey: queueKey, - initObjs: []runtime.Object{ - testinghelpers.NewPlacement(placementNamespace, placementName).WithUID(placementUID).Build(), - }, - validateActions: func(t *testing.T, actions []clienttesting.Action) { - // check if PlacementDecision has been created - testinghelpers.AssertActions(t, actions, "create") - actual := actions[0].(clienttesting.CreateActionImpl).Object - placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision) - if !ok { - t.Errorf("expected PlacementDecision was created") - } - assertValidPlacementDecision(t, placementNamespace, placementName, placementDecision) - }, - }, - { - name: "placementdecision exits", - queueKey: queueKey, - initObjs: []runtime.Object{ - testinghelpers.NewPlacement(placementNamespace, placementName).WithUID(placementUID).Build(), - testinghelpers.NewPlacementDecision(placementNamespace, "decison1"). - WithPlacementLabel(placementName).WithController(placementUID).Build(), - }, - validateActions: func(t *testing.T, actions []clienttesting.Action) { - if len(actions) != 0 { - t.Errorf("expected no action but got %d: %v", len(actions), actions) - } - }, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - clusterClient := clusterfake.NewSimpleClientset(c.initObjs...) - clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10) - placementStore := clusterInformerFactory.Cluster().V1alpha1().Placements().Informer().GetStore() - placementDecisionStore := clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Informer().GetStore() - for _, obj := range c.initObjs { - switch obj.(type) { - case *clusterapiv1alpha1.Placement: - placementStore.Add(obj) - case *clusterapiv1alpha1.PlacementDecision: - placementDecisionStore.Add(obj) - } - } - - ctrl := placementDecisionCreatingController{ - clusterClient: clusterClient, - placementLister: clusterInformerFactory.Cluster().V1alpha1().Placements().Lister(), - placementDecisionLister: clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(), - } - syncErr := ctrl.sync(context.TODO(), testinghelpers.NewFakeSyncContext(t, c.queueKey)) - if syncErr != nil { - t.Errorf("unexpected err: %v", syncErr) - } - - c.validateActions(t, clusterClient.Actions()) - }) - } -} diff --git a/pkg/controllers/scheduling/predicate.go b/pkg/controllers/scheduling/predicate.go new file mode 100644 index 000000000..e6c9db237 --- /dev/null +++ b/pkg/controllers/scheduling/predicate.go @@ -0,0 +1,94 @@ +package scheduling + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" +) + +type predicateSelector struct { + labelSelector labels.Selector + claimSelector labels.Selector +} + +// matchWithClusterPredicates returns a slice of clusters. Each of them must match at least one of the predicates. +func matchWithClusterPredicates(predicates []clusterapiv1alpha1.ClusterPredicate, clusters []*clusterapiv1.ManagedCluster) ([]*clusterapiv1.ManagedCluster, error) { + if len(predicates) == 0 { + return clusters, nil + } + if len(clusters) == 0 { + return clusters, nil + } + + // prebuild label/claim selectors for each predicate + predicateSelectors := []predicateSelector{} + for _, predicate := range predicates { + // build label selector + labelSelector, err := convertLabelSelector(predicate.RequiredClusterSelector.LabelSelector) + if err != nil { + return nil, err + } + // build claim selector + claimSelector, err := convertClaimSelector(predicate.RequiredClusterSelector.ClaimSelector) + if err != nil { + return nil, err + } + predicateSelectors = append(predicateSelectors, predicateSelector{ + labelSelector: labelSelector, + claimSelector: claimSelector, + }) + } + + // match cluster with selectors one by one + matched := []*clusterapiv1.ManagedCluster{} + for _, cluster := range clusters { + claims := getClusterClaims(cluster) + for _, ps := range predicateSelectors { + // match with label selector + if ok := ps.labelSelector.Matches(labels.Set(cluster.Labels)); !ok { + continue + } + // match with claim selector + if ok := ps.claimSelector.Matches(labels.Set(claims)); !ok { + continue + } + matched = append(matched, cluster) + break + } + } + + return matched, nil +} + +// getClusterClaims returns a map containing cluster claims from the status of cluster +func getClusterClaims(cluster *clusterapiv1.ManagedCluster) map[string]string { + claims := map[string]string{} + for _, claim := range cluster.Status.ClusterClaims { + claims[claim.Name] = claim.Value + } + return claims +} + +// convertLabelSelector converts metav1.LabelSelector to labels.Selector +func convertLabelSelector(labelSelector metav1.LabelSelector) (labels.Selector, error) { + selector, err := metav1.LabelSelectorAsSelector(&labelSelector) + if err != nil { + return labels.Nothing(), err + } + + return selector, nil +} + +// convertClaimSelector converts ClusterClaimSelector to labels.Selector +func convertClaimSelector(clusterClaimSelector clusterapiv1alpha1.ClusterClaimSelector) (labels.Selector, error) { + selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchExpressions: clusterClaimSelector.MatchExpressions, + }) + if err != nil { + return labels.Nothing(), err + } + + return selector, nil +} diff --git a/pkg/controllers/scheduling/predicate_test.go b/pkg/controllers/scheduling/predicate_test.go new file mode 100644 index 000000000..d075fa410 --- /dev/null +++ b/pkg/controllers/scheduling/predicate_test.go @@ -0,0 +1,131 @@ +package scheduling + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" + testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing" +) + +func TestMatchWithClusterPredicates(t *testing.T) { + cases := []struct { + name string + predicates []clusterapiv1alpha1.ClusterPredicate + clusters []*clusterapiv1.ManagedCluster + expectedClusterNames []string + }{ + { + name: "match with label", + predicates: []clusterapiv1alpha1.ClusterPredicate{ + testinghelpers.NewClusterPredicate(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cloud": "Amazon", + }, + }, nil), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Google").Build(), + }, + expectedClusterNames: []string{"cluster1"}, + }, + { + name: "match with claim", + predicates: []clusterapiv1alpha1.ClusterPredicate{ + testinghelpers.NewClusterPredicate(nil, + &clusterapiv1alpha1.ClusterClaimSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "cloud", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"Amazon"}, + }, + }, + }), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithClaim("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithClaim("cloud", "Google").Build(), + }, + expectedClusterNames: []string{"cluster1"}, + }, + { + name: "match with both label and claim", + predicates: []clusterapiv1alpha1.ClusterPredicate{ + testinghelpers.NewClusterPredicate(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cloud": "Amazon", + }, + }, &clusterapiv1alpha1.ClusterClaimSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "region", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"us-east-1"}, + }, + }, + }), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1"). + WithLabel("cloud", "Amazon"). + WithClaim("region", "us-east-1").Build(), + testinghelpers.NewManagedCluster("cluster2"). + WithLabel("cloud", "Amazon"). + WithClaim("region", "us-east-2").Build(), + }, + expectedClusterNames: []string{"cluster1"}, + }, + { + name: "match with multiple predicates", + predicates: []clusterapiv1alpha1.ClusterPredicate{ + testinghelpers.NewClusterPredicate(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cloud": "Amazon", + }, + }, nil), + testinghelpers.NewClusterPredicate(nil, &clusterapiv1alpha1.ClusterClaimSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "region", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"us-east-1"}, + }, + }, + }), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithClaim("region", "us-east-1").Build(), + testinghelpers.NewManagedCluster("cluster3").WithClaim("region", "us-east-2").Build(), + }, + expectedClusterNames: []string{"cluster1", "cluster2"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + clusters, err := matchWithClusterPredicates(c.predicates, c.clusters) + if err != nil { + t.Errorf("unexpected err: %v", err) + } + + expectedClusterNames := sets.NewString(c.expectedClusterNames...) + if len(clusters) != expectedClusterNames.Len() { + t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters)) + } + for _, cluster := range clusters { + expectedClusterNames.Delete(cluster.Name) + } + if expectedClusterNames.Len() > 0 { + t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ",")) + } + }) + } + +} diff --git a/pkg/controllers/scheduling/schedule.go b/pkg/controllers/scheduling/schedule.go new file mode 100644 index 000000000..ec4b42d7e --- /dev/null +++ b/pkg/controllers/scheduling/schedule.go @@ -0,0 +1,148 @@ +package scheduling + +import ( + "context" + "fmt" + "reflect" + "sort" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + + clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned" + clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" +) + +type scheduleFunc func( + ctx context.Context, + placement *clusterapiv1alpha1.Placement, + clusters []*clusterapiv1.ManagedCluster, + clusterClient clusterclient.Interface, + placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister, +) (*scheduleResult, error) + +type scheduleResult struct { + scheduled int + unscheduled int +} + +func schedule( + ctx context.Context, + placement *clusterapiv1alpha1.Placement, + clusters []*clusterapiv1.ManagedCluster, + clusterClient clusterclient.Interface, + placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister, +) (*scheduleResult, error) { + // filter clusters with cluster predicates + feasibleClusters, err := matchWithClusterPredicates(placement.Spec.Predicates, clusters) + if err != nil { + return nil, err + } + + // select clusters and generate cluster decisions + decisions := selectClusters(placement, feasibleClusters) + scheduled, unscheduled := len(decisions), 0 + if placement.Spec.NumberOfClusters != nil { + unscheduled = int(*placement.Spec.NumberOfClusters) - scheduled + } + + // bind the cluster decisions into placementdecisions + err = bind(ctx, placement, decisions, clusterClient, placementDecisionLister) + if err != nil { + return nil, err + } + + return &scheduleResult{ + scheduled: scheduled, + unscheduled: unscheduled, + }, nil +} + +// makeClusterDecisions selects clusters based on given cluster slice and then creates +// cluster decisions. +func selectClusters(placement *clusterapiv1alpha1.Placement, clusters []*clusterapiv1.ManagedCluster) []clusterapiv1alpha1.ClusterDecision { + numOfDecisions := len(clusters) + if placement.Spec.NumberOfClusters != nil { + numOfDecisions = int(*placement.Spec.NumberOfClusters) + } + + // truncate the cluster slice if the desired number of decisions is less than + // the number of the candidate clusters + if numOfDecisions < len(clusters) { + clusters = clusters[:numOfDecisions] + } + + decisions := []clusterapiv1alpha1.ClusterDecision{} + for _, cluster := range clusters { + decisions = append(decisions, clusterapiv1alpha1.ClusterDecision{ + ClusterName: cluster.Name, + }) + } + return decisions +} + +// bind updates the cluster decisions in the status of the placementdecisions with the given +// cluster decision slice. New placementdecision will be created if no one exists. +func bind( + ctx context.Context, + placement *clusterapiv1alpha1.Placement, + clusterDecisions []clusterapiv1alpha1.ClusterDecision, + clusterClient clusterclient.Interface, + placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister, +) error { + // query placementdecisions with label selector + requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name}) + if err != nil { + return err + } + labelSelector := labels.NewSelector().Add(*requirement) + placementDecisions, err := placementDecisionLister.PlacementDecisions(placement.Namespace).List(labelSelector) + if err != nil { + return err + } + + // TODO: support multiple placementdecisions for a placement + var placementDecision *clusterapiv1alpha1.PlacementDecision + switch { + case len(placementDecisions) > 0: + placementDecision = placementDecisions[0] + default: + // create a placementdecision if not exists + owner := metav1.NewControllerRef(placement, clusterapiv1alpha1.GroupVersion.WithKind("Placement")) + placementDecision = &clusterapiv1alpha1.PlacementDecision{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-", placement.Name), + Namespace: placement.Namespace, + Labels: map[string]string{ + placementLabel: placement.Name, + }, + OwnerReferences: []metav1.OwnerReference{*owner}, + }, + } + placementDecision, err = clusterClient.ClusterV1alpha1().PlacementDecisions(placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{}) + if err != nil { + return err + } + } + + // sort by cluster name + sort.SliceStable(clusterDecisions, func(i, j int) bool { + return clusterDecisions[i].ClusterName < clusterDecisions[j].ClusterName + }) + + // update the status of the placementdecision if necessary + if reflect.DeepEqual(placementDecision.Status.Decisions, clusterDecisions) { + return nil + } + newPlacementDecision := placementDecision.DeepCopy() + newPlacementDecision.Status.Decisions = clusterDecisions + _, err = clusterClient.ClusterV1alpha1().PlacementDecisions(newPlacementDecision.Namespace). + UpdateStatus(ctx, newPlacementDecision, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil +} diff --git a/pkg/controllers/scheduling/schedule_test.go b/pkg/controllers/scheduling/schedule_test.go new file mode 100644 index 000000000..fa5b149d5 --- /dev/null +++ b/pkg/controllers/scheduling/schedule_test.go @@ -0,0 +1,185 @@ +package scheduling + +import ( + "context" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + clienttesting "k8s.io/client-go/testing" + + clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" + testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing" +) + +func TestSchedule(t *testing.T) { + clusterSetName := "clusterSets" + placementNamespace := "ns1" + placementName := "placement1" + placementDecisionName := "placement1-decision1" + + cases := []struct { + name string + placement *clusterapiv1alpha1.Placement + initObjs []runtime.Object + clusters []*clusterapiv1.ManagedCluster + scheduleResult scheduleResult + validateActions func(t *testing.T, actions []clienttesting.Action) + }{ + { + name: "new placement satisfied", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + scheduleResult: scheduleResult{ + scheduled: 1, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + // check if PlacementDecision has been created + testinghelpers.AssertActions(t, actions, "create", "update") + actual := actions[1].(clienttesting.UpdateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was updated") + } + assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1") + }, + }, + { + name: "new placement unsatisfied", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(3).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + scheduleResult: scheduleResult{ + scheduled: 1, + unscheduled: 2, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + // check if PlacementDecision has been updated + testinghelpers.AssertActions(t, actions, "create", "update") + actual := actions[1].(clienttesting.UpdateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was updated") + } + assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1") + }, + }, + { + name: "placement with all decisions scheduled", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(2).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName). + WithLabel(placementLabel, placementName). + WithDecisions("cluster1", "cluster2").Build(), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + scheduleResult: scheduleResult{ + scheduled: 2, + }, + validateActions: testinghelpers.AssertNoActions, + }, + { + name: "placement with part of decisions scheduled", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName). + WithLabel(placementLabel, placementName).WithDecisions("cluster1").Build(), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + scheduleResult: scheduleResult{ + scheduled: 2, + unscheduled: 2, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + // check if PlacementDecision has been updated + testinghelpers.AssertActions(t, actions, "update") + actual := actions[0].(clienttesting.UpdateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was updated") + } + assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1", "cluster2") + }, + }, + { + name: "placement without more feasible cluster available", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName). + WithLabel(placementLabel, placementName).WithDecisions("cluster1").Build(), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + scheduleResult: scheduleResult{ + scheduled: 1, + unscheduled: 3, + }, + validateActions: testinghelpers.AssertNoActions, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.initObjs = append(c.initObjs, c.placement) + clusterClient := clusterfake.NewSimpleClientset(c.initObjs...) + clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...) + result, err := schedule( + context.TODO(), + c.placement, + c.clusters, + clusterClient, + clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(), + ) + if err != nil { + t.Errorf("unexpected err: %v", err) + } + if result.scheduled != c.scheduleResult.scheduled { + t.Errorf("expected %d scheduled, but got %d", c.scheduleResult.scheduled, result.scheduled) + } + if result.unscheduled != c.scheduleResult.unscheduled { + t.Errorf("expected %d unscheduled, but got %d", c.scheduleResult.unscheduled, result.unscheduled) + } + c.validateActions(t, clusterClient.Actions()) + }) + } +} + +func assertClustersSelected(t *testing.T, decisons []clusterapiv1alpha1.ClusterDecision, clusterNames ...string) { + names := sets.NewString(clusterNames...) + for _, decision := range decisons { + if names.Has(decision.ClusterName) { + names.Delete(decision.ClusterName) + } + } + + if names.Len() != 0 { + t.Errorf("expected clusters selected: %s", strings.Join(names.UnsortedList(), ",")) + } +} diff --git a/pkg/controllers/scheduling/scheduling_controller.go b/pkg/controllers/scheduling/scheduling_controller.go new file mode 100644 index 000000000..668a29d34 --- /dev/null +++ b/pkg/controllers/scheduling/scheduling_controller.go @@ -0,0 +1,226 @@ +package scheduling + +import ( + "context" + "fmt" + "reflect" + "time" + + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/events" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + cache "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned" + clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1" + clusterlisterv1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1" + clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" +) + +const ( + clusterSetLabel = "cluster.open-cluster-management.io/clusterset" + placementLabel = "cluster.open-cluster-management.io/placement" +) + +var ResyncInterval = 2 * time.Minute + +// schedulingController schedules cluster decisions for Placements +type schedulingController struct { + clusterClient clusterclient.Interface + clusterLister clusterlisterv1.ManagedClusterLister + clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister + clusterSetBindingLister clusterlisterv1alpha1.ManagedClusterSetBindingLister + placementLister clusterlisterv1alpha1.PlacementLister + placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister + scheduleFunc scheduleFunc +} + +// NewDecisionSchedulingController return an instance of schedulingController +func NewSchedulingController( + clusterClient clusterclient.Interface, + clusterLister clusterlisterv1.ManagedClusterLister, + clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister, + clusterSetBindingLister clusterlisterv1alpha1.ManagedClusterSetBindingLister, + placementInformer clusterinformerv1alpha1.PlacementInformer, + placementDecisionInformer clusterinformerv1alpha1.PlacementDecisionInformer, + recorder events.Recorder, +) factory.Controller { + // build controller + c := schedulingController{ + clusterClient: clusterClient, + clusterLister: clusterLister, + clusterSetLister: clusterSetLister, + clusterSetBindingLister: clusterSetBindingLister, + placementLister: placementInformer.Lister(), + placementDecisionLister: placementDecisionInformer.Lister(), + scheduleFunc: schedule, + } + + return factory.New(). + WithInformersQueueKeyFunc(func(obj runtime.Object) string { + key, _ := cache.MetaNamespaceKeyFunc(obj) + return key + }, placementInformer.Informer()). + WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string { + accessor, _ := meta.Accessor(obj) + labels := accessor.GetLabels() + placementName := labels[placementLabel] + return fmt.Sprintf("%s/%s", accessor.GetNamespace(), placementName) + }, func(obj interface{}) bool { + accessor, err := meta.Accessor(obj) + if err != nil { + return false + } + labels := accessor.GetLabels() + if _, ok := labels[placementLabel]; ok { + return true + } + return false + }, placementDecisionInformer.Informer()). + // TODO: monitor more resources, like clusters, clustersets and clustersetbindings + WithSync(c.sync). + ResyncEvery(ResyncInterval). + ToController("SchedulingController", recorder) +} + +func (c *schedulingController) sync(ctx context.Context, syncCtx factory.SyncContext) error { + queueKey := syncCtx.QueueKey() + + // handle resync + if queueKey == factory.DefaultQueueKey { + placements, err := c.placementLister.List(labels.Everything()) + if err != nil { + return err + } + for _, placement := range placements { + syncCtx.Queue().Add(fmt.Sprintf("%s/%s", placement.Namespace, placement.Name)) + } + return nil + } + + // sync a placement + namespace, name, err := cache.SplitMetaNamespaceKey(queueKey) + if err != nil { + // ignore placement whose key is not in format: namespace/name + utilruntime.HandleError(err) + return nil + } + + klog.V(4).Infof("Reconciling placement %q", queueKey) + placement, err := c.placementLister.Placements(namespace).Get(name) + if errors.IsNotFound(err) { + // no work if placement is deleted + return nil + } + if err != nil { + return err + } + + // no work if placement is deleting + if !placement.DeletionTimestamp.IsZero() { + return nil + } + + // get available clusters for this placement + clusters, err := c.getAvailableClusters(placement) + if err != nil { + return err + } + + // schedule placement with scheduler + scheduleResult, err := c.scheduleFunc(ctx, placement, clusters, c.clusterClient, c.placementDecisionLister) + if err != nil { + return err + } + + // update placement status if necessary + return c.updateStatus(ctx, placement, scheduleResult.scheduled, scheduleResult.unscheduled) +} + +// getAvailableClusters returns available clusters for the given placement. The clusters must +// 1) Be from clustersets bound to the placement namespace; +// 2) Belong to one of particular clustersets if .spec.clusterSets is specified; +func (c *schedulingController) getAvailableClusters(placement *clusterapiv1alpha1.Placement) ([]*clusterapiv1.ManagedCluster, error) { + // get all clusterset bindings under the placement namespace + bindings, err := c.clusterSetBindingLister.ManagedClusterSetBindings(placement.Namespace).List(labels.Everything()) + if err != nil { + return nil, err + } + if len(bindings) == 0 { + return nil, nil + } + + // filter out invaid clustersetbindings + clusterSetNames := sets.NewString() + for _, binding := range bindings { + // ignore clusterset does not exist + _, err := c.clusterSetLister.Get(binding.Name) + if errors.IsNotFound(err) { + continue + } + if err != nil { + return nil, err + } + + clusterSetNames.Insert(binding.Name) + } + + // get intersection of clustesets bound to placement namespace and clustesets specified + // in placement spec + if len(placement.Spec.ClusterSets) != 0 { + clusterSetNames = clusterSetNames.Intersection(sets.NewString(placement.Spec.ClusterSets...)) + } + + if len(clusterSetNames) == 0 { + return nil, nil + } + + // list clusters from particular clustersets + requirement, err := labels.NewRequirement(clusterSetLabel, selection.In, clusterSetNames.List()) + if err != nil { + return nil, err + } + labelSelector := labels.NewSelector().Add(*requirement) + return c.clusterLister.List(labelSelector) +} + +// updateStatus updates the status of the placement according to schedule result. +func (c *schedulingController) updateStatus(ctx context.Context, placement *clusterapiv1alpha1.Placement, numOfScheduledDecisions, numOfUnscheduledDecisions int) error { + newPlacement := placement.DeepCopy() + newPlacement.Status.NumberOfSelectedClusters = int32(numOfScheduledDecisions) + satisfiedCondition := newSatisfiedCondition(numOfUnscheduledDecisions) + meta.SetStatusCondition(&newPlacement.Status.Conditions, satisfiedCondition) + if reflect.DeepEqual(newPlacement.Status, placement.Status) { + return nil + } + _, err := c.clusterClient.ClusterV1alpha1().Placements(newPlacement.Namespace).UpdateStatus(ctx, newPlacement, metav1.UpdateOptions{}) + return err +} + +// newSatisfiedCondition returns a new condition with type PlacementConditionSatisfied +func newSatisfiedCondition(numOfUnscheduledDecisions int) metav1.Condition { + condition := metav1.Condition{ + Type: clusterapiv1alpha1.PlacementConditionSatisfied, + } + switch { + case numOfUnscheduledDecisions == 0: + condition.Status = metav1.ConditionTrue + condition.Reason = "AllDecisionsScheduled" + condition.Message = "All cluster decisions scheduled" + default: + condition.Status = metav1.ConditionFalse + condition.Reason = "NotAllDecisionsScheduled" + condition.Message = fmt.Sprintf("%d cluster decisions unscheduled", numOfUnscheduledDecisions) + } + return condition +} diff --git a/pkg/controllers/scheduling/scheduling_controller_test.go b/pkg/controllers/scheduling/scheduling_controller_test.go new file mode 100644 index 000000000..f5e064deb --- /dev/null +++ b/pkg/controllers/scheduling/scheduling_controller_test.go @@ -0,0 +1,210 @@ +package scheduling + +import ( + "context" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + clienttesting "k8s.io/client-go/testing" + + clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned" + clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake" + clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" + testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing" +) + +func TestSchedulingController_sync(t *testing.T) { + placementNamespace := "ns1" + placementName := "placement1" + + cases := []struct { + name string + placement *clusterapiv1alpha1.Placement + initObjs []runtime.Object + scheduleResult *scheduleResult + validateActions func(t *testing.T, actions []clienttesting.Action) + }{ + { + name: "placement satisfied", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + scheduleResult: &scheduleResult{ + scheduled: 3, + unscheduled: 0, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + // check if PlacementDecision has been updated + testinghelpers.AssertActions(t, actions, "update") + // check if Placement has been updated + actual := actions[0].(clienttesting.UpdateActionImpl).Object + placement, ok := actual.(*clusterapiv1alpha1.Placement) + if !ok { + t.Errorf("expected Placement was updated") + } + + if placement.Status.NumberOfSelectedClusters != int32(3) { + t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters) + } + testinghelpers.HasCondition( + placement.Status.Conditions, + clusterapiv1alpha1.PlacementConditionSatisfied, + "AllDecisionsScheduled", + metav1.ConditionTrue, + ) + }, + }, + { + name: "placement unsatisfied", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + scheduleResult: &scheduleResult{ + scheduled: 3, + unscheduled: 1, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + // check if PlacementDecision has been updated + testinghelpers.AssertActions(t, actions, "update") + // check if Placement has been updated + actual := actions[0].(clienttesting.UpdateActionImpl).Object + placement, ok := actual.(*clusterapiv1alpha1.Placement) + if !ok { + t.Errorf("expected Placement was updated") + } + + if placement.Status.NumberOfSelectedClusters != int32(3) { + t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters) + } + testinghelpers.HasCondition( + placement.Status.Conditions, + clusterapiv1alpha1.PlacementConditionSatisfied, + "NotAllDecisionsScheduled", + metav1.ConditionFalse, + ) + }, + }, + { + name: "placement status not changed", + placement: testinghelpers.NewPlacement(placementNamespace, placementName). + WithNumOfSelectedClusters(3).WithSatisfiedCondition(3, 0).Build(), + scheduleResult: &scheduleResult{ + scheduled: 3, + unscheduled: 0, + }, + validateActions: testinghelpers.AssertNoActions, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.initObjs = append(c.initObjs, c.placement) + clusterClient := clusterfake.NewSimpleClientset(c.initObjs...) + clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...) + + ctrl := schedulingController{ + clusterClient: clusterClient, + clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(), + clusterSetLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Lister(), + clusterSetBindingLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(), + placementLister: clusterInformerFactory.Cluster().V1alpha1().Placements().Lister(), + placementDecisionLister: clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(), + scheduleFunc: func( + ctx context.Context, + placement *clusterapiv1alpha1.Placement, + clusters []*clusterapiv1.ManagedCluster, + clusterClient clusterclient.Interface, + placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister, + ) (*scheduleResult, error) { + return c.scheduleResult, nil + }, + } + + sysCtx := testinghelpers.NewFakeSyncContext(t, c.placement.Namespace+"/"+c.placement.Name) + syncErr := ctrl.sync(context.TODO(), sysCtx) + if syncErr != nil { + t.Errorf("unexpected err: %v", syncErr) + } + + c.validateActions(t, clusterClient.Actions()) + }) + } +} + +func TestGetAvailableClusters(t *testing.T) { + placementNamespace := "ns1" + placementName := "placement1" + + cases := []struct { + name string + placement *clusterapiv1alpha1.Placement + initObjs []runtime.Object + expectedClusterNames []string + }{ + { + name: "no bound clusterset", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet("clusterset1"), + testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"), + }, + }, + { + name: "select all clusters from bound clustersets", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet("clusterset1"), + testinghelpers.NewClusterSet("clusterset2"), + testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"), + testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset2"), + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, "clusterset2").Build(), + }, + expectedClusterNames: []string{"cluster1", "cluster2"}, + }, + { + name: "select clusters from a bound clusterset", + placement: testinghelpers.NewPlacement(placementNamespace, placementName). + WithClusterSets([]string{"clusterset1"}).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet("clusterset1"), + testinghelpers.NewClusterSet("clusterset2"), + testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"), + testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset2"), + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, "clusterset2").Build(), + }, + expectedClusterNames: []string{"cluster1"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.initObjs = append(c.initObjs, c.placement) + clusterClient := clusterfake.NewSimpleClientset(c.initObjs...) + clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...) + + ctrl := &schedulingController{ + clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(), + clusterSetLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Lister(), + clusterSetBindingLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(), + } + clusters, err := ctrl.getAvailableClusters(c.placement) + if err != nil { + t.Errorf("unexpected err: %v", err) + } + + expectedClusterNames := sets.NewString(c.expectedClusterNames...) + if len(clusters) != expectedClusterNames.Len() { + t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters)) + } + for _, cluster := range clusters { + expectedClusterNames.Delete(cluster.Name) + } + if expectedClusterNames.Len() > 0 { + t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ",")) + } + }) + } +} diff --git a/pkg/helpers/testing/builders.go b/pkg/helpers/testing/builders.go index 1f1f4cc8b..a7aff0aee 100644 --- a/pkg/helpers/testing/builders.go +++ b/pkg/helpers/testing/builders.go @@ -1,16 +1,16 @@ package testing import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" ) -const ( - placementLabel = "cluster.open-cluster-management.io/placement" -) - type placementBuilder struct { placement *clusterapiv1alpha1.Placement } @@ -31,16 +31,73 @@ func (b *placementBuilder) WithUID(uid string) *placementBuilder { return b } +func (b *placementBuilder) WithNOC(noc int32) *placementBuilder { + b.placement.Spec.NumberOfClusters = &noc + return b +} + +func (b *placementBuilder) WithClusterSets(clusterSets []string) *placementBuilder { + b.placement.Spec.ClusterSets = clusterSets + return b +} + func (b *placementBuilder) WithDeletionTimestamp() *placementBuilder { now := metav1.Now() b.placement.DeletionTimestamp = &now return b } +func (b *placementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1alpha1.ClusterClaimSelector) *placementBuilder { + if b.placement.Spec.Predicates == nil { + b.placement.Spec.Predicates = []clusterapiv1alpha1.ClusterPredicate{} + } + b.placement.Spec.Predicates = append(b.placement.Spec.Predicates, NewClusterPredicate(labelSelector, claimSelector)) + return b +} + +func (b *placementBuilder) WithNumOfSelectedClusters(nosc int) *placementBuilder { + b.placement.Status.NumberOfSelectedClusters = int32(nosc) + return b +} + +func (b *placementBuilder) WithSatisfiedCondition(numbOfScheduledDecisions, numbOfUnscheduledDecisions int) *placementBuilder { + condition := metav1.Condition{ + Type: clusterapiv1alpha1.PlacementConditionSatisfied, + } + switch { + case numbOfUnscheduledDecisions == 0: + condition.Status = metav1.ConditionTrue + condition.Reason = "AllDecisionsScheduled" + condition.Message = "All cluster decisions scheduled" + default: + condition.Status = metav1.ConditionFalse + condition.Reason = "NotAllDecisionsScheduled" + condition.Message = fmt.Sprintf("%d cluster decisions unscheduled", numbOfUnscheduledDecisions) + } + meta.SetStatusCondition(&b.placement.Status.Conditions, condition) + return b +} + func (b *placementBuilder) Build() *clusterapiv1alpha1.Placement { return b.placement } +func NewClusterPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1alpha1.ClusterClaimSelector) clusterapiv1alpha1.ClusterPredicate { + predicate := clusterapiv1alpha1.ClusterPredicate{ + RequiredClusterSelector: clusterapiv1alpha1.ClusterSelector{}, + } + + if labelSelector != nil { + predicate.RequiredClusterSelector.LabelSelector = *labelSelector + } + + if claimSelector != nil { + predicate.RequiredClusterSelector.ClaimSelector = *claimSelector + } + + return predicate +} + type placementDecisionBuilder struct { placementDecision *clusterapiv1alpha1.PlacementDecision } @@ -65,14 +122,96 @@ func (b *placementDecisionBuilder) WithController(uid string) *placementDecision return b } -func (b *placementDecisionBuilder) WithPlacementLabel(placementName string) *placementDecisionBuilder { +func (b *placementDecisionBuilder) WithLabel(name, value string) *placementDecisionBuilder { if b.placementDecision.Labels == nil { b.placementDecision.Labels = map[string]string{} } - b.placementDecision.Labels[placementLabel] = placementName + b.placementDecision.Labels[name] = value + return b +} + +func (b *placementDecisionBuilder) WithDeletionTimestamp() *placementDecisionBuilder { + now := metav1.Now() + b.placementDecision.DeletionTimestamp = &now + return b +} + +func (b *placementDecisionBuilder) WithDecisions(clusterNames ...string) *placementDecisionBuilder { + decisions := []clusterapiv1alpha1.ClusterDecision{} + for _, clusterName := range clusterNames { + decisions = append(decisions, clusterapiv1alpha1.ClusterDecision{ + ClusterName: clusterName, + }) + } + b.placementDecision.Status.Decisions = decisions return b } func (b *placementDecisionBuilder) Build() *clusterapiv1alpha1.PlacementDecision { return b.placementDecision } + +type managedClusterBuilder struct { + cluster *clusterapiv1.ManagedCluster +} + +func NewManagedCluster(clusterName string) *managedClusterBuilder { + return &managedClusterBuilder{ + cluster: &clusterapiv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + }, + }, + } +} + +func (b *managedClusterBuilder) WithLabel(name, value string) *managedClusterBuilder { + if b.cluster.Labels == nil { + b.cluster.Labels = map[string]string{} + } + b.cluster.Labels[name] = value + return b +} + +func (b *managedClusterBuilder) WithClaim(name, value string) *managedClusterBuilder { + claimMap := map[string]string{} + for _, claim := range b.cluster.Status.ClusterClaims { + claimMap[claim.Name] = claim.Value + } + claimMap[name] = value + + clusterClaims := []clusterapiv1.ManagedClusterClaim{} + for k, v := range claimMap { + clusterClaims = append(clusterClaims, clusterapiv1.ManagedClusterClaim{ + Name: k, + Value: v, + }) + } + + b.cluster.Status.ClusterClaims = clusterClaims + return b +} + +func (b *managedClusterBuilder) Build() *clusterapiv1.ManagedCluster { + return b.cluster +} + +func NewClusterSet(clusterSetName string) *clusterapiv1alpha1.ManagedClusterSet { + return &clusterapiv1alpha1.ManagedClusterSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterSetName, + }, + } +} + +func NewClusterSetBinding(namespace, clusterSetName string) *clusterapiv1alpha1.ManagedClusterSetBinding { + return &clusterapiv1alpha1.ManagedClusterSetBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: clusterSetName, + }, + Spec: clusterapiv1alpha1.ManagedClusterSetBindingSpec{ + ClusterSet: clusterSetName, + }, + } +} diff --git a/pkg/helpers/testing/helpers.go b/pkg/helpers/testing/helpers.go index 82148cdd6..c63dceb3e 100644 --- a/pkg/helpers/testing/helpers.go +++ b/pkg/helpers/testing/helpers.go @@ -5,6 +5,7 @@ import ( "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/events/eventstesting" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/util/workqueue" ) @@ -43,3 +44,25 @@ func AssertActions(t *testing.T, actualActions []clienttesting.Action, expectedV func AssertNoActions(t *testing.T, actualActions []clienttesting.Action) { AssertActions(t, actualActions) } + +func HasCondition(conditions []metav1.Condition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool { + found := false + for _, condition := range conditions { + if condition.Type != expectedType { + continue + } + found = true + + if condition.Status != expectedStatus { + return false + } + + if condition.Reason != expectedReason { + return false + } + + return true + } + + return found +} diff --git a/pkg/helpers/testing/informer.go b/pkg/helpers/testing/informer.go new file mode 100644 index 000000000..72fbac139 --- /dev/null +++ b/pkg/helpers/testing/informer.go @@ -0,0 +1,37 @@ +package testing + +import ( + "time" + + clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned" + clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" + clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" +) + +func NewClusterInformerFactory(clusterClient clusterclient.Interface, objects ...runtime.Object) clusterinformers.SharedInformerFactory { + clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10) + clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore() + clusterSetStore := clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Informer().GetStore() + clusterSetBindingStore := clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Informer().GetStore() + placementStore := clusterInformerFactory.Cluster().V1alpha1().Placements().Informer().GetStore() + placementDecisionStore := clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Informer().GetStore() + + for _, obj := range objects { + switch obj.(type) { + case *clusterapiv1.ManagedCluster: + clusterStore.Add(obj) + case *clusterapiv1alpha1.ManagedClusterSet: + clusterSetStore.Add(obj) + case *clusterapiv1alpha1.ManagedClusterSetBinding: + clusterSetBindingStore.Add(obj) + case *clusterapiv1alpha1.Placement: + placementStore.Add(obj) + case *clusterapiv1alpha1.PlacementDecision: + placementDecisionStore.Add(obj) + } + } + + return clusterInformerFactory +} diff --git a/test/integration/placement_test.go b/test/integration/placement_test.go index cb78d6a2e..2d4067785 100644 --- a/test/integration/placement_test.go +++ b/test/integration/placement_test.go @@ -3,6 +3,7 @@ package integration import ( "context" "fmt" + "time" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -12,25 +13,32 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" + clusterapiv1 "github.com/open-cluster-management/api/cluster/v1" clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1" controllers "github.com/open-cluster-management/placement/pkg/controllers" + "github.com/open-cluster-management/placement/pkg/controllers/scheduling" "github.com/open-cluster-management/placement/test/integration/util" ) const ( - placementLabel = "cluster.open-cluster-management.io/placement" + clusterSetLabel = "cluster.open-cluster-management.io/clusterset" + placementLabel = "cluster.open-cluster-management.io/placement" ) -var _ = ginkgo.Describe("Placement Scheduling", func() { +var _ = ginkgo.Describe("Placement", func() { var cancel context.CancelFunc var namespace string var placementName string + var clusterSet1Name, clusterSet2Name string + var suffix string var err error ginkgo.BeforeEach(func() { - suffix := rand.String(5) + suffix = rand.String(5) namespace = fmt.Sprintf("ns-%s", suffix) placementName = fmt.Sprintf("placement-%s", suffix) + clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix) + clusterSet2Name = fmt.Sprintf("clusterset-%s", rand.String(5)) // create testing namespace ns := &corev1.Namespace{ @@ -42,6 +50,7 @@ var _ = ginkgo.Describe("Placement Scheduling", func() { gomega.Expect(err).ToNot(gomega.HaveOccurred()) // start controller manager + scheduling.ResyncInterval = 10 * time.Second var ctx context.Context ctx, cancel = context.WithCancel(context.Background()) go controllers.RunControllerManager(ctx, &controllercmd.ControllerContext{ @@ -58,22 +67,11 @@ var _ = ginkgo.Describe("Placement Scheduling", func() { gomega.Expect(err).ToNot(gomega.HaveOccurred()) }) - ginkgo.It("Should create placement successfully", func() { - ginkgo.By("Create placement") - placement := &clusterapiv1alpha1.Placement{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: placementName, - }, - } - - placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) - gomega.Expect(err).ToNot(gomega.HaveOccurred()) - + assertPlacementDecisionCreated := func(placement *clusterapiv1alpha1.Placement) { ginkgo.By("Check if placementdecision is created") gomega.Eventually(func() bool { pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: placementLabel + "=" + placementName, + LabelSelector: placementLabel + "=" + placement.Name, }) if err != nil { return false @@ -88,11 +86,9 @@ var _ = ginkgo.Describe("Placement Scheduling", func() { } return true }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + } - ginkgo.By("Delete placement") - err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.Background(), placementName, metav1.DeleteOptions{}) - gomega.Expect(err).ToNot(gomega.HaveOccurred()) - + assertPlacementDeleted := func(placementName string) { ginkgo.By("Check if placement is gone") gomega.Eventually(func() bool { _, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) @@ -101,5 +97,189 @@ var _ = ginkgo.Describe("Placement Scheduling", func() { } return errors.IsNotFound(err) }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + } + + assertNumberOfDecisions := func(placementName string, desiredNOD int) { + ginkgo.By("Check the number of decisions in placementdecisions") + gomega.Eventually(func() bool { + pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: placementLabel + "=" + placementName, + }) + if err != nil { + return false + } + if len(pdl.Items) == 0 { + return false + } + actualNOD := 0 + for _, pd := range pdl.Items { + actualNOD += len(pd.Status.Decisions) + } + return actualNOD == desiredNOD + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + } + + assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) { + ginkgo.By("Check the status of placement") + gomega.Eventually(func() bool { + placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) + if err != nil { + return false + } + if satisfied && !util.HasCondition( + placement.Status.Conditions, + clusterapiv1alpha1.PlacementConditionSatisfied, + "AllDecisionsScheduled", + metav1.ConditionTrue, + ) { + return false + } + if !satisfied && !util.HasCondition( + placement.Status.Conditions, + clusterapiv1alpha1.PlacementConditionSatisfied, + "NotAllDecisionsScheduled", + metav1.ConditionFalse, + ) { + return false + } + return placement.Status.NumberOfSelectedClusters == int32(numOfSelectedClusters) + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + } + + assertBindingClusterSet := func(clusterSetName string) { + ginkgo.By("Create clusterset/clustersetbinding") + clusterset := &clusterapiv1alpha1.ManagedClusterSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterSetName, + }, + } + _, err = clusterClient.ClusterV1alpha1().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + csb := &clusterapiv1alpha1.ManagedClusterSetBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: clusterSetName, + }, + Spec: clusterapiv1alpha1.ManagedClusterSetBindingSpec{ + ClusterSet: clusterSetName, + }, + } + _, err = clusterClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + assertCreatingClusters := func(clusterSetName string, num int) { + ginkgo.By(fmt.Sprintf("Create %d clusters", num)) + for i := 0; i < num; i++ { + cluster := &clusterapiv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cluster-", + Labels: map[string]string{ + clusterSetLabel: clusterSetName, + }, + }, + } + _, err = clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + } + + assertCreatingPlacement := func(name string, noc *int32, nod int) { + ginkgo.By("Create placement") + placement := &clusterapiv1alpha1.Placement{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: clusterapiv1alpha1.PlacementSpec{ + NumberOfClusters: noc, + }, + } + placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + assertPlacementDecisionCreated(placement) + assertNumberOfDecisions(placementName, nod) + if noc != nil { + assertPlacementStatus(placementName, nod, nod == int(*noc)) + } + } + + ginkgo.Context("Scheduling", func() { + ginkgo.AfterEach(func() { + ginkgo.By("Delete placement") + err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.Background(), placementName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + assertPlacementDeleted(placementName) + }) + + ginkgo.It("Should schedule successfully once spec.NumberOfClusters is reduced", func() { + assertBindingClusterSet(clusterSet1Name) + assertCreatingClusters(clusterSet1Name, 5) + assertCreatingPlacement(placementName, noc(10), 5) + + ginkgo.By("Reduce NOC of the placement") + placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + noc := int32(4) + placement.Spec.NumberOfClusters = &noc + placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + nod := int(noc) + assertNumberOfDecisions(placementName, nod) + assertPlacementStatus(placementName, nod, true) + }) + + ginkgo.It("Should schedule successfully once spec.NumberOfClusters is increased", func() { + assertBindingClusterSet(clusterSet1Name) + assertCreatingClusters(clusterSet1Name, 10) + assertCreatingPlacement(placementName, noc(5), 5) + + ginkgo.By("Increase NOC of the placement") + placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + noc := int32(8) + placement.Spec.NumberOfClusters = &noc + placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + nod := int(noc) + assertNumberOfDecisions(placementName, nod) + assertPlacementStatus(placementName, nod, true) + }) + + ginkgo.It("Should be satisfied once new clusters are added", func() { + assertBindingClusterSet(clusterSet1Name) + assertCreatingClusters(clusterSet1Name, 5) + assertCreatingPlacement(placementName, noc(10), 5) + + // add more clusters + assertCreatingClusters(clusterSet1Name, 5) + + nod := 10 + assertNumberOfDecisions(placementName, nod) + assertPlacementStatus(placementName, nod, true) + }) + + ginkgo.It("Should schedule successfully once new clusterset is bound", func() { + assertBindingClusterSet(clusterSet1Name) + assertCreatingClusters(clusterSet1Name, 5) + assertCreatingPlacement(placementName, noc(10), 5) + + ginkgo.By("Bind one more clusterset to the placement namespace") + assertBindingClusterSet(clusterSet2Name) + assertCreatingClusters(clusterSet2Name, 3) + + nod := 8 + assertNumberOfDecisions(placementName, nod) + assertPlacementStatus(placementName, nod, false) + }) }) }) + +func noc(n int) *int32 { + noc := int32(n) + return &noc +} diff --git a/test/integration/util/util.go b/test/integration/util/util.go index c7665974c..ee880ebd2 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/onsi/ginkgo" - "github.com/openshift/library-go/pkg/operator/events" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func NewIntegrationTestEventRecorder(componet string) events.Recorder { @@ -47,3 +47,25 @@ func (r *IntegrationTestEventRecorder) Warningf(reason, messageFmt string, args func (r *IntegrationTestEventRecorder) Shutdown() { return } + +func HasCondition(conditions []metav1.Condition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool { + found := false + for _, condition := range conditions { + if condition.Type != expectedType { + continue + } + found = true + + if condition.Status != expectedStatus { + return false + } + + if condition.Reason != expectedReason { + return false + } + + return true + } + + return found +}