add scheduling controller

Signed-off-by: Yang Le <yangle@redhat.com>
This commit is contained in:
Yang Le
2021-05-07 18:32:50 +08:00
parent ef56ab5e72
commit 3d64c80b75
16 changed files with 1428 additions and 420 deletions

View File

@@ -8,8 +8,7 @@ import (
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
placement "github.com/open-cluster-management/placement/pkg/controllers/placement"
placementdecision "github.com/open-cluster-management/placement/pkg/controllers/placementdecision"
scheduling "github.com/open-cluster-management/placement/pkg/controllers/scheduling"
)
// RunControllerManager starts the controllers on hub to make placement decisions.
@@ -20,14 +19,11 @@ func RunControllerManager(ctx context.Context, controllerContext *controllercmd.
}
clusterInformers := clusterinformers.NewSharedInformerFactory(clusterClient, 10*time.Minute)
placementController := placement.NewPlacementController(
clusterInformers.Cluster().V1().ManagedClusters(),
clusterInformers.Cluster().V1alpha1().ManagedClusterSets(),
controllerContext.EventRecorder,
)
placementDecisionCreatingController := placementdecision.NewPlacementDecisionCreatingController(
schedulingController := scheduling.NewSchedulingController(
clusterClient,
clusterInformers.Cluster().V1().ManagedClusters().Lister(),
clusterInformers.Cluster().V1alpha1().ManagedClusterSets().Lister(),
clusterInformers.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(),
clusterInformers.Cluster().V1alpha1().Placements(),
clusterInformers.Cluster().V1alpha1().PlacementDecisions(),
controllerContext.EventRecorder,
@@ -35,8 +31,7 @@ func RunControllerManager(ctx context.Context, controllerContext *controllercmd.
go clusterInformers.Start(ctx.Done())
go placementController.Run(ctx, 1)
go placementDecisionCreatingController.Run(ctx, 1)
go schedulingController.Run(ctx, 1)
<-ctx.Done()
return nil

View File

@@ -1,66 +0,0 @@
package placement
import (
"context"
"fmt"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/events"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
clusterinformerv1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1"
clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1"
clusterlisterv1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1"
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
)
const (
clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
)
// placementController makes placement decisions for Placements
type placementController struct {
clusterLister clusterlisterv1.ManagedClusterLister
clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister
}
// NewPlacementController return an instance of placementController
func NewPlacementController(
clusterInformer clusterinformerv1.ManagedClusterInformer,
clusterSetInformer clusterinformerv1alpha1.ManagedClusterSetInformer,
recorder events.Recorder,
) factory.Controller {
c := placementController{
clusterLister: clusterInformer.Lister(),
clusterSetLister: clusterSetInformer.Lister(),
}
return factory.New().
WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string {
accessor, _ := meta.Accessor(obj)
return fmt.Sprintf("cluster:%s", accessor.GetName())
}, func(obj interface{}) bool {
accessor, err := meta.Accessor(obj)
if err != nil {
return false
}
// ignore cluster belongs to no clusterset
labels := accessor.GetLabels()
clusterSetName, ok := labels[clusterSetLabel]
if !ok {
return false
}
// ignore cluster if its clusterset does not exist
_, err = c.clusterSetLister.Get(clusterSetName)
return err == nil
}, clusterInformer.Informer()).
WithSync(c.sync).
ToController("PlacementController", recorder)
}
func (c *placementController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
return nil
}

View File

@@ -1,44 +0,0 @@
package placement
import (
"context"
"testing"
"time"
"k8s.io/apimachinery/pkg/runtime"
clienttesting "k8s.io/client-go/testing"
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
)
func TestSync(t *testing.T) {
cases := []struct {
name string
queueKey string
initObjs []runtime.Object
validateActions func(t *testing.T, hubActions, agentActions []clienttesting.Action)
}{}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)
clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore()
for _, cluster := range c.initObjs {
clusterStore.Add(cluster)
}
ctrl := placementController{
clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(),
}
syncErr := ctrl.sync(context.TODO(), testinghelpers.NewFakeSyncContext(t, c.queueKey))
if syncErr != nil {
t.Errorf("unexpected err: %v", syncErr)
}
//c.validateActions(t, nil)
})
}
}

View File

@@ -1,132 +0,0 @@
package placementdecision
import (
"context"
"fmt"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/events"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
cache "k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1"
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
)
const (
placementLabel = "cluster.open-cluster-management.io/placement"
)
// placementDecisionCreatingController creates PlacementDecisions for Placements
type placementDecisionCreatingController struct {
clusterClient clusterclient.Interface
placementLister clusterlisterv1alpha1.PlacementLister
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister
}
// NewPlacementDecisionCreatingController return an instance of placementDecisionCreatingController
func NewPlacementDecisionCreatingController(
clusterClient clusterclient.Interface,
placementInformer clusterinformerv1alpha1.PlacementInformer,
placementDecisionInformer clusterinformerv1alpha1.PlacementDecisionInformer,
recorder events.Recorder,
) factory.Controller {
c := placementDecisionCreatingController{
clusterClient: clusterClient,
placementLister: placementInformer.Lister(),
placementDecisionLister: placementDecisionInformer.Lister(),
}
return factory.New().
WithInformersQueueKeyFunc(func(obj runtime.Object) string {
key, _ := cache.MetaNamespaceKeyFunc(obj)
return key
}, placementInformer.Informer()).
WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string {
accessor, _ := meta.Accessor(obj)
labels := accessor.GetLabels()
placementName := labels[placementLabel]
return fmt.Sprintf("%s/%s", accessor.GetNamespace(), placementName)
}, func(obj interface{}) bool {
accessor, err := meta.Accessor(obj)
if err != nil {
return false
}
labels := accessor.GetLabels()
if _, ok := labels[placementLabel]; ok {
return true
}
return false
}, placementDecisionInformer.Informer()).
WithSync(c.sync).
ToController("PlacementDecisionCreatingController", recorder)
}
func (c *placementDecisionCreatingController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
queueKey := syncCtx.QueueKey()
namespace, name, err := cache.SplitMetaNamespaceKey(queueKey)
if err != nil {
// ignore placement whose key is not in format: namespace/name
return nil
}
klog.V(4).Infof("Reconciling placement %q", queueKey)
placement, err := c.placementLister.Placements(namespace).Get(name)
if errors.IsNotFound(err) {
// no work if placement is deleted
return nil
}
if err != nil {
return err
}
// no work if placement is deleting
if !placement.DeletionTimestamp.IsZero() {
return nil
}
// query placementdecisions with label selector
requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name})
if err != nil {
return err
}
labelSelector := labels.NewSelector().Add(*requirement)
placementDecisions, err := c.placementDecisionLister.PlacementDecisions(namespace).List(labelSelector)
if err != nil {
return err
}
// no work if PlacementDecision has been created
if len(placementDecisions) > 0 {
return nil
}
// otherwise create placementdecision
return c.createPlacementDecision(ctx, placement)
}
// createPlacementDecision creates PlacementDecision for the Placement
func (c *placementDecisionCreatingController) createPlacementDecision(ctx context.Context, placement *clusterapiv1alpha1.Placement) error {
owner := metav1.NewControllerRef(placement, clusterapiv1alpha1.GroupVersion.WithKind("Placement"))
placementDecision := &clusterapiv1alpha1.PlacementDecision{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-", placement.Name),
Namespace: placement.Namespace,
Labels: map[string]string{
placementLabel: placement.Name,
},
OwnerReferences: []metav1.OwnerReference{*owner},
},
}
_, err := c.clusterClient.ClusterV1alpha1().PlacementDecisions(placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{})
return err
}

View File

@@ -1,140 +0,0 @@
package placementdecision
import (
"context"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/rand"
clienttesting "k8s.io/client-go/testing"
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
)
func assertValidPlacementDecision(t *testing.T, placementNamespace, placementName string, placementDecision *clusterapiv1alpha1.PlacementDecision) {
// check namespace
if placementDecision.Namespace != placementNamespace {
t.Errorf("expected PlacementDecision is created under namespace %q, but got %q",
placementNamespace, placementDecision.Namespace)
}
// check label
if _, ok := placementDecision.Labels[placementLabel]; !ok {
t.Errorf("expected Placement label on Placement decision")
}
// check owner reference
owner := metav1.GetControllerOf(&placementDecision.ObjectMeta)
if owner == nil {
t.Errorf("expected PlacementDecision with ownerreference")
}
if owner.Kind != "Placement" {
t.Errorf("expected ownerreference with kind %q, but got %q", "Placement", owner.Kind)
}
if owner.Name != placementName {
t.Errorf("expected ownerreference with name %q, but got %q", placementName, owner.Name)
}
}
func TestPlacementDecisionCreatingControllerSync(t *testing.T) {
placementNamespace := "ns1"
placementName := "placement1"
queueKey := placementNamespace + "/" + placementName
placementUID := rand.String(16)
cases := []struct {
name string
queueKey string
initObjs []runtime.Object
validateActions func(t *testing.T, actions []clienttesting.Action)
}{
{
name: "placement not found",
queueKey: queueKey,
validateActions: func(t *testing.T, actions []clienttesting.Action) {
if len(actions) != 0 {
t.Errorf("expected no action but got: %v ", actions)
}
},
},
{
name: "placement is deleting",
queueKey: queueKey,
initObjs: []runtime.Object{
testinghelpers.NewPlacement(placementNamespace, placementName).WithDeletionTimestamp().Build(),
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
if len(actions) != 0 {
t.Errorf("expected no action but got: %v ", actions)
}
},
},
{
name: "new placement",
queueKey: queueKey,
initObjs: []runtime.Object{
testinghelpers.NewPlacement(placementNamespace, placementName).WithUID(placementUID).Build(),
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
// check if PlacementDecision has been created
testinghelpers.AssertActions(t, actions, "create")
actual := actions[0].(clienttesting.CreateActionImpl).Object
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
if !ok {
t.Errorf("expected PlacementDecision was created")
}
assertValidPlacementDecision(t, placementNamespace, placementName, placementDecision)
},
},
{
name: "placementdecision exits",
queueKey: queueKey,
initObjs: []runtime.Object{
testinghelpers.NewPlacement(placementNamespace, placementName).WithUID(placementUID).Build(),
testinghelpers.NewPlacementDecision(placementNamespace, "decison1").
WithPlacementLabel(placementName).WithController(placementUID).Build(),
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
if len(actions) != 0 {
t.Errorf("expected no action but got %d: %v", len(actions), actions)
}
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)
placementStore := clusterInformerFactory.Cluster().V1alpha1().Placements().Informer().GetStore()
placementDecisionStore := clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Informer().GetStore()
for _, obj := range c.initObjs {
switch obj.(type) {
case *clusterapiv1alpha1.Placement:
placementStore.Add(obj)
case *clusterapiv1alpha1.PlacementDecision:
placementDecisionStore.Add(obj)
}
}
ctrl := placementDecisionCreatingController{
clusterClient: clusterClient,
placementLister: clusterInformerFactory.Cluster().V1alpha1().Placements().Lister(),
placementDecisionLister: clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(),
}
syncErr := ctrl.sync(context.TODO(), testinghelpers.NewFakeSyncContext(t, c.queueKey))
if syncErr != nil {
t.Errorf("unexpected err: %v", syncErr)
}
c.validateActions(t, clusterClient.Actions())
})
}
}

View File

@@ -0,0 +1,94 @@
package scheduling
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
)
type predicateSelector struct {
labelSelector labels.Selector
claimSelector labels.Selector
}
// matchWithClusterPredicates returns a slice of clusters. Each of them must match at least one of the predicates.
func matchWithClusterPredicates(predicates []clusterapiv1alpha1.ClusterPredicate, clusters []*clusterapiv1.ManagedCluster) ([]*clusterapiv1.ManagedCluster, error) {
if len(predicates) == 0 {
return clusters, nil
}
if len(clusters) == 0 {
return clusters, nil
}
// prebuild label/claim selectors for each predicate
predicateSelectors := []predicateSelector{}
for _, predicate := range predicates {
// build label selector
labelSelector, err := convertLabelSelector(predicate.RequiredClusterSelector.LabelSelector)
if err != nil {
return nil, err
}
// build claim selector
claimSelector, err := convertClaimSelector(predicate.RequiredClusterSelector.ClaimSelector)
if err != nil {
return nil, err
}
predicateSelectors = append(predicateSelectors, predicateSelector{
labelSelector: labelSelector,
claimSelector: claimSelector,
})
}
// match cluster with selectors one by one
matched := []*clusterapiv1.ManagedCluster{}
for _, cluster := range clusters {
claims := getClusterClaims(cluster)
for _, ps := range predicateSelectors {
// match with label selector
if ok := ps.labelSelector.Matches(labels.Set(cluster.Labels)); !ok {
continue
}
// match with claim selector
if ok := ps.claimSelector.Matches(labels.Set(claims)); !ok {
continue
}
matched = append(matched, cluster)
break
}
}
return matched, nil
}
// getClusterClaims returns a map containing cluster claims from the status of cluster
func getClusterClaims(cluster *clusterapiv1.ManagedCluster) map[string]string {
claims := map[string]string{}
for _, claim := range cluster.Status.ClusterClaims {
claims[claim.Name] = claim.Value
}
return claims
}
// convertLabelSelector converts metav1.LabelSelector to labels.Selector
func convertLabelSelector(labelSelector metav1.LabelSelector) (labels.Selector, error) {
selector, err := metav1.LabelSelectorAsSelector(&labelSelector)
if err != nil {
return labels.Nothing(), err
}
return selector, nil
}
// convertClaimSelector converts ClusterClaimSelector to labels.Selector
func convertClaimSelector(clusterClaimSelector clusterapiv1alpha1.ClusterClaimSelector) (labels.Selector, error) {
selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
MatchExpressions: clusterClaimSelector.MatchExpressions,
})
if err != nil {
return labels.Nothing(), err
}
return selector, nil
}

View File

@@ -0,0 +1,131 @@
package scheduling
import (
"strings"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
)
func TestMatchWithClusterPredicates(t *testing.T) {
cases := []struct {
name string
predicates []clusterapiv1alpha1.ClusterPredicate
clusters []*clusterapiv1.ManagedCluster
expectedClusterNames []string
}{
{
name: "match with label",
predicates: []clusterapiv1alpha1.ClusterPredicate{
testinghelpers.NewClusterPredicate(&metav1.LabelSelector{
MatchLabels: map[string]string{
"cloud": "Amazon",
},
}, nil),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(),
testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Google").Build(),
},
expectedClusterNames: []string{"cluster1"},
},
{
name: "match with claim",
predicates: []clusterapiv1alpha1.ClusterPredicate{
testinghelpers.NewClusterPredicate(nil,
&clusterapiv1alpha1.ClusterClaimSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "cloud",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"Amazon"},
},
},
}),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithClaim("cloud", "Amazon").Build(),
testinghelpers.NewManagedCluster("cluster2").WithClaim("cloud", "Google").Build(),
},
expectedClusterNames: []string{"cluster1"},
},
{
name: "match with both label and claim",
predicates: []clusterapiv1alpha1.ClusterPredicate{
testinghelpers.NewClusterPredicate(&metav1.LabelSelector{
MatchLabels: map[string]string{
"cloud": "Amazon",
},
}, &clusterapiv1alpha1.ClusterClaimSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "region",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"us-east-1"},
},
},
}),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").
WithLabel("cloud", "Amazon").
WithClaim("region", "us-east-1").Build(),
testinghelpers.NewManagedCluster("cluster2").
WithLabel("cloud", "Amazon").
WithClaim("region", "us-east-2").Build(),
},
expectedClusterNames: []string{"cluster1"},
},
{
name: "match with multiple predicates",
predicates: []clusterapiv1alpha1.ClusterPredicate{
testinghelpers.NewClusterPredicate(&metav1.LabelSelector{
MatchLabels: map[string]string{
"cloud": "Amazon",
},
}, nil),
testinghelpers.NewClusterPredicate(nil, &clusterapiv1alpha1.ClusterClaimSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "region",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"us-east-1"},
},
},
}),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(),
testinghelpers.NewManagedCluster("cluster2").WithClaim("region", "us-east-1").Build(),
testinghelpers.NewManagedCluster("cluster3").WithClaim("region", "us-east-2").Build(),
},
expectedClusterNames: []string{"cluster1", "cluster2"},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
clusters, err := matchWithClusterPredicates(c.predicates, c.clusters)
if err != nil {
t.Errorf("unexpected err: %v", err)
}
expectedClusterNames := sets.NewString(c.expectedClusterNames...)
if len(clusters) != expectedClusterNames.Len() {
t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters))
}
for _, cluster := range clusters {
expectedClusterNames.Delete(cluster.Name)
}
if expectedClusterNames.Len() > 0 {
t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ","))
}
})
}
}

View File

@@ -0,0 +1,148 @@
package scheduling
import (
"context"
"fmt"
"reflect"
"sort"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
)
type scheduleFunc func(
ctx context.Context,
placement *clusterapiv1alpha1.Placement,
clusters []*clusterapiv1.ManagedCluster,
clusterClient clusterclient.Interface,
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
) (*scheduleResult, error)
type scheduleResult struct {
scheduled int
unscheduled int
}
func schedule(
ctx context.Context,
placement *clusterapiv1alpha1.Placement,
clusters []*clusterapiv1.ManagedCluster,
clusterClient clusterclient.Interface,
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
) (*scheduleResult, error) {
// filter clusters with cluster predicates
feasibleClusters, err := matchWithClusterPredicates(placement.Spec.Predicates, clusters)
if err != nil {
return nil, err
}
// select clusters and generate cluster decisions
decisions := selectClusters(placement, feasibleClusters)
scheduled, unscheduled := len(decisions), 0
if placement.Spec.NumberOfClusters != nil {
unscheduled = int(*placement.Spec.NumberOfClusters) - scheduled
}
// bind the cluster decisions into placementdecisions
err = bind(ctx, placement, decisions, clusterClient, placementDecisionLister)
if err != nil {
return nil, err
}
return &scheduleResult{
scheduled: scheduled,
unscheduled: unscheduled,
}, nil
}
// makeClusterDecisions selects clusters based on given cluster slice and then creates
// cluster decisions.
func selectClusters(placement *clusterapiv1alpha1.Placement, clusters []*clusterapiv1.ManagedCluster) []clusterapiv1alpha1.ClusterDecision {
numOfDecisions := len(clusters)
if placement.Spec.NumberOfClusters != nil {
numOfDecisions = int(*placement.Spec.NumberOfClusters)
}
// truncate the cluster slice if the desired number of decisions is less than
// the number of the candidate clusters
if numOfDecisions < len(clusters) {
clusters = clusters[:numOfDecisions]
}
decisions := []clusterapiv1alpha1.ClusterDecision{}
for _, cluster := range clusters {
decisions = append(decisions, clusterapiv1alpha1.ClusterDecision{
ClusterName: cluster.Name,
})
}
return decisions
}
// bind updates the cluster decisions in the status of the placementdecisions with the given
// cluster decision slice. New placementdecision will be created if no one exists.
func bind(
ctx context.Context,
placement *clusterapiv1alpha1.Placement,
clusterDecisions []clusterapiv1alpha1.ClusterDecision,
clusterClient clusterclient.Interface,
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
) error {
// query placementdecisions with label selector
requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name})
if err != nil {
return err
}
labelSelector := labels.NewSelector().Add(*requirement)
placementDecisions, err := placementDecisionLister.PlacementDecisions(placement.Namespace).List(labelSelector)
if err != nil {
return err
}
// TODO: support multiple placementdecisions for a placement
var placementDecision *clusterapiv1alpha1.PlacementDecision
switch {
case len(placementDecisions) > 0:
placementDecision = placementDecisions[0]
default:
// create a placementdecision if not exists
owner := metav1.NewControllerRef(placement, clusterapiv1alpha1.GroupVersion.WithKind("Placement"))
placementDecision = &clusterapiv1alpha1.PlacementDecision{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-", placement.Name),
Namespace: placement.Namespace,
Labels: map[string]string{
placementLabel: placement.Name,
},
OwnerReferences: []metav1.OwnerReference{*owner},
},
}
placementDecision, err = clusterClient.ClusterV1alpha1().PlacementDecisions(placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{})
if err != nil {
return err
}
}
// sort by cluster name
sort.SliceStable(clusterDecisions, func(i, j int) bool {
return clusterDecisions[i].ClusterName < clusterDecisions[j].ClusterName
})
// update the status of the placementdecision if necessary
if reflect.DeepEqual(placementDecision.Status.Decisions, clusterDecisions) {
return nil
}
newPlacementDecision := placementDecision.DeepCopy()
newPlacementDecision.Status.Decisions = clusterDecisions
_, err = clusterClient.ClusterV1alpha1().PlacementDecisions(newPlacementDecision.Namespace).
UpdateStatus(ctx, newPlacementDecision, metav1.UpdateOptions{})
if err != nil {
return err
}
return nil
}

View File

@@ -0,0 +1,185 @@
package scheduling
import (
"context"
"strings"
"testing"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
clienttesting "k8s.io/client-go/testing"
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
)
func TestSchedule(t *testing.T) {
clusterSetName := "clusterSets"
placementNamespace := "ns1"
placementName := "placement1"
placementDecisionName := "placement1-decision1"
cases := []struct {
name string
placement *clusterapiv1alpha1.Placement
initObjs []runtime.Object
clusters []*clusterapiv1.ManagedCluster
scheduleResult scheduleResult
validateActions func(t *testing.T, actions []clienttesting.Action)
}{
{
name: "new placement satisfied",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
},
scheduleResult: scheduleResult{
scheduled: 1,
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
// check if PlacementDecision has been created
testinghelpers.AssertActions(t, actions, "create", "update")
actual := actions[1].(clienttesting.UpdateActionImpl).Object
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
if !ok {
t.Errorf("expected PlacementDecision was updated")
}
assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1")
},
},
{
name: "new placement unsatisfied",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(3).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
},
scheduleResult: scheduleResult{
scheduled: 1,
unscheduled: 2,
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
// check if PlacementDecision has been updated
testinghelpers.AssertActions(t, actions, "create", "update")
actual := actions[1].(clienttesting.UpdateActionImpl).Object
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
if !ok {
t.Errorf("expected PlacementDecision was updated")
}
assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1")
},
},
{
name: "placement with all decisions scheduled",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(2).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName).
WithLabel(placementLabel, placementName).
WithDecisions("cluster1", "cluster2").Build(),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).Build(),
},
scheduleResult: scheduleResult{
scheduled: 2,
},
validateActions: testinghelpers.AssertNoActions,
},
{
name: "placement with part of decisions scheduled",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName).
WithLabel(placementLabel, placementName).WithDecisions("cluster1").Build(),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).Build(),
},
scheduleResult: scheduleResult{
scheduled: 2,
unscheduled: 2,
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
// check if PlacementDecision has been updated
testinghelpers.AssertActions(t, actions, "update")
actual := actions[0].(clienttesting.UpdateActionImpl).Object
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
if !ok {
t.Errorf("expected PlacementDecision was updated")
}
assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1", "cluster2")
},
},
{
name: "placement without more feasible cluster available",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName).
WithLabel(placementLabel, placementName).WithDecisions("cluster1").Build(),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
},
scheduleResult: scheduleResult{
scheduled: 1,
unscheduled: 3,
},
validateActions: testinghelpers.AssertNoActions,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
c.initObjs = append(c.initObjs, c.placement)
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...)
result, err := schedule(
context.TODO(),
c.placement,
c.clusters,
clusterClient,
clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(),
)
if err != nil {
t.Errorf("unexpected err: %v", err)
}
if result.scheduled != c.scheduleResult.scheduled {
t.Errorf("expected %d scheduled, but got %d", c.scheduleResult.scheduled, result.scheduled)
}
if result.unscheduled != c.scheduleResult.unscheduled {
t.Errorf("expected %d unscheduled, but got %d", c.scheduleResult.unscheduled, result.unscheduled)
}
c.validateActions(t, clusterClient.Actions())
})
}
}
func assertClustersSelected(t *testing.T, decisons []clusterapiv1alpha1.ClusterDecision, clusterNames ...string) {
names := sets.NewString(clusterNames...)
for _, decision := range decisons {
if names.Has(decision.ClusterName) {
names.Delete(decision.ClusterName)
}
}
if names.Len() != 0 {
t.Errorf("expected clusters selected: %s", strings.Join(names.UnsortedList(), ","))
}
}

View File

@@ -0,0 +1,226 @@
package scheduling
import (
"context"
"fmt"
"reflect"
"time"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/events"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
cache "k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1"
clusterlisterv1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1"
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
)
const (
clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
placementLabel = "cluster.open-cluster-management.io/placement"
)
var ResyncInterval = 2 * time.Minute
// schedulingController schedules cluster decisions for Placements
type schedulingController struct {
clusterClient clusterclient.Interface
clusterLister clusterlisterv1.ManagedClusterLister
clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister
clusterSetBindingLister clusterlisterv1alpha1.ManagedClusterSetBindingLister
placementLister clusterlisterv1alpha1.PlacementLister
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister
scheduleFunc scheduleFunc
}
// NewDecisionSchedulingController return an instance of schedulingController
func NewSchedulingController(
clusterClient clusterclient.Interface,
clusterLister clusterlisterv1.ManagedClusterLister,
clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister,
clusterSetBindingLister clusterlisterv1alpha1.ManagedClusterSetBindingLister,
placementInformer clusterinformerv1alpha1.PlacementInformer,
placementDecisionInformer clusterinformerv1alpha1.PlacementDecisionInformer,
recorder events.Recorder,
) factory.Controller {
// build controller
c := schedulingController{
clusterClient: clusterClient,
clusterLister: clusterLister,
clusterSetLister: clusterSetLister,
clusterSetBindingLister: clusterSetBindingLister,
placementLister: placementInformer.Lister(),
placementDecisionLister: placementDecisionInformer.Lister(),
scheduleFunc: schedule,
}
return factory.New().
WithInformersQueueKeyFunc(func(obj runtime.Object) string {
key, _ := cache.MetaNamespaceKeyFunc(obj)
return key
}, placementInformer.Informer()).
WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string {
accessor, _ := meta.Accessor(obj)
labels := accessor.GetLabels()
placementName := labels[placementLabel]
return fmt.Sprintf("%s/%s", accessor.GetNamespace(), placementName)
}, func(obj interface{}) bool {
accessor, err := meta.Accessor(obj)
if err != nil {
return false
}
labels := accessor.GetLabels()
if _, ok := labels[placementLabel]; ok {
return true
}
return false
}, placementDecisionInformer.Informer()).
// TODO: monitor more resources, like clusters, clustersets and clustersetbindings
WithSync(c.sync).
ResyncEvery(ResyncInterval).
ToController("SchedulingController", recorder)
}
func (c *schedulingController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
queueKey := syncCtx.QueueKey()
// handle resync
if queueKey == factory.DefaultQueueKey {
placements, err := c.placementLister.List(labels.Everything())
if err != nil {
return err
}
for _, placement := range placements {
syncCtx.Queue().Add(fmt.Sprintf("%s/%s", placement.Namespace, placement.Name))
}
return nil
}
// sync a placement
namespace, name, err := cache.SplitMetaNamespaceKey(queueKey)
if err != nil {
// ignore placement whose key is not in format: namespace/name
utilruntime.HandleError(err)
return nil
}
klog.V(4).Infof("Reconciling placement %q", queueKey)
placement, err := c.placementLister.Placements(namespace).Get(name)
if errors.IsNotFound(err) {
// no work if placement is deleted
return nil
}
if err != nil {
return err
}
// no work if placement is deleting
if !placement.DeletionTimestamp.IsZero() {
return nil
}
// get available clusters for this placement
clusters, err := c.getAvailableClusters(placement)
if err != nil {
return err
}
// schedule placement with scheduler
scheduleResult, err := c.scheduleFunc(ctx, placement, clusters, c.clusterClient, c.placementDecisionLister)
if err != nil {
return err
}
// update placement status if necessary
return c.updateStatus(ctx, placement, scheduleResult.scheduled, scheduleResult.unscheduled)
}
// getAvailableClusters returns available clusters for the given placement. The clusters must
// 1) Be from clustersets bound to the placement namespace;
// 2) Belong to one of particular clustersets if .spec.clusterSets is specified;
func (c *schedulingController) getAvailableClusters(placement *clusterapiv1alpha1.Placement) ([]*clusterapiv1.ManagedCluster, error) {
// get all clusterset bindings under the placement namespace
bindings, err := c.clusterSetBindingLister.ManagedClusterSetBindings(placement.Namespace).List(labels.Everything())
if err != nil {
return nil, err
}
if len(bindings) == 0 {
return nil, nil
}
// filter out invaid clustersetbindings
clusterSetNames := sets.NewString()
for _, binding := range bindings {
// ignore clusterset does not exist
_, err := c.clusterSetLister.Get(binding.Name)
if errors.IsNotFound(err) {
continue
}
if err != nil {
return nil, err
}
clusterSetNames.Insert(binding.Name)
}
// get intersection of clustesets bound to placement namespace and clustesets specified
// in placement spec
if len(placement.Spec.ClusterSets) != 0 {
clusterSetNames = clusterSetNames.Intersection(sets.NewString(placement.Spec.ClusterSets...))
}
if len(clusterSetNames) == 0 {
return nil, nil
}
// list clusters from particular clustersets
requirement, err := labels.NewRequirement(clusterSetLabel, selection.In, clusterSetNames.List())
if err != nil {
return nil, err
}
labelSelector := labels.NewSelector().Add(*requirement)
return c.clusterLister.List(labelSelector)
}
// updateStatus updates the status of the placement according to schedule result.
func (c *schedulingController) updateStatus(ctx context.Context, placement *clusterapiv1alpha1.Placement, numOfScheduledDecisions, numOfUnscheduledDecisions int) error {
newPlacement := placement.DeepCopy()
newPlacement.Status.NumberOfSelectedClusters = int32(numOfScheduledDecisions)
satisfiedCondition := newSatisfiedCondition(numOfUnscheduledDecisions)
meta.SetStatusCondition(&newPlacement.Status.Conditions, satisfiedCondition)
if reflect.DeepEqual(newPlacement.Status, placement.Status) {
return nil
}
_, err := c.clusterClient.ClusterV1alpha1().Placements(newPlacement.Namespace).UpdateStatus(ctx, newPlacement, metav1.UpdateOptions{})
return err
}
// newSatisfiedCondition returns a new condition with type PlacementConditionSatisfied
func newSatisfiedCondition(numOfUnscheduledDecisions int) metav1.Condition {
condition := metav1.Condition{
Type: clusterapiv1alpha1.PlacementConditionSatisfied,
}
switch {
case numOfUnscheduledDecisions == 0:
condition.Status = metav1.ConditionTrue
condition.Reason = "AllDecisionsScheduled"
condition.Message = "All cluster decisions scheduled"
default:
condition.Status = metav1.ConditionFalse
condition.Reason = "NotAllDecisionsScheduled"
condition.Message = fmt.Sprintf("%d cluster decisions unscheduled", numOfUnscheduledDecisions)
}
return condition
}

View File

@@ -0,0 +1,210 @@
package scheduling
import (
"context"
"strings"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
clienttesting "k8s.io/client-go/testing"
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
)
func TestSchedulingController_sync(t *testing.T) {
placementNamespace := "ns1"
placementName := "placement1"
cases := []struct {
name string
placement *clusterapiv1alpha1.Placement
initObjs []runtime.Object
scheduleResult *scheduleResult
validateActions func(t *testing.T, actions []clienttesting.Action)
}{
{
name: "placement satisfied",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
scheduleResult: &scheduleResult{
scheduled: 3,
unscheduled: 0,
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
// check if PlacementDecision has been updated
testinghelpers.AssertActions(t, actions, "update")
// check if Placement has been updated
actual := actions[0].(clienttesting.UpdateActionImpl).Object
placement, ok := actual.(*clusterapiv1alpha1.Placement)
if !ok {
t.Errorf("expected Placement was updated")
}
if placement.Status.NumberOfSelectedClusters != int32(3) {
t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters)
}
testinghelpers.HasCondition(
placement.Status.Conditions,
clusterapiv1alpha1.PlacementConditionSatisfied,
"AllDecisionsScheduled",
metav1.ConditionTrue,
)
},
},
{
name: "placement unsatisfied",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
scheduleResult: &scheduleResult{
scheduled: 3,
unscheduled: 1,
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
// check if PlacementDecision has been updated
testinghelpers.AssertActions(t, actions, "update")
// check if Placement has been updated
actual := actions[0].(clienttesting.UpdateActionImpl).Object
placement, ok := actual.(*clusterapiv1alpha1.Placement)
if !ok {
t.Errorf("expected Placement was updated")
}
if placement.Status.NumberOfSelectedClusters != int32(3) {
t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters)
}
testinghelpers.HasCondition(
placement.Status.Conditions,
clusterapiv1alpha1.PlacementConditionSatisfied,
"NotAllDecisionsScheduled",
metav1.ConditionFalse,
)
},
},
{
name: "placement status not changed",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).
WithNumOfSelectedClusters(3).WithSatisfiedCondition(3, 0).Build(),
scheduleResult: &scheduleResult{
scheduled: 3,
unscheduled: 0,
},
validateActions: testinghelpers.AssertNoActions,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
c.initObjs = append(c.initObjs, c.placement)
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...)
ctrl := schedulingController{
clusterClient: clusterClient,
clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(),
clusterSetLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Lister(),
clusterSetBindingLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(),
placementLister: clusterInformerFactory.Cluster().V1alpha1().Placements().Lister(),
placementDecisionLister: clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(),
scheduleFunc: func(
ctx context.Context,
placement *clusterapiv1alpha1.Placement,
clusters []*clusterapiv1.ManagedCluster,
clusterClient clusterclient.Interface,
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
) (*scheduleResult, error) {
return c.scheduleResult, nil
},
}
sysCtx := testinghelpers.NewFakeSyncContext(t, c.placement.Namespace+"/"+c.placement.Name)
syncErr := ctrl.sync(context.TODO(), sysCtx)
if syncErr != nil {
t.Errorf("unexpected err: %v", syncErr)
}
c.validateActions(t, clusterClient.Actions())
})
}
}
func TestGetAvailableClusters(t *testing.T) {
placementNamespace := "ns1"
placementName := "placement1"
cases := []struct {
name string
placement *clusterapiv1alpha1.Placement
initObjs []runtime.Object
expectedClusterNames []string
}{
{
name: "no bound clusterset",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet("clusterset1"),
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"),
},
},
{
name: "select all clusters from bound clustersets",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet("clusterset1"),
testinghelpers.NewClusterSet("clusterset2"),
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"),
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset2"),
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, "clusterset2").Build(),
},
expectedClusterNames: []string{"cluster1", "cluster2"},
},
{
name: "select clusters from a bound clusterset",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).
WithClusterSets([]string{"clusterset1"}).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet("clusterset1"),
testinghelpers.NewClusterSet("clusterset2"),
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"),
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset2"),
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, "clusterset2").Build(),
},
expectedClusterNames: []string{"cluster1"},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
c.initObjs = append(c.initObjs, c.placement)
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...)
ctrl := &schedulingController{
clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(),
clusterSetLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Lister(),
clusterSetBindingLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(),
}
clusters, err := ctrl.getAvailableClusters(c.placement)
if err != nil {
t.Errorf("unexpected err: %v", err)
}
expectedClusterNames := sets.NewString(c.expectedClusterNames...)
if len(clusters) != expectedClusterNames.Len() {
t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters))
}
for _, cluster := range clusters {
expectedClusterNames.Delete(cluster.Name)
}
if expectedClusterNames.Len() > 0 {
t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ","))
}
})
}
}

View File

@@ -1,16 +1,16 @@
package testing
import (
"fmt"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
)
const (
placementLabel = "cluster.open-cluster-management.io/placement"
)
type placementBuilder struct {
placement *clusterapiv1alpha1.Placement
}
@@ -31,16 +31,73 @@ func (b *placementBuilder) WithUID(uid string) *placementBuilder {
return b
}
func (b *placementBuilder) WithNOC(noc int32) *placementBuilder {
b.placement.Spec.NumberOfClusters = &noc
return b
}
func (b *placementBuilder) WithClusterSets(clusterSets []string) *placementBuilder {
b.placement.Spec.ClusterSets = clusterSets
return b
}
func (b *placementBuilder) WithDeletionTimestamp() *placementBuilder {
now := metav1.Now()
b.placement.DeletionTimestamp = &now
return b
}
func (b *placementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1alpha1.ClusterClaimSelector) *placementBuilder {
if b.placement.Spec.Predicates == nil {
b.placement.Spec.Predicates = []clusterapiv1alpha1.ClusterPredicate{}
}
b.placement.Spec.Predicates = append(b.placement.Spec.Predicates, NewClusterPredicate(labelSelector, claimSelector))
return b
}
func (b *placementBuilder) WithNumOfSelectedClusters(nosc int) *placementBuilder {
b.placement.Status.NumberOfSelectedClusters = int32(nosc)
return b
}
func (b *placementBuilder) WithSatisfiedCondition(numbOfScheduledDecisions, numbOfUnscheduledDecisions int) *placementBuilder {
condition := metav1.Condition{
Type: clusterapiv1alpha1.PlacementConditionSatisfied,
}
switch {
case numbOfUnscheduledDecisions == 0:
condition.Status = metav1.ConditionTrue
condition.Reason = "AllDecisionsScheduled"
condition.Message = "All cluster decisions scheduled"
default:
condition.Status = metav1.ConditionFalse
condition.Reason = "NotAllDecisionsScheduled"
condition.Message = fmt.Sprintf("%d cluster decisions unscheduled", numbOfUnscheduledDecisions)
}
meta.SetStatusCondition(&b.placement.Status.Conditions, condition)
return b
}
func (b *placementBuilder) Build() *clusterapiv1alpha1.Placement {
return b.placement
}
func NewClusterPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1alpha1.ClusterClaimSelector) clusterapiv1alpha1.ClusterPredicate {
predicate := clusterapiv1alpha1.ClusterPredicate{
RequiredClusterSelector: clusterapiv1alpha1.ClusterSelector{},
}
if labelSelector != nil {
predicate.RequiredClusterSelector.LabelSelector = *labelSelector
}
if claimSelector != nil {
predicate.RequiredClusterSelector.ClaimSelector = *claimSelector
}
return predicate
}
type placementDecisionBuilder struct {
placementDecision *clusterapiv1alpha1.PlacementDecision
}
@@ -65,14 +122,96 @@ func (b *placementDecisionBuilder) WithController(uid string) *placementDecision
return b
}
func (b *placementDecisionBuilder) WithPlacementLabel(placementName string) *placementDecisionBuilder {
func (b *placementDecisionBuilder) WithLabel(name, value string) *placementDecisionBuilder {
if b.placementDecision.Labels == nil {
b.placementDecision.Labels = map[string]string{}
}
b.placementDecision.Labels[placementLabel] = placementName
b.placementDecision.Labels[name] = value
return b
}
func (b *placementDecisionBuilder) WithDeletionTimestamp() *placementDecisionBuilder {
now := metav1.Now()
b.placementDecision.DeletionTimestamp = &now
return b
}
func (b *placementDecisionBuilder) WithDecisions(clusterNames ...string) *placementDecisionBuilder {
decisions := []clusterapiv1alpha1.ClusterDecision{}
for _, clusterName := range clusterNames {
decisions = append(decisions, clusterapiv1alpha1.ClusterDecision{
ClusterName: clusterName,
})
}
b.placementDecision.Status.Decisions = decisions
return b
}
func (b *placementDecisionBuilder) Build() *clusterapiv1alpha1.PlacementDecision {
return b.placementDecision
}
type managedClusterBuilder struct {
cluster *clusterapiv1.ManagedCluster
}
func NewManagedCluster(clusterName string) *managedClusterBuilder {
return &managedClusterBuilder{
cluster: &clusterapiv1.ManagedCluster{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
},
},
}
}
func (b *managedClusterBuilder) WithLabel(name, value string) *managedClusterBuilder {
if b.cluster.Labels == nil {
b.cluster.Labels = map[string]string{}
}
b.cluster.Labels[name] = value
return b
}
func (b *managedClusterBuilder) WithClaim(name, value string) *managedClusterBuilder {
claimMap := map[string]string{}
for _, claim := range b.cluster.Status.ClusterClaims {
claimMap[claim.Name] = claim.Value
}
claimMap[name] = value
clusterClaims := []clusterapiv1.ManagedClusterClaim{}
for k, v := range claimMap {
clusterClaims = append(clusterClaims, clusterapiv1.ManagedClusterClaim{
Name: k,
Value: v,
})
}
b.cluster.Status.ClusterClaims = clusterClaims
return b
}
func (b *managedClusterBuilder) Build() *clusterapiv1.ManagedCluster {
return b.cluster
}
func NewClusterSet(clusterSetName string) *clusterapiv1alpha1.ManagedClusterSet {
return &clusterapiv1alpha1.ManagedClusterSet{
ObjectMeta: metav1.ObjectMeta{
Name: clusterSetName,
},
}
}
func NewClusterSetBinding(namespace, clusterSetName string) *clusterapiv1alpha1.ManagedClusterSetBinding {
return &clusterapiv1alpha1.ManagedClusterSetBinding{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: clusterSetName,
},
Spec: clusterapiv1alpha1.ManagedClusterSetBindingSpec{
ClusterSet: clusterSetName,
},
}
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/events/eventstesting"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clienttesting "k8s.io/client-go/testing"
"k8s.io/client-go/util/workqueue"
)
@@ -43,3 +44,25 @@ func AssertActions(t *testing.T, actualActions []clienttesting.Action, expectedV
func AssertNoActions(t *testing.T, actualActions []clienttesting.Action) {
AssertActions(t, actualActions)
}
func HasCondition(conditions []metav1.Condition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool {
found := false
for _, condition := range conditions {
if condition.Type != expectedType {
continue
}
found = true
if condition.Status != expectedStatus {
return false
}
if condition.Reason != expectedReason {
return false
}
return true
}
return found
}

View File

@@ -0,0 +1,37 @@
package testing
import (
"time"
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
"k8s.io/apimachinery/pkg/runtime"
)
func NewClusterInformerFactory(clusterClient clusterclient.Interface, objects ...runtime.Object) clusterinformers.SharedInformerFactory {
clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)
clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore()
clusterSetStore := clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Informer().GetStore()
clusterSetBindingStore := clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Informer().GetStore()
placementStore := clusterInformerFactory.Cluster().V1alpha1().Placements().Informer().GetStore()
placementDecisionStore := clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Informer().GetStore()
for _, obj := range objects {
switch obj.(type) {
case *clusterapiv1.ManagedCluster:
clusterStore.Add(obj)
case *clusterapiv1alpha1.ManagedClusterSet:
clusterSetStore.Add(obj)
case *clusterapiv1alpha1.ManagedClusterSetBinding:
clusterSetBindingStore.Add(obj)
case *clusterapiv1alpha1.Placement:
placementStore.Add(obj)
case *clusterapiv1alpha1.PlacementDecision:
placementDecisionStore.Add(obj)
}
}
return clusterInformerFactory
}

View File

@@ -3,6 +3,7 @@ package integration
import (
"context"
"fmt"
"time"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
@@ -12,25 +13,32 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
controllers "github.com/open-cluster-management/placement/pkg/controllers"
"github.com/open-cluster-management/placement/pkg/controllers/scheduling"
"github.com/open-cluster-management/placement/test/integration/util"
)
const (
placementLabel = "cluster.open-cluster-management.io/placement"
clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
placementLabel = "cluster.open-cluster-management.io/placement"
)
var _ = ginkgo.Describe("Placement Scheduling", func() {
var _ = ginkgo.Describe("Placement", func() {
var cancel context.CancelFunc
var namespace string
var placementName string
var clusterSet1Name, clusterSet2Name string
var suffix string
var err error
ginkgo.BeforeEach(func() {
suffix := rand.String(5)
suffix = rand.String(5)
namespace = fmt.Sprintf("ns-%s", suffix)
placementName = fmt.Sprintf("placement-%s", suffix)
clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix)
clusterSet2Name = fmt.Sprintf("clusterset-%s", rand.String(5))
// create testing namespace
ns := &corev1.Namespace{
@@ -42,6 +50,7 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
gomega.Expect(err).ToNot(gomega.HaveOccurred())
// start controller manager
scheduling.ResyncInterval = 10 * time.Second
var ctx context.Context
ctx, cancel = context.WithCancel(context.Background())
go controllers.RunControllerManager(ctx, &controllercmd.ControllerContext{
@@ -58,22 +67,11 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
gomega.Expect(err).ToNot(gomega.HaveOccurred())
})
ginkgo.It("Should create placement successfully", func() {
ginkgo.By("Create placement")
placement := &clusterapiv1alpha1.Placement{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: placementName,
},
}
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
assertPlacementDecisionCreated := func(placement *clusterapiv1alpha1.Placement) {
ginkgo.By("Check if placementdecision is created")
gomega.Eventually(func() bool {
pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: placementLabel + "=" + placementName,
LabelSelector: placementLabel + "=" + placement.Name,
})
if err != nil {
return false
@@ -88,11 +86,9 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
}
return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
ginkgo.By("Delete placement")
err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.Background(), placementName, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
assertPlacementDeleted := func(placementName string) {
ginkgo.By("Check if placement is gone")
gomega.Eventually(func() bool {
_, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
@@ -101,5 +97,189 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
}
return errors.IsNotFound(err)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
assertNumberOfDecisions := func(placementName string, desiredNOD int) {
ginkgo.By("Check the number of decisions in placementdecisions")
gomega.Eventually(func() bool {
pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: placementLabel + "=" + placementName,
})
if err != nil {
return false
}
if len(pdl.Items) == 0 {
return false
}
actualNOD := 0
for _, pd := range pdl.Items {
actualNOD += len(pd.Status.Decisions)
}
return actualNOD == desiredNOD
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) {
ginkgo.By("Check the status of placement")
gomega.Eventually(func() bool {
placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
if err != nil {
return false
}
if satisfied && !util.HasCondition(
placement.Status.Conditions,
clusterapiv1alpha1.PlacementConditionSatisfied,
"AllDecisionsScheduled",
metav1.ConditionTrue,
) {
return false
}
if !satisfied && !util.HasCondition(
placement.Status.Conditions,
clusterapiv1alpha1.PlacementConditionSatisfied,
"NotAllDecisionsScheduled",
metav1.ConditionFalse,
) {
return false
}
return placement.Status.NumberOfSelectedClusters == int32(numOfSelectedClusters)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
assertBindingClusterSet := func(clusterSetName string) {
ginkgo.By("Create clusterset/clustersetbinding")
clusterset := &clusterapiv1alpha1.ManagedClusterSet{
ObjectMeta: metav1.ObjectMeta{
Name: clusterSetName,
},
}
_, err = clusterClient.ClusterV1alpha1().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
csb := &clusterapiv1alpha1.ManagedClusterSetBinding{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: clusterSetName,
},
Spec: clusterapiv1alpha1.ManagedClusterSetBindingSpec{
ClusterSet: clusterSetName,
},
}
_, err = clusterClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
assertCreatingClusters := func(clusterSetName string, num int) {
ginkgo.By(fmt.Sprintf("Create %d clusters", num))
for i := 0; i < num; i++ {
cluster := &clusterapiv1.ManagedCluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Labels: map[string]string{
clusterSetLabel: clusterSetName,
},
},
}
_, err = clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
}
assertCreatingPlacement := func(name string, noc *int32, nod int) {
ginkgo.By("Create placement")
placement := &clusterapiv1alpha1.Placement{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
},
Spec: clusterapiv1alpha1.PlacementSpec{
NumberOfClusters: noc,
},
}
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
assertPlacementDecisionCreated(placement)
assertNumberOfDecisions(placementName, nod)
if noc != nil {
assertPlacementStatus(placementName, nod, nod == int(*noc))
}
}
ginkgo.Context("Scheduling", func() {
ginkgo.AfterEach(func() {
ginkgo.By("Delete placement")
err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.Background(), placementName, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
assertPlacementDeleted(placementName)
})
ginkgo.It("Should schedule successfully once spec.NumberOfClusters is reduced", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5)
ginkgo.By("Reduce NOC of the placement")
placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
noc := int32(4)
placement.Spec.NumberOfClusters = &noc
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
nod := int(noc)
assertNumberOfDecisions(placementName, nod)
assertPlacementStatus(placementName, nod, true)
})
ginkgo.It("Should schedule successfully once spec.NumberOfClusters is increased", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 10)
assertCreatingPlacement(placementName, noc(5), 5)
ginkgo.By("Increase NOC of the placement")
placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
noc := int32(8)
placement.Spec.NumberOfClusters = &noc
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
nod := int(noc)
assertNumberOfDecisions(placementName, nod)
assertPlacementStatus(placementName, nod, true)
})
ginkgo.It("Should be satisfied once new clusters are added", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5)
// add more clusters
assertCreatingClusters(clusterSet1Name, 5)
nod := 10
assertNumberOfDecisions(placementName, nod)
assertPlacementStatus(placementName, nod, true)
})
ginkgo.It("Should schedule successfully once new clusterset is bound", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5)
ginkgo.By("Bind one more clusterset to the placement namespace")
assertBindingClusterSet(clusterSet2Name)
assertCreatingClusters(clusterSet2Name, 3)
nod := 8
assertNumberOfDecisions(placementName, nod)
assertPlacementStatus(placementName, nod, false)
})
})
})
func noc(n int) *int32 {
noc := int32(n)
return &noc
}

View File

@@ -4,8 +4,8 @@ import (
"fmt"
"github.com/onsi/ginkgo"
"github.com/openshift/library-go/pkg/operator/events"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func NewIntegrationTestEventRecorder(componet string) events.Recorder {
@@ -47,3 +47,25 @@ func (r *IntegrationTestEventRecorder) Warningf(reason, messageFmt string, args
func (r *IntegrationTestEventRecorder) Shutdown() {
return
}
func HasCondition(conditions []metav1.Condition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool {
found := false
for _, condition := range conditions {
if condition.Type != expectedType {
continue
}
found = true
if condition.Status != expectedStatus {
return false
}
if condition.Reason != expectedReason {
return false
}
return true
}
return found
}