mirror of
https://github.com/open-cluster-management-io/ocm.git
synced 2026-05-22 00:54:00 +00:00
@@ -8,8 +8,7 @@ import (
|
||||
|
||||
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
|
||||
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
|
||||
placement "github.com/open-cluster-management/placement/pkg/controllers/placement"
|
||||
placementdecision "github.com/open-cluster-management/placement/pkg/controllers/placementdecision"
|
||||
scheduling "github.com/open-cluster-management/placement/pkg/controllers/scheduling"
|
||||
)
|
||||
|
||||
// RunControllerManager starts the controllers on hub to make placement decisions.
|
||||
@@ -20,14 +19,11 @@ func RunControllerManager(ctx context.Context, controllerContext *controllercmd.
|
||||
}
|
||||
clusterInformers := clusterinformers.NewSharedInformerFactory(clusterClient, 10*time.Minute)
|
||||
|
||||
placementController := placement.NewPlacementController(
|
||||
clusterInformers.Cluster().V1().ManagedClusters(),
|
||||
clusterInformers.Cluster().V1alpha1().ManagedClusterSets(),
|
||||
controllerContext.EventRecorder,
|
||||
)
|
||||
|
||||
placementDecisionCreatingController := placementdecision.NewPlacementDecisionCreatingController(
|
||||
schedulingController := scheduling.NewSchedulingController(
|
||||
clusterClient,
|
||||
clusterInformers.Cluster().V1().ManagedClusters().Lister(),
|
||||
clusterInformers.Cluster().V1alpha1().ManagedClusterSets().Lister(),
|
||||
clusterInformers.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(),
|
||||
clusterInformers.Cluster().V1alpha1().Placements(),
|
||||
clusterInformers.Cluster().V1alpha1().PlacementDecisions(),
|
||||
controllerContext.EventRecorder,
|
||||
@@ -35,8 +31,7 @@ func RunControllerManager(ctx context.Context, controllerContext *controllercmd.
|
||||
|
||||
go clusterInformers.Start(ctx.Done())
|
||||
|
||||
go placementController.Run(ctx, 1)
|
||||
go placementDecisionCreatingController.Run(ctx, 1)
|
||||
go schedulingController.Run(ctx, 1)
|
||||
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/openshift/library-go/pkg/controller/factory"
|
||||
"github.com/openshift/library-go/pkg/operator/events"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
clusterinformerv1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1"
|
||||
clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1"
|
||||
clusterlisterv1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1"
|
||||
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
|
||||
)
|
||||
|
||||
// placementController makes placement decisions for Placements
|
||||
type placementController struct {
|
||||
clusterLister clusterlisterv1.ManagedClusterLister
|
||||
clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister
|
||||
}
|
||||
|
||||
// NewPlacementController return an instance of placementController
|
||||
func NewPlacementController(
|
||||
clusterInformer clusterinformerv1.ManagedClusterInformer,
|
||||
clusterSetInformer clusterinformerv1alpha1.ManagedClusterSetInformer,
|
||||
recorder events.Recorder,
|
||||
) factory.Controller {
|
||||
c := placementController{
|
||||
clusterLister: clusterInformer.Lister(),
|
||||
clusterSetLister: clusterSetInformer.Lister(),
|
||||
}
|
||||
|
||||
return factory.New().
|
||||
WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string {
|
||||
accessor, _ := meta.Accessor(obj)
|
||||
return fmt.Sprintf("cluster:%s", accessor.GetName())
|
||||
}, func(obj interface{}) bool {
|
||||
accessor, err := meta.Accessor(obj)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// ignore cluster belongs to no clusterset
|
||||
labels := accessor.GetLabels()
|
||||
clusterSetName, ok := labels[clusterSetLabel]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
// ignore cluster if its clusterset does not exist
|
||||
_, err = c.clusterSetLister.Get(clusterSetName)
|
||||
return err == nil
|
||||
}, clusterInformer.Informer()).
|
||||
WithSync(c.sync).
|
||||
ToController("PlacementController", recorder)
|
||||
}
|
||||
|
||||
func (c *placementController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
clienttesting "k8s.io/client-go/testing"
|
||||
|
||||
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
|
||||
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
|
||||
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
|
||||
)
|
||||
|
||||
func TestSync(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
queueKey string
|
||||
initObjs []runtime.Object
|
||||
validateActions func(t *testing.T, hubActions, agentActions []clienttesting.Action)
|
||||
}{}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
|
||||
clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)
|
||||
clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore()
|
||||
for _, cluster := range c.initObjs {
|
||||
clusterStore.Add(cluster)
|
||||
}
|
||||
|
||||
ctrl := placementController{
|
||||
clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(),
|
||||
}
|
||||
syncErr := ctrl.sync(context.TODO(), testinghelpers.NewFakeSyncContext(t, c.queueKey))
|
||||
if syncErr != nil {
|
||||
t.Errorf("unexpected err: %v", syncErr)
|
||||
}
|
||||
|
||||
//c.validateActions(t, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package placementdecision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/openshift/library-go/pkg/controller/factory"
|
||||
"github.com/openshift/library-go/pkg/operator/events"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
cache "k8s.io/client-go/tools/cache"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
|
||||
clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1"
|
||||
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
placementLabel = "cluster.open-cluster-management.io/placement"
|
||||
)
|
||||
|
||||
// placementDecisionCreatingController creates PlacementDecisions for Placements
|
||||
type placementDecisionCreatingController struct {
|
||||
clusterClient clusterclient.Interface
|
||||
placementLister clusterlisterv1alpha1.PlacementLister
|
||||
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister
|
||||
}
|
||||
|
||||
// NewPlacementDecisionCreatingController return an instance of placementDecisionCreatingController
|
||||
func NewPlacementDecisionCreatingController(
|
||||
clusterClient clusterclient.Interface,
|
||||
placementInformer clusterinformerv1alpha1.PlacementInformer,
|
||||
placementDecisionInformer clusterinformerv1alpha1.PlacementDecisionInformer,
|
||||
recorder events.Recorder,
|
||||
) factory.Controller {
|
||||
c := placementDecisionCreatingController{
|
||||
clusterClient: clusterClient,
|
||||
placementLister: placementInformer.Lister(),
|
||||
placementDecisionLister: placementDecisionInformer.Lister(),
|
||||
}
|
||||
|
||||
return factory.New().
|
||||
WithInformersQueueKeyFunc(func(obj runtime.Object) string {
|
||||
key, _ := cache.MetaNamespaceKeyFunc(obj)
|
||||
return key
|
||||
}, placementInformer.Informer()).
|
||||
WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string {
|
||||
accessor, _ := meta.Accessor(obj)
|
||||
labels := accessor.GetLabels()
|
||||
placementName := labels[placementLabel]
|
||||
return fmt.Sprintf("%s/%s", accessor.GetNamespace(), placementName)
|
||||
}, func(obj interface{}) bool {
|
||||
accessor, err := meta.Accessor(obj)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
labels := accessor.GetLabels()
|
||||
if _, ok := labels[placementLabel]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}, placementDecisionInformer.Informer()).
|
||||
WithSync(c.sync).
|
||||
ToController("PlacementDecisionCreatingController", recorder)
|
||||
}
|
||||
|
||||
func (c *placementDecisionCreatingController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
|
||||
queueKey := syncCtx.QueueKey()
|
||||
namespace, name, err := cache.SplitMetaNamespaceKey(queueKey)
|
||||
if err != nil {
|
||||
// ignore placement whose key is not in format: namespace/name
|
||||
return nil
|
||||
}
|
||||
|
||||
klog.V(4).Infof("Reconciling placement %q", queueKey)
|
||||
placement, err := c.placementLister.Placements(namespace).Get(name)
|
||||
if errors.IsNotFound(err) {
|
||||
// no work if placement is deleted
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// no work if placement is deleting
|
||||
if !placement.DeletionTimestamp.IsZero() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// query placementdecisions with label selector
|
||||
requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
labelSelector := labels.NewSelector().Add(*requirement)
|
||||
placementDecisions, err := c.placementDecisionLister.PlacementDecisions(namespace).List(labelSelector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// no work if PlacementDecision has been created
|
||||
if len(placementDecisions) > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// otherwise create placementdecision
|
||||
return c.createPlacementDecision(ctx, placement)
|
||||
}
|
||||
|
||||
// createPlacementDecision creates PlacementDecision for the Placement
|
||||
func (c *placementDecisionCreatingController) createPlacementDecision(ctx context.Context, placement *clusterapiv1alpha1.Placement) error {
|
||||
owner := metav1.NewControllerRef(placement, clusterapiv1alpha1.GroupVersion.WithKind("Placement"))
|
||||
placementDecision := &clusterapiv1alpha1.PlacementDecision{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("%s-", placement.Name),
|
||||
Namespace: placement.Namespace,
|
||||
Labels: map[string]string{
|
||||
placementLabel: placement.Name,
|
||||
},
|
||||
OwnerReferences: []metav1.OwnerReference{*owner},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := c.clusterClient.ClusterV1alpha1().PlacementDecisions(placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{})
|
||||
return err
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package placementdecision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/rand"
|
||||
clienttesting "k8s.io/client-go/testing"
|
||||
|
||||
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
|
||||
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
|
||||
)
|
||||
|
||||
func assertValidPlacementDecision(t *testing.T, placementNamespace, placementName string, placementDecision *clusterapiv1alpha1.PlacementDecision) {
|
||||
// check namespace
|
||||
if placementDecision.Namespace != placementNamespace {
|
||||
t.Errorf("expected PlacementDecision is created under namespace %q, but got %q",
|
||||
placementNamespace, placementDecision.Namespace)
|
||||
}
|
||||
|
||||
// check label
|
||||
if _, ok := placementDecision.Labels[placementLabel]; !ok {
|
||||
t.Errorf("expected Placement label on Placement decision")
|
||||
}
|
||||
|
||||
// check owner reference
|
||||
owner := metav1.GetControllerOf(&placementDecision.ObjectMeta)
|
||||
if owner == nil {
|
||||
t.Errorf("expected PlacementDecision with ownerreference")
|
||||
}
|
||||
|
||||
if owner.Kind != "Placement" {
|
||||
t.Errorf("expected ownerreference with kind %q, but got %q", "Placement", owner.Kind)
|
||||
}
|
||||
|
||||
if owner.Name != placementName {
|
||||
t.Errorf("expected ownerreference with name %q, but got %q", placementName, owner.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementDecisionCreatingControllerSync(t *testing.T) {
|
||||
placementNamespace := "ns1"
|
||||
placementName := "placement1"
|
||||
queueKey := placementNamespace + "/" + placementName
|
||||
placementUID := rand.String(16)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
queueKey string
|
||||
initObjs []runtime.Object
|
||||
validateActions func(t *testing.T, actions []clienttesting.Action)
|
||||
}{
|
||||
{
|
||||
name: "placement not found",
|
||||
queueKey: queueKey,
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
if len(actions) != 0 {
|
||||
t.Errorf("expected no action but got: %v ", actions)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "placement is deleting",
|
||||
queueKey: queueKey,
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewPlacement(placementNamespace, placementName).WithDeletionTimestamp().Build(),
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
if len(actions) != 0 {
|
||||
t.Errorf("expected no action but got: %v ", actions)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "new placement",
|
||||
queueKey: queueKey,
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewPlacement(placementNamespace, placementName).WithUID(placementUID).Build(),
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
// check if PlacementDecision has been created
|
||||
testinghelpers.AssertActions(t, actions, "create")
|
||||
actual := actions[0].(clienttesting.CreateActionImpl).Object
|
||||
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
|
||||
if !ok {
|
||||
t.Errorf("expected PlacementDecision was created")
|
||||
}
|
||||
assertValidPlacementDecision(t, placementNamespace, placementName, placementDecision)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "placementdecision exits",
|
||||
queueKey: queueKey,
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewPlacement(placementNamespace, placementName).WithUID(placementUID).Build(),
|
||||
testinghelpers.NewPlacementDecision(placementNamespace, "decison1").
|
||||
WithPlacementLabel(placementName).WithController(placementUID).Build(),
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
if len(actions) != 0 {
|
||||
t.Errorf("expected no action but got %d: %v", len(actions), actions)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
|
||||
clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)
|
||||
placementStore := clusterInformerFactory.Cluster().V1alpha1().Placements().Informer().GetStore()
|
||||
placementDecisionStore := clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Informer().GetStore()
|
||||
for _, obj := range c.initObjs {
|
||||
switch obj.(type) {
|
||||
case *clusterapiv1alpha1.Placement:
|
||||
placementStore.Add(obj)
|
||||
case *clusterapiv1alpha1.PlacementDecision:
|
||||
placementDecisionStore.Add(obj)
|
||||
}
|
||||
}
|
||||
|
||||
ctrl := placementDecisionCreatingController{
|
||||
clusterClient: clusterClient,
|
||||
placementLister: clusterInformerFactory.Cluster().V1alpha1().Placements().Lister(),
|
||||
placementDecisionLister: clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(),
|
||||
}
|
||||
syncErr := ctrl.sync(context.TODO(), testinghelpers.NewFakeSyncContext(t, c.queueKey))
|
||||
if syncErr != nil {
|
||||
t.Errorf("unexpected err: %v", syncErr)
|
||||
}
|
||||
|
||||
c.validateActions(t, clusterClient.Actions())
|
||||
})
|
||||
}
|
||||
}
|
||||
94
pkg/controllers/scheduling/predicate.go
Normal file
94
pkg/controllers/scheduling/predicate.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
)
|
||||
|
||||
type predicateSelector struct {
|
||||
labelSelector labels.Selector
|
||||
claimSelector labels.Selector
|
||||
}
|
||||
|
||||
// matchWithClusterPredicates returns a slice of clusters. Each of them must match at least one of the predicates.
|
||||
func matchWithClusterPredicates(predicates []clusterapiv1alpha1.ClusterPredicate, clusters []*clusterapiv1.ManagedCluster) ([]*clusterapiv1.ManagedCluster, error) {
|
||||
if len(predicates) == 0 {
|
||||
return clusters, nil
|
||||
}
|
||||
if len(clusters) == 0 {
|
||||
return clusters, nil
|
||||
}
|
||||
|
||||
// prebuild label/claim selectors for each predicate
|
||||
predicateSelectors := []predicateSelector{}
|
||||
for _, predicate := range predicates {
|
||||
// build label selector
|
||||
labelSelector, err := convertLabelSelector(predicate.RequiredClusterSelector.LabelSelector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// build claim selector
|
||||
claimSelector, err := convertClaimSelector(predicate.RequiredClusterSelector.ClaimSelector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
predicateSelectors = append(predicateSelectors, predicateSelector{
|
||||
labelSelector: labelSelector,
|
||||
claimSelector: claimSelector,
|
||||
})
|
||||
}
|
||||
|
||||
// match cluster with selectors one by one
|
||||
matched := []*clusterapiv1.ManagedCluster{}
|
||||
for _, cluster := range clusters {
|
||||
claims := getClusterClaims(cluster)
|
||||
for _, ps := range predicateSelectors {
|
||||
// match with label selector
|
||||
if ok := ps.labelSelector.Matches(labels.Set(cluster.Labels)); !ok {
|
||||
continue
|
||||
}
|
||||
// match with claim selector
|
||||
if ok := ps.claimSelector.Matches(labels.Set(claims)); !ok {
|
||||
continue
|
||||
}
|
||||
matched = append(matched, cluster)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return matched, nil
|
||||
}
|
||||
|
||||
// getClusterClaims returns a map containing cluster claims from the status of cluster
|
||||
func getClusterClaims(cluster *clusterapiv1.ManagedCluster) map[string]string {
|
||||
claims := map[string]string{}
|
||||
for _, claim := range cluster.Status.ClusterClaims {
|
||||
claims[claim.Name] = claim.Value
|
||||
}
|
||||
return claims
|
||||
}
|
||||
|
||||
// convertLabelSelector converts metav1.LabelSelector to labels.Selector
|
||||
func convertLabelSelector(labelSelector metav1.LabelSelector) (labels.Selector, error) {
|
||||
selector, err := metav1.LabelSelectorAsSelector(&labelSelector)
|
||||
if err != nil {
|
||||
return labels.Nothing(), err
|
||||
}
|
||||
|
||||
return selector, nil
|
||||
}
|
||||
|
||||
// convertClaimSelector converts ClusterClaimSelector to labels.Selector
|
||||
func convertClaimSelector(clusterClaimSelector clusterapiv1alpha1.ClusterClaimSelector) (labels.Selector, error) {
|
||||
selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
|
||||
MatchExpressions: clusterClaimSelector.MatchExpressions,
|
||||
})
|
||||
if err != nil {
|
||||
return labels.Nothing(), err
|
||||
}
|
||||
|
||||
return selector, nil
|
||||
}
|
||||
131
pkg/controllers/scheduling/predicate_test.go
Normal file
131
pkg/controllers/scheduling/predicate_test.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
|
||||
)
|
||||
|
||||
func TestMatchWithClusterPredicates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
predicates []clusterapiv1alpha1.ClusterPredicate
|
||||
clusters []*clusterapiv1.ManagedCluster
|
||||
expectedClusterNames []string
|
||||
}{
|
||||
{
|
||||
name: "match with label",
|
||||
predicates: []clusterapiv1alpha1.ClusterPredicate{
|
||||
testinghelpers.NewClusterPredicate(&metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"cloud": "Amazon",
|
||||
},
|
||||
}, nil),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Google").Build(),
|
||||
},
|
||||
expectedClusterNames: []string{"cluster1"},
|
||||
},
|
||||
{
|
||||
name: "match with claim",
|
||||
predicates: []clusterapiv1alpha1.ClusterPredicate{
|
||||
testinghelpers.NewClusterPredicate(nil,
|
||||
&clusterapiv1alpha1.ClusterClaimSelector{
|
||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||
{
|
||||
Key: "cloud",
|
||||
Operator: metav1.LabelSelectorOpIn,
|
||||
Values: []string{"Amazon"},
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithClaim("cloud", "Amazon").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithClaim("cloud", "Google").Build(),
|
||||
},
|
||||
expectedClusterNames: []string{"cluster1"},
|
||||
},
|
||||
{
|
||||
name: "match with both label and claim",
|
||||
predicates: []clusterapiv1alpha1.ClusterPredicate{
|
||||
testinghelpers.NewClusterPredicate(&metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"cloud": "Amazon",
|
||||
},
|
||||
}, &clusterapiv1alpha1.ClusterClaimSelector{
|
||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||
{
|
||||
Key: "region",
|
||||
Operator: metav1.LabelSelectorOpIn,
|
||||
Values: []string{"us-east-1"},
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").
|
||||
WithLabel("cloud", "Amazon").
|
||||
WithClaim("region", "us-east-1").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").
|
||||
WithLabel("cloud", "Amazon").
|
||||
WithClaim("region", "us-east-2").Build(),
|
||||
},
|
||||
expectedClusterNames: []string{"cluster1"},
|
||||
},
|
||||
{
|
||||
name: "match with multiple predicates",
|
||||
predicates: []clusterapiv1alpha1.ClusterPredicate{
|
||||
testinghelpers.NewClusterPredicate(&metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"cloud": "Amazon",
|
||||
},
|
||||
}, nil),
|
||||
testinghelpers.NewClusterPredicate(nil, &clusterapiv1alpha1.ClusterClaimSelector{
|
||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||
{
|
||||
Key: "region",
|
||||
Operator: metav1.LabelSelectorOpIn,
|
||||
Values: []string{"us-east-1"},
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithClaim("region", "us-east-1").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster3").WithClaim("region", "us-east-2").Build(),
|
||||
},
|
||||
expectedClusterNames: []string{"cluster1", "cluster2"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
clusters, err := matchWithClusterPredicates(c.predicates, c.clusters)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
expectedClusterNames := sets.NewString(c.expectedClusterNames...)
|
||||
if len(clusters) != expectedClusterNames.Len() {
|
||||
t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters))
|
||||
}
|
||||
for _, cluster := range clusters {
|
||||
expectedClusterNames.Delete(cluster.Name)
|
||||
}
|
||||
if expectedClusterNames.Len() > 0 {
|
||||
t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ","))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
148
pkg/controllers/scheduling/schedule.go
Normal file
148
pkg/controllers/scheduling/schedule.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
|
||||
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
|
||||
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
)
|
||||
|
||||
type scheduleFunc func(
|
||||
ctx context.Context,
|
||||
placement *clusterapiv1alpha1.Placement,
|
||||
clusters []*clusterapiv1.ManagedCluster,
|
||||
clusterClient clusterclient.Interface,
|
||||
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
|
||||
) (*scheduleResult, error)
|
||||
|
||||
type scheduleResult struct {
|
||||
scheduled int
|
||||
unscheduled int
|
||||
}
|
||||
|
||||
func schedule(
|
||||
ctx context.Context,
|
||||
placement *clusterapiv1alpha1.Placement,
|
||||
clusters []*clusterapiv1.ManagedCluster,
|
||||
clusterClient clusterclient.Interface,
|
||||
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
|
||||
) (*scheduleResult, error) {
|
||||
// filter clusters with cluster predicates
|
||||
feasibleClusters, err := matchWithClusterPredicates(placement.Spec.Predicates, clusters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// select clusters and generate cluster decisions
|
||||
decisions := selectClusters(placement, feasibleClusters)
|
||||
scheduled, unscheduled := len(decisions), 0
|
||||
if placement.Spec.NumberOfClusters != nil {
|
||||
unscheduled = int(*placement.Spec.NumberOfClusters) - scheduled
|
||||
}
|
||||
|
||||
// bind the cluster decisions into placementdecisions
|
||||
err = bind(ctx, placement, decisions, clusterClient, placementDecisionLister)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &scheduleResult{
|
||||
scheduled: scheduled,
|
||||
unscheduled: unscheduled,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// makeClusterDecisions selects clusters based on given cluster slice and then creates
|
||||
// cluster decisions.
|
||||
func selectClusters(placement *clusterapiv1alpha1.Placement, clusters []*clusterapiv1.ManagedCluster) []clusterapiv1alpha1.ClusterDecision {
|
||||
numOfDecisions := len(clusters)
|
||||
if placement.Spec.NumberOfClusters != nil {
|
||||
numOfDecisions = int(*placement.Spec.NumberOfClusters)
|
||||
}
|
||||
|
||||
// truncate the cluster slice if the desired number of decisions is less than
|
||||
// the number of the candidate clusters
|
||||
if numOfDecisions < len(clusters) {
|
||||
clusters = clusters[:numOfDecisions]
|
||||
}
|
||||
|
||||
decisions := []clusterapiv1alpha1.ClusterDecision{}
|
||||
for _, cluster := range clusters {
|
||||
decisions = append(decisions, clusterapiv1alpha1.ClusterDecision{
|
||||
ClusterName: cluster.Name,
|
||||
})
|
||||
}
|
||||
return decisions
|
||||
}
|
||||
|
||||
// bind updates the cluster decisions in the status of the placementdecisions with the given
|
||||
// cluster decision slice. New placementdecision will be created if no one exists.
|
||||
func bind(
|
||||
ctx context.Context,
|
||||
placement *clusterapiv1alpha1.Placement,
|
||||
clusterDecisions []clusterapiv1alpha1.ClusterDecision,
|
||||
clusterClient clusterclient.Interface,
|
||||
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
|
||||
) error {
|
||||
// query placementdecisions with label selector
|
||||
requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
labelSelector := labels.NewSelector().Add(*requirement)
|
||||
placementDecisions, err := placementDecisionLister.PlacementDecisions(placement.Namespace).List(labelSelector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: support multiple placementdecisions for a placement
|
||||
var placementDecision *clusterapiv1alpha1.PlacementDecision
|
||||
switch {
|
||||
case len(placementDecisions) > 0:
|
||||
placementDecision = placementDecisions[0]
|
||||
default:
|
||||
// create a placementdecision if not exists
|
||||
owner := metav1.NewControllerRef(placement, clusterapiv1alpha1.GroupVersion.WithKind("Placement"))
|
||||
placementDecision = &clusterapiv1alpha1.PlacementDecision{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("%s-", placement.Name),
|
||||
Namespace: placement.Namespace,
|
||||
Labels: map[string]string{
|
||||
placementLabel: placement.Name,
|
||||
},
|
||||
OwnerReferences: []metav1.OwnerReference{*owner},
|
||||
},
|
||||
}
|
||||
placementDecision, err = clusterClient.ClusterV1alpha1().PlacementDecisions(placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sort by cluster name
|
||||
sort.SliceStable(clusterDecisions, func(i, j int) bool {
|
||||
return clusterDecisions[i].ClusterName < clusterDecisions[j].ClusterName
|
||||
})
|
||||
|
||||
// update the status of the placementdecision if necessary
|
||||
if reflect.DeepEqual(placementDecision.Status.Decisions, clusterDecisions) {
|
||||
return nil
|
||||
}
|
||||
newPlacementDecision := placementDecision.DeepCopy()
|
||||
newPlacementDecision.Status.Decisions = clusterDecisions
|
||||
_, err = clusterClient.ClusterV1alpha1().PlacementDecisions(newPlacementDecision.Namespace).
|
||||
UpdateStatus(ctx, newPlacementDecision, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
185
pkg/controllers/scheduling/schedule_test.go
Normal file
185
pkg/controllers/scheduling/schedule_test.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
clienttesting "k8s.io/client-go/testing"
|
||||
|
||||
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
|
||||
)
|
||||
|
||||
func TestSchedule(t *testing.T) {
|
||||
clusterSetName := "clusterSets"
|
||||
placementNamespace := "ns1"
|
||||
placementName := "placement1"
|
||||
placementDecisionName := "placement1-decision1"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
placement *clusterapiv1alpha1.Placement
|
||||
initObjs []runtime.Object
|
||||
clusters []*clusterapiv1.ManagedCluster
|
||||
scheduleResult scheduleResult
|
||||
validateActions func(t *testing.T, actions []clienttesting.Action)
|
||||
}{
|
||||
{
|
||||
name: "new placement satisfied",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet(clusterSetName),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
},
|
||||
scheduleResult: scheduleResult{
|
||||
scheduled: 1,
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
// check if PlacementDecision has been created
|
||||
testinghelpers.AssertActions(t, actions, "create", "update")
|
||||
actual := actions[1].(clienttesting.UpdateActionImpl).Object
|
||||
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
|
||||
if !ok {
|
||||
t.Errorf("expected PlacementDecision was updated")
|
||||
}
|
||||
assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "new placement unsatisfied",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(3).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet(clusterSetName),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
},
|
||||
scheduleResult: scheduleResult{
|
||||
scheduled: 1,
|
||||
unscheduled: 2,
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
// check if PlacementDecision has been updated
|
||||
testinghelpers.AssertActions(t, actions, "create", "update")
|
||||
actual := actions[1].(clienttesting.UpdateActionImpl).Object
|
||||
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
|
||||
if !ok {
|
||||
t.Errorf("expected PlacementDecision was updated")
|
||||
}
|
||||
assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "placement with all decisions scheduled",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(2).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet(clusterSetName),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
|
||||
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName).
|
||||
WithLabel(placementLabel, placementName).
|
||||
WithDecisions("cluster1", "cluster2").Build(),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
},
|
||||
scheduleResult: scheduleResult{
|
||||
scheduled: 2,
|
||||
},
|
||||
validateActions: testinghelpers.AssertNoActions,
|
||||
},
|
||||
{
|
||||
name: "placement with part of decisions scheduled",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet(clusterSetName),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
|
||||
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName).
|
||||
WithLabel(placementLabel, placementName).WithDecisions("cluster1").Build(),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
},
|
||||
scheduleResult: scheduleResult{
|
||||
scheduled: 2,
|
||||
unscheduled: 2,
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
// check if PlacementDecision has been updated
|
||||
testinghelpers.AssertActions(t, actions, "update")
|
||||
actual := actions[0].(clienttesting.UpdateActionImpl).Object
|
||||
placementDecision, ok := actual.(*clusterapiv1alpha1.PlacementDecision)
|
||||
if !ok {
|
||||
t.Errorf("expected PlacementDecision was updated")
|
||||
}
|
||||
assertClustersSelected(t, placementDecision.Status.Decisions, "cluster1", "cluster2")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "placement without more feasible cluster available",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet(clusterSetName),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
|
||||
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName).
|
||||
WithLabel(placementLabel, placementName).WithDecisions("cluster1").Build(),
|
||||
},
|
||||
clusters: []*clusterapiv1.ManagedCluster{
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(),
|
||||
},
|
||||
scheduleResult: scheduleResult{
|
||||
scheduled: 1,
|
||||
unscheduled: 3,
|
||||
},
|
||||
validateActions: testinghelpers.AssertNoActions,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
c.initObjs = append(c.initObjs, c.placement)
|
||||
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
|
||||
clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...)
|
||||
result, err := schedule(
|
||||
context.TODO(),
|
||||
c.placement,
|
||||
c.clusters,
|
||||
clusterClient,
|
||||
clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected err: %v", err)
|
||||
}
|
||||
if result.scheduled != c.scheduleResult.scheduled {
|
||||
t.Errorf("expected %d scheduled, but got %d", c.scheduleResult.scheduled, result.scheduled)
|
||||
}
|
||||
if result.unscheduled != c.scheduleResult.unscheduled {
|
||||
t.Errorf("expected %d unscheduled, but got %d", c.scheduleResult.unscheduled, result.unscheduled)
|
||||
}
|
||||
c.validateActions(t, clusterClient.Actions())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertClustersSelected(t *testing.T, decisons []clusterapiv1alpha1.ClusterDecision, clusterNames ...string) {
|
||||
names := sets.NewString(clusterNames...)
|
||||
for _, decision := range decisons {
|
||||
if names.Has(decision.ClusterName) {
|
||||
names.Delete(decision.ClusterName)
|
||||
}
|
||||
}
|
||||
|
||||
if names.Len() != 0 {
|
||||
t.Errorf("expected clusters selected: %s", strings.Join(names.UnsortedList(), ","))
|
||||
}
|
||||
}
|
||||
226
pkg/controllers/scheduling/scheduling_controller.go
Normal file
226
pkg/controllers/scheduling/scheduling_controller.go
Normal file
@@ -0,0 +1,226 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/openshift/library-go/pkg/controller/factory"
|
||||
"github.com/openshift/library-go/pkg/operator/events"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
cache "k8s.io/client-go/tools/cache"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
|
||||
clusterinformerv1alpha1 "github.com/open-cluster-management/api/client/cluster/informers/externalversions/cluster/v1alpha1"
|
||||
clusterlisterv1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1"
|
||||
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
|
||||
placementLabel = "cluster.open-cluster-management.io/placement"
|
||||
)
|
||||
|
||||
var ResyncInterval = 2 * time.Minute
|
||||
|
||||
// schedulingController schedules cluster decisions for Placements
|
||||
type schedulingController struct {
|
||||
clusterClient clusterclient.Interface
|
||||
clusterLister clusterlisterv1.ManagedClusterLister
|
||||
clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister
|
||||
clusterSetBindingLister clusterlisterv1alpha1.ManagedClusterSetBindingLister
|
||||
placementLister clusterlisterv1alpha1.PlacementLister
|
||||
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister
|
||||
scheduleFunc scheduleFunc
|
||||
}
|
||||
|
||||
// NewDecisionSchedulingController return an instance of schedulingController
|
||||
func NewSchedulingController(
|
||||
clusterClient clusterclient.Interface,
|
||||
clusterLister clusterlisterv1.ManagedClusterLister,
|
||||
clusterSetLister clusterlisterv1alpha1.ManagedClusterSetLister,
|
||||
clusterSetBindingLister clusterlisterv1alpha1.ManagedClusterSetBindingLister,
|
||||
placementInformer clusterinformerv1alpha1.PlacementInformer,
|
||||
placementDecisionInformer clusterinformerv1alpha1.PlacementDecisionInformer,
|
||||
recorder events.Recorder,
|
||||
) factory.Controller {
|
||||
// build controller
|
||||
c := schedulingController{
|
||||
clusterClient: clusterClient,
|
||||
clusterLister: clusterLister,
|
||||
clusterSetLister: clusterSetLister,
|
||||
clusterSetBindingLister: clusterSetBindingLister,
|
||||
placementLister: placementInformer.Lister(),
|
||||
placementDecisionLister: placementDecisionInformer.Lister(),
|
||||
scheduleFunc: schedule,
|
||||
}
|
||||
|
||||
return factory.New().
|
||||
WithInformersQueueKeyFunc(func(obj runtime.Object) string {
|
||||
key, _ := cache.MetaNamespaceKeyFunc(obj)
|
||||
return key
|
||||
}, placementInformer.Informer()).
|
||||
WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string {
|
||||
accessor, _ := meta.Accessor(obj)
|
||||
labels := accessor.GetLabels()
|
||||
placementName := labels[placementLabel]
|
||||
return fmt.Sprintf("%s/%s", accessor.GetNamespace(), placementName)
|
||||
}, func(obj interface{}) bool {
|
||||
accessor, err := meta.Accessor(obj)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
labels := accessor.GetLabels()
|
||||
if _, ok := labels[placementLabel]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}, placementDecisionInformer.Informer()).
|
||||
// TODO: monitor more resources, like clusters, clustersets and clustersetbindings
|
||||
WithSync(c.sync).
|
||||
ResyncEvery(ResyncInterval).
|
||||
ToController("SchedulingController", recorder)
|
||||
}
|
||||
|
||||
func (c *schedulingController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
|
||||
queueKey := syncCtx.QueueKey()
|
||||
|
||||
// handle resync
|
||||
if queueKey == factory.DefaultQueueKey {
|
||||
placements, err := c.placementLister.List(labels.Everything())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, placement := range placements {
|
||||
syncCtx.Queue().Add(fmt.Sprintf("%s/%s", placement.Namespace, placement.Name))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sync a placement
|
||||
namespace, name, err := cache.SplitMetaNamespaceKey(queueKey)
|
||||
if err != nil {
|
||||
// ignore placement whose key is not in format: namespace/name
|
||||
utilruntime.HandleError(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
klog.V(4).Infof("Reconciling placement %q", queueKey)
|
||||
placement, err := c.placementLister.Placements(namespace).Get(name)
|
||||
if errors.IsNotFound(err) {
|
||||
// no work if placement is deleted
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// no work if placement is deleting
|
||||
if !placement.DeletionTimestamp.IsZero() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// get available clusters for this placement
|
||||
clusters, err := c.getAvailableClusters(placement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// schedule placement with scheduler
|
||||
scheduleResult, err := c.scheduleFunc(ctx, placement, clusters, c.clusterClient, c.placementDecisionLister)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// update placement status if necessary
|
||||
return c.updateStatus(ctx, placement, scheduleResult.scheduled, scheduleResult.unscheduled)
|
||||
}
|
||||
|
||||
// getAvailableClusters returns available clusters for the given placement. The clusters must
|
||||
// 1) Be from clustersets bound to the placement namespace;
|
||||
// 2) Belong to one of particular clustersets if .spec.clusterSets is specified;
|
||||
func (c *schedulingController) getAvailableClusters(placement *clusterapiv1alpha1.Placement) ([]*clusterapiv1.ManagedCluster, error) {
|
||||
// get all clusterset bindings under the placement namespace
|
||||
bindings, err := c.clusterSetBindingLister.ManagedClusterSetBindings(placement.Namespace).List(labels.Everything())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(bindings) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// filter out invaid clustersetbindings
|
||||
clusterSetNames := sets.NewString()
|
||||
for _, binding := range bindings {
|
||||
// ignore clusterset does not exist
|
||||
_, err := c.clusterSetLister.Get(binding.Name)
|
||||
if errors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clusterSetNames.Insert(binding.Name)
|
||||
}
|
||||
|
||||
// get intersection of clustesets bound to placement namespace and clustesets specified
|
||||
// in placement spec
|
||||
if len(placement.Spec.ClusterSets) != 0 {
|
||||
clusterSetNames = clusterSetNames.Intersection(sets.NewString(placement.Spec.ClusterSets...))
|
||||
}
|
||||
|
||||
if len(clusterSetNames) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// list clusters from particular clustersets
|
||||
requirement, err := labels.NewRequirement(clusterSetLabel, selection.In, clusterSetNames.List())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labelSelector := labels.NewSelector().Add(*requirement)
|
||||
return c.clusterLister.List(labelSelector)
|
||||
}
|
||||
|
||||
// updateStatus updates the status of the placement according to schedule result.
|
||||
func (c *schedulingController) updateStatus(ctx context.Context, placement *clusterapiv1alpha1.Placement, numOfScheduledDecisions, numOfUnscheduledDecisions int) error {
|
||||
newPlacement := placement.DeepCopy()
|
||||
newPlacement.Status.NumberOfSelectedClusters = int32(numOfScheduledDecisions)
|
||||
satisfiedCondition := newSatisfiedCondition(numOfUnscheduledDecisions)
|
||||
meta.SetStatusCondition(&newPlacement.Status.Conditions, satisfiedCondition)
|
||||
if reflect.DeepEqual(newPlacement.Status, placement.Status) {
|
||||
return nil
|
||||
}
|
||||
_, err := c.clusterClient.ClusterV1alpha1().Placements(newPlacement.Namespace).UpdateStatus(ctx, newPlacement, metav1.UpdateOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
// newSatisfiedCondition returns a new condition with type PlacementConditionSatisfied
|
||||
func newSatisfiedCondition(numOfUnscheduledDecisions int) metav1.Condition {
|
||||
condition := metav1.Condition{
|
||||
Type: clusterapiv1alpha1.PlacementConditionSatisfied,
|
||||
}
|
||||
switch {
|
||||
case numOfUnscheduledDecisions == 0:
|
||||
condition.Status = metav1.ConditionTrue
|
||||
condition.Reason = "AllDecisionsScheduled"
|
||||
condition.Message = "All cluster decisions scheduled"
|
||||
default:
|
||||
condition.Status = metav1.ConditionFalse
|
||||
condition.Reason = "NotAllDecisionsScheduled"
|
||||
condition.Message = fmt.Sprintf("%d cluster decisions unscheduled", numOfUnscheduledDecisions)
|
||||
}
|
||||
return condition
|
||||
}
|
||||
210
pkg/controllers/scheduling/scheduling_controller_test.go
Normal file
210
pkg/controllers/scheduling/scheduling_controller_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
clienttesting "k8s.io/client-go/testing"
|
||||
|
||||
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
|
||||
clusterfake "github.com/open-cluster-management/api/client/cluster/clientset/versioned/fake"
|
||||
clusterlisterv1alpha1 "github.com/open-cluster-management/api/client/cluster/listers/cluster/v1alpha1"
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
testinghelpers "github.com/open-cluster-management/placement/pkg/helpers/testing"
|
||||
)
|
||||
|
||||
func TestSchedulingController_sync(t *testing.T) {
|
||||
placementNamespace := "ns1"
|
||||
placementName := "placement1"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
placement *clusterapiv1alpha1.Placement
|
||||
initObjs []runtime.Object
|
||||
scheduleResult *scheduleResult
|
||||
validateActions func(t *testing.T, actions []clienttesting.Action)
|
||||
}{
|
||||
{
|
||||
name: "placement satisfied",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
|
||||
scheduleResult: &scheduleResult{
|
||||
scheduled: 3,
|
||||
unscheduled: 0,
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
// check if PlacementDecision has been updated
|
||||
testinghelpers.AssertActions(t, actions, "update")
|
||||
// check if Placement has been updated
|
||||
actual := actions[0].(clienttesting.UpdateActionImpl).Object
|
||||
placement, ok := actual.(*clusterapiv1alpha1.Placement)
|
||||
if !ok {
|
||||
t.Errorf("expected Placement was updated")
|
||||
}
|
||||
|
||||
if placement.Status.NumberOfSelectedClusters != int32(3) {
|
||||
t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters)
|
||||
}
|
||||
testinghelpers.HasCondition(
|
||||
placement.Status.Conditions,
|
||||
clusterapiv1alpha1.PlacementConditionSatisfied,
|
||||
"AllDecisionsScheduled",
|
||||
metav1.ConditionTrue,
|
||||
)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "placement unsatisfied",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
|
||||
scheduleResult: &scheduleResult{
|
||||
scheduled: 3,
|
||||
unscheduled: 1,
|
||||
},
|
||||
validateActions: func(t *testing.T, actions []clienttesting.Action) {
|
||||
// check if PlacementDecision has been updated
|
||||
testinghelpers.AssertActions(t, actions, "update")
|
||||
// check if Placement has been updated
|
||||
actual := actions[0].(clienttesting.UpdateActionImpl).Object
|
||||
placement, ok := actual.(*clusterapiv1alpha1.Placement)
|
||||
if !ok {
|
||||
t.Errorf("expected Placement was updated")
|
||||
}
|
||||
|
||||
if placement.Status.NumberOfSelectedClusters != int32(3) {
|
||||
t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters)
|
||||
}
|
||||
testinghelpers.HasCondition(
|
||||
placement.Status.Conditions,
|
||||
clusterapiv1alpha1.PlacementConditionSatisfied,
|
||||
"NotAllDecisionsScheduled",
|
||||
metav1.ConditionFalse,
|
||||
)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "placement status not changed",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).
|
||||
WithNumOfSelectedClusters(3).WithSatisfiedCondition(3, 0).Build(),
|
||||
scheduleResult: &scheduleResult{
|
||||
scheduled: 3,
|
||||
unscheduled: 0,
|
||||
},
|
||||
validateActions: testinghelpers.AssertNoActions,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
c.initObjs = append(c.initObjs, c.placement)
|
||||
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
|
||||
clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...)
|
||||
|
||||
ctrl := schedulingController{
|
||||
clusterClient: clusterClient,
|
||||
clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(),
|
||||
clusterSetLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Lister(),
|
||||
clusterSetBindingLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(),
|
||||
placementLister: clusterInformerFactory.Cluster().V1alpha1().Placements().Lister(),
|
||||
placementDecisionLister: clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Lister(),
|
||||
scheduleFunc: func(
|
||||
ctx context.Context,
|
||||
placement *clusterapiv1alpha1.Placement,
|
||||
clusters []*clusterapiv1.ManagedCluster,
|
||||
clusterClient clusterclient.Interface,
|
||||
placementDecisionLister clusterlisterv1alpha1.PlacementDecisionLister,
|
||||
) (*scheduleResult, error) {
|
||||
return c.scheduleResult, nil
|
||||
},
|
||||
}
|
||||
|
||||
sysCtx := testinghelpers.NewFakeSyncContext(t, c.placement.Namespace+"/"+c.placement.Name)
|
||||
syncErr := ctrl.sync(context.TODO(), sysCtx)
|
||||
if syncErr != nil {
|
||||
t.Errorf("unexpected err: %v", syncErr)
|
||||
}
|
||||
|
||||
c.validateActions(t, clusterClient.Actions())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAvailableClusters(t *testing.T) {
|
||||
placementNamespace := "ns1"
|
||||
placementName := "placement1"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
placement *clusterapiv1alpha1.Placement
|
||||
initObjs []runtime.Object
|
||||
expectedClusterNames []string
|
||||
}{
|
||||
{
|
||||
name: "no bound clusterset",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet("clusterset1"),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "select all clusters from bound clustersets",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet("clusterset1"),
|
||||
testinghelpers.NewClusterSet("clusterset2"),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset2"),
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, "clusterset2").Build(),
|
||||
},
|
||||
expectedClusterNames: []string{"cluster1", "cluster2"},
|
||||
},
|
||||
{
|
||||
name: "select clusters from a bound clusterset",
|
||||
placement: testinghelpers.NewPlacement(placementNamespace, placementName).
|
||||
WithClusterSets([]string{"clusterset1"}).Build(),
|
||||
initObjs: []runtime.Object{
|
||||
testinghelpers.NewClusterSet("clusterset1"),
|
||||
testinghelpers.NewClusterSet("clusterset2"),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"),
|
||||
testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset2"),
|
||||
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
|
||||
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, "clusterset2").Build(),
|
||||
},
|
||||
expectedClusterNames: []string{"cluster1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
c.initObjs = append(c.initObjs, c.placement)
|
||||
clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
|
||||
clusterInformerFactory := testinghelpers.NewClusterInformerFactory(clusterClient, c.initObjs...)
|
||||
|
||||
ctrl := &schedulingController{
|
||||
clusterLister: clusterInformerFactory.Cluster().V1().ManagedClusters().Lister(),
|
||||
clusterSetLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Lister(),
|
||||
clusterSetBindingLister: clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Lister(),
|
||||
}
|
||||
clusters, err := ctrl.getAvailableClusters(c.placement)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
expectedClusterNames := sets.NewString(c.expectedClusterNames...)
|
||||
if len(clusters) != expectedClusterNames.Len() {
|
||||
t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters))
|
||||
}
|
||||
for _, cluster := range clusters {
|
||||
expectedClusterNames.Delete(cluster.Name)
|
||||
}
|
||||
if expectedClusterNames.Len() > 0 {
|
||||
t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ","))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,16 @@
|
||||
package testing
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
placementLabel = "cluster.open-cluster-management.io/placement"
|
||||
)
|
||||
|
||||
type placementBuilder struct {
|
||||
placement *clusterapiv1alpha1.Placement
|
||||
}
|
||||
@@ -31,16 +31,73 @@ func (b *placementBuilder) WithUID(uid string) *placementBuilder {
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) WithNOC(noc int32) *placementBuilder {
|
||||
b.placement.Spec.NumberOfClusters = &noc
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) WithClusterSets(clusterSets []string) *placementBuilder {
|
||||
b.placement.Spec.ClusterSets = clusterSets
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) WithDeletionTimestamp() *placementBuilder {
|
||||
now := metav1.Now()
|
||||
b.placement.DeletionTimestamp = &now
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1alpha1.ClusterClaimSelector) *placementBuilder {
|
||||
if b.placement.Spec.Predicates == nil {
|
||||
b.placement.Spec.Predicates = []clusterapiv1alpha1.ClusterPredicate{}
|
||||
}
|
||||
b.placement.Spec.Predicates = append(b.placement.Spec.Predicates, NewClusterPredicate(labelSelector, claimSelector))
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) WithNumOfSelectedClusters(nosc int) *placementBuilder {
|
||||
b.placement.Status.NumberOfSelectedClusters = int32(nosc)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) WithSatisfiedCondition(numbOfScheduledDecisions, numbOfUnscheduledDecisions int) *placementBuilder {
|
||||
condition := metav1.Condition{
|
||||
Type: clusterapiv1alpha1.PlacementConditionSatisfied,
|
||||
}
|
||||
switch {
|
||||
case numbOfUnscheduledDecisions == 0:
|
||||
condition.Status = metav1.ConditionTrue
|
||||
condition.Reason = "AllDecisionsScheduled"
|
||||
condition.Message = "All cluster decisions scheduled"
|
||||
default:
|
||||
condition.Status = metav1.ConditionFalse
|
||||
condition.Reason = "NotAllDecisionsScheduled"
|
||||
condition.Message = fmt.Sprintf("%d cluster decisions unscheduled", numbOfUnscheduledDecisions)
|
||||
}
|
||||
meta.SetStatusCondition(&b.placement.Status.Conditions, condition)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementBuilder) Build() *clusterapiv1alpha1.Placement {
|
||||
return b.placement
|
||||
}
|
||||
|
||||
func NewClusterPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1alpha1.ClusterClaimSelector) clusterapiv1alpha1.ClusterPredicate {
|
||||
predicate := clusterapiv1alpha1.ClusterPredicate{
|
||||
RequiredClusterSelector: clusterapiv1alpha1.ClusterSelector{},
|
||||
}
|
||||
|
||||
if labelSelector != nil {
|
||||
predicate.RequiredClusterSelector.LabelSelector = *labelSelector
|
||||
}
|
||||
|
||||
if claimSelector != nil {
|
||||
predicate.RequiredClusterSelector.ClaimSelector = *claimSelector
|
||||
}
|
||||
|
||||
return predicate
|
||||
}
|
||||
|
||||
type placementDecisionBuilder struct {
|
||||
placementDecision *clusterapiv1alpha1.PlacementDecision
|
||||
}
|
||||
@@ -65,14 +122,96 @@ func (b *placementDecisionBuilder) WithController(uid string) *placementDecision
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementDecisionBuilder) WithPlacementLabel(placementName string) *placementDecisionBuilder {
|
||||
func (b *placementDecisionBuilder) WithLabel(name, value string) *placementDecisionBuilder {
|
||||
if b.placementDecision.Labels == nil {
|
||||
b.placementDecision.Labels = map[string]string{}
|
||||
}
|
||||
b.placementDecision.Labels[placementLabel] = placementName
|
||||
b.placementDecision.Labels[name] = value
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementDecisionBuilder) WithDeletionTimestamp() *placementDecisionBuilder {
|
||||
now := metav1.Now()
|
||||
b.placementDecision.DeletionTimestamp = &now
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementDecisionBuilder) WithDecisions(clusterNames ...string) *placementDecisionBuilder {
|
||||
decisions := []clusterapiv1alpha1.ClusterDecision{}
|
||||
for _, clusterName := range clusterNames {
|
||||
decisions = append(decisions, clusterapiv1alpha1.ClusterDecision{
|
||||
ClusterName: clusterName,
|
||||
})
|
||||
}
|
||||
b.placementDecision.Status.Decisions = decisions
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *placementDecisionBuilder) Build() *clusterapiv1alpha1.PlacementDecision {
|
||||
return b.placementDecision
|
||||
}
|
||||
|
||||
type managedClusterBuilder struct {
|
||||
cluster *clusterapiv1.ManagedCluster
|
||||
}
|
||||
|
||||
func NewManagedCluster(clusterName string) *managedClusterBuilder {
|
||||
return &managedClusterBuilder{
|
||||
cluster: &clusterapiv1.ManagedCluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: clusterName,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (b *managedClusterBuilder) WithLabel(name, value string) *managedClusterBuilder {
|
||||
if b.cluster.Labels == nil {
|
||||
b.cluster.Labels = map[string]string{}
|
||||
}
|
||||
b.cluster.Labels[name] = value
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *managedClusterBuilder) WithClaim(name, value string) *managedClusterBuilder {
|
||||
claimMap := map[string]string{}
|
||||
for _, claim := range b.cluster.Status.ClusterClaims {
|
||||
claimMap[claim.Name] = claim.Value
|
||||
}
|
||||
claimMap[name] = value
|
||||
|
||||
clusterClaims := []clusterapiv1.ManagedClusterClaim{}
|
||||
for k, v := range claimMap {
|
||||
clusterClaims = append(clusterClaims, clusterapiv1.ManagedClusterClaim{
|
||||
Name: k,
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
|
||||
b.cluster.Status.ClusterClaims = clusterClaims
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *managedClusterBuilder) Build() *clusterapiv1.ManagedCluster {
|
||||
return b.cluster
|
||||
}
|
||||
|
||||
func NewClusterSet(clusterSetName string) *clusterapiv1alpha1.ManagedClusterSet {
|
||||
return &clusterapiv1alpha1.ManagedClusterSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: clusterSetName,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewClusterSetBinding(namespace, clusterSetName string) *clusterapiv1alpha1.ManagedClusterSetBinding {
|
||||
return &clusterapiv1alpha1.ManagedClusterSetBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
Name: clusterSetName,
|
||||
},
|
||||
Spec: clusterapiv1alpha1.ManagedClusterSetBindingSpec{
|
||||
ClusterSet: clusterSetName,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/openshift/library-go/pkg/operator/events"
|
||||
"github.com/openshift/library-go/pkg/operator/events/eventstesting"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
clienttesting "k8s.io/client-go/testing"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
)
|
||||
@@ -43,3 +44,25 @@ func AssertActions(t *testing.T, actualActions []clienttesting.Action, expectedV
|
||||
func AssertNoActions(t *testing.T, actualActions []clienttesting.Action) {
|
||||
AssertActions(t, actualActions)
|
||||
}
|
||||
|
||||
func HasCondition(conditions []metav1.Condition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool {
|
||||
found := false
|
||||
for _, condition := range conditions {
|
||||
if condition.Type != expectedType {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
|
||||
if condition.Status != expectedStatus {
|
||||
return false
|
||||
}
|
||||
|
||||
if condition.Reason != expectedReason {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
37
pkg/helpers/testing/informer.go
Normal file
37
pkg/helpers/testing/informer.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package testing
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
clusterclient "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
|
||||
clusterinformers "github.com/open-cluster-management/api/client/cluster/informers/externalversions"
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
)
|
||||
|
||||
func NewClusterInformerFactory(clusterClient clusterclient.Interface, objects ...runtime.Object) clusterinformers.SharedInformerFactory {
|
||||
clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)
|
||||
clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore()
|
||||
clusterSetStore := clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSets().Informer().GetStore()
|
||||
clusterSetBindingStore := clusterInformerFactory.Cluster().V1alpha1().ManagedClusterSetBindings().Informer().GetStore()
|
||||
placementStore := clusterInformerFactory.Cluster().V1alpha1().Placements().Informer().GetStore()
|
||||
placementDecisionStore := clusterInformerFactory.Cluster().V1alpha1().PlacementDecisions().Informer().GetStore()
|
||||
|
||||
for _, obj := range objects {
|
||||
switch obj.(type) {
|
||||
case *clusterapiv1.ManagedCluster:
|
||||
clusterStore.Add(obj)
|
||||
case *clusterapiv1alpha1.ManagedClusterSet:
|
||||
clusterSetStore.Add(obj)
|
||||
case *clusterapiv1alpha1.ManagedClusterSetBinding:
|
||||
clusterSetBindingStore.Add(obj)
|
||||
case *clusterapiv1alpha1.Placement:
|
||||
placementStore.Add(obj)
|
||||
case *clusterapiv1alpha1.PlacementDecision:
|
||||
placementDecisionStore.Add(obj)
|
||||
}
|
||||
}
|
||||
|
||||
return clusterInformerFactory
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package integration
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
"github.com/onsi/gomega"
|
||||
@@ -12,25 +13,32 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/rand"
|
||||
|
||||
clusterapiv1 "github.com/open-cluster-management/api/cluster/v1"
|
||||
clusterapiv1alpha1 "github.com/open-cluster-management/api/cluster/v1alpha1"
|
||||
controllers "github.com/open-cluster-management/placement/pkg/controllers"
|
||||
"github.com/open-cluster-management/placement/pkg/controllers/scheduling"
|
||||
"github.com/open-cluster-management/placement/test/integration/util"
|
||||
)
|
||||
|
||||
const (
|
||||
placementLabel = "cluster.open-cluster-management.io/placement"
|
||||
clusterSetLabel = "cluster.open-cluster-management.io/clusterset"
|
||||
placementLabel = "cluster.open-cluster-management.io/placement"
|
||||
)
|
||||
|
||||
var _ = ginkgo.Describe("Placement Scheduling", func() {
|
||||
var _ = ginkgo.Describe("Placement", func() {
|
||||
var cancel context.CancelFunc
|
||||
var namespace string
|
||||
var placementName string
|
||||
var clusterSet1Name, clusterSet2Name string
|
||||
var suffix string
|
||||
var err error
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
suffix := rand.String(5)
|
||||
suffix = rand.String(5)
|
||||
namespace = fmt.Sprintf("ns-%s", suffix)
|
||||
placementName = fmt.Sprintf("placement-%s", suffix)
|
||||
clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix)
|
||||
clusterSet2Name = fmt.Sprintf("clusterset-%s", rand.String(5))
|
||||
|
||||
// create testing namespace
|
||||
ns := &corev1.Namespace{
|
||||
@@ -42,6 +50,7 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
// start controller manager
|
||||
scheduling.ResyncInterval = 10 * time.Second
|
||||
var ctx context.Context
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
go controllers.RunControllerManager(ctx, &controllercmd.ControllerContext{
|
||||
@@ -58,22 +67,11 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
})
|
||||
|
||||
ginkgo.It("Should create placement successfully", func() {
|
||||
ginkgo.By("Create placement")
|
||||
placement := &clusterapiv1alpha1.Placement{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
Name: placementName,
|
||||
},
|
||||
}
|
||||
|
||||
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
assertPlacementDecisionCreated := func(placement *clusterapiv1alpha1.Placement) {
|
||||
ginkgo.By("Check if placementdecision is created")
|
||||
gomega.Eventually(func() bool {
|
||||
pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
|
||||
LabelSelector: placementLabel + "=" + placementName,
|
||||
LabelSelector: placementLabel + "=" + placement.Name,
|
||||
})
|
||||
if err != nil {
|
||||
return false
|
||||
@@ -88,11 +86,9 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
|
||||
}
|
||||
return true
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
|
||||
}
|
||||
|
||||
ginkgo.By("Delete placement")
|
||||
err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.Background(), placementName, metav1.DeleteOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
assertPlacementDeleted := func(placementName string) {
|
||||
ginkgo.By("Check if placement is gone")
|
||||
gomega.Eventually(func() bool {
|
||||
_, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
|
||||
@@ -101,5 +97,189 @@ var _ = ginkgo.Describe("Placement Scheduling", func() {
|
||||
}
|
||||
return errors.IsNotFound(err)
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
|
||||
}
|
||||
|
||||
assertNumberOfDecisions := func(placementName string, desiredNOD int) {
|
||||
ginkgo.By("Check the number of decisions in placementdecisions")
|
||||
gomega.Eventually(func() bool {
|
||||
pdl, err := clusterClient.ClusterV1alpha1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
|
||||
LabelSelector: placementLabel + "=" + placementName,
|
||||
})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if len(pdl.Items) == 0 {
|
||||
return false
|
||||
}
|
||||
actualNOD := 0
|
||||
for _, pd := range pdl.Items {
|
||||
actualNOD += len(pd.Status.Decisions)
|
||||
}
|
||||
return actualNOD == desiredNOD
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
|
||||
}
|
||||
|
||||
assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) {
|
||||
ginkgo.By("Check the status of placement")
|
||||
gomega.Eventually(func() bool {
|
||||
placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if satisfied && !util.HasCondition(
|
||||
placement.Status.Conditions,
|
||||
clusterapiv1alpha1.PlacementConditionSatisfied,
|
||||
"AllDecisionsScheduled",
|
||||
metav1.ConditionTrue,
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if !satisfied && !util.HasCondition(
|
||||
placement.Status.Conditions,
|
||||
clusterapiv1alpha1.PlacementConditionSatisfied,
|
||||
"NotAllDecisionsScheduled",
|
||||
metav1.ConditionFalse,
|
||||
) {
|
||||
return false
|
||||
}
|
||||
return placement.Status.NumberOfSelectedClusters == int32(numOfSelectedClusters)
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
|
||||
}
|
||||
|
||||
assertBindingClusterSet := func(clusterSetName string) {
|
||||
ginkgo.By("Create clusterset/clustersetbinding")
|
||||
clusterset := &clusterapiv1alpha1.ManagedClusterSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: clusterSetName,
|
||||
},
|
||||
}
|
||||
_, err = clusterClient.ClusterV1alpha1().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
csb := &clusterapiv1alpha1.ManagedClusterSetBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
Name: clusterSetName,
|
||||
},
|
||||
Spec: clusterapiv1alpha1.ManagedClusterSetBindingSpec{
|
||||
ClusterSet: clusterSetName,
|
||||
},
|
||||
}
|
||||
_, err = clusterClient.ClusterV1alpha1().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
}
|
||||
|
||||
assertCreatingClusters := func(clusterSetName string, num int) {
|
||||
ginkgo.By(fmt.Sprintf("Create %d clusters", num))
|
||||
for i := 0; i < num; i++ {
|
||||
cluster := &clusterapiv1.ManagedCluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: "cluster-",
|
||||
Labels: map[string]string{
|
||||
clusterSetLabel: clusterSetName,
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
}
|
||||
}
|
||||
|
||||
assertCreatingPlacement := func(name string, noc *int32, nod int) {
|
||||
ginkgo.By("Create placement")
|
||||
placement := &clusterapiv1alpha1.Placement{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
},
|
||||
Spec: clusterapiv1alpha1.PlacementSpec{
|
||||
NumberOfClusters: noc,
|
||||
},
|
||||
}
|
||||
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
assertPlacementDecisionCreated(placement)
|
||||
assertNumberOfDecisions(placementName, nod)
|
||||
if noc != nil {
|
||||
assertPlacementStatus(placementName, nod, nod == int(*noc))
|
||||
}
|
||||
}
|
||||
|
||||
ginkgo.Context("Scheduling", func() {
|
||||
ginkgo.AfterEach(func() {
|
||||
ginkgo.By("Delete placement")
|
||||
err = clusterClient.ClusterV1alpha1().Placements(namespace).Delete(context.Background(), placementName, metav1.DeleteOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
assertPlacementDeleted(placementName)
|
||||
})
|
||||
|
||||
ginkgo.It("Should schedule successfully once spec.NumberOfClusters is reduced", func() {
|
||||
assertBindingClusterSet(clusterSet1Name)
|
||||
assertCreatingClusters(clusterSet1Name, 5)
|
||||
assertCreatingPlacement(placementName, noc(10), 5)
|
||||
|
||||
ginkgo.By("Reduce NOC of the placement")
|
||||
placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
noc := int32(4)
|
||||
placement.Spec.NumberOfClusters = &noc
|
||||
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
nod := int(noc)
|
||||
assertNumberOfDecisions(placementName, nod)
|
||||
assertPlacementStatus(placementName, nod, true)
|
||||
})
|
||||
|
||||
ginkgo.It("Should schedule successfully once spec.NumberOfClusters is increased", func() {
|
||||
assertBindingClusterSet(clusterSet1Name)
|
||||
assertCreatingClusters(clusterSet1Name, 10)
|
||||
assertCreatingPlacement(placementName, noc(5), 5)
|
||||
|
||||
ginkgo.By("Increase NOC of the placement")
|
||||
placement, err := clusterClient.ClusterV1alpha1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
noc := int32(8)
|
||||
placement.Spec.NumberOfClusters = &noc
|
||||
placement, err = clusterClient.ClusterV1alpha1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
|
||||
nod := int(noc)
|
||||
assertNumberOfDecisions(placementName, nod)
|
||||
assertPlacementStatus(placementName, nod, true)
|
||||
})
|
||||
|
||||
ginkgo.It("Should be satisfied once new clusters are added", func() {
|
||||
assertBindingClusterSet(clusterSet1Name)
|
||||
assertCreatingClusters(clusterSet1Name, 5)
|
||||
assertCreatingPlacement(placementName, noc(10), 5)
|
||||
|
||||
// add more clusters
|
||||
assertCreatingClusters(clusterSet1Name, 5)
|
||||
|
||||
nod := 10
|
||||
assertNumberOfDecisions(placementName, nod)
|
||||
assertPlacementStatus(placementName, nod, true)
|
||||
})
|
||||
|
||||
ginkgo.It("Should schedule successfully once new clusterset is bound", func() {
|
||||
assertBindingClusterSet(clusterSet1Name)
|
||||
assertCreatingClusters(clusterSet1Name, 5)
|
||||
assertCreatingPlacement(placementName, noc(10), 5)
|
||||
|
||||
ginkgo.By("Bind one more clusterset to the placement namespace")
|
||||
assertBindingClusterSet(clusterSet2Name)
|
||||
assertCreatingClusters(clusterSet2Name, 3)
|
||||
|
||||
nod := 8
|
||||
assertNumberOfDecisions(placementName, nod)
|
||||
assertPlacementStatus(placementName, nod, false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func noc(n int) *int32 {
|
||||
noc := int32(n)
|
||||
return &noc
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
|
||||
"github.com/openshift/library-go/pkg/operator/events"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func NewIntegrationTestEventRecorder(componet string) events.Recorder {
|
||||
@@ -47,3 +47,25 @@ func (r *IntegrationTestEventRecorder) Warningf(reason, messageFmt string, args
|
||||
func (r *IntegrationTestEventRecorder) Shutdown() {
|
||||
return
|
||||
}
|
||||
|
||||
func HasCondition(conditions []metav1.Condition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool {
|
||||
found := false
|
||||
for _, condition := range conditions {
|
||||
if condition.Type != expectedType {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
|
||||
if condition.Status != expectedStatus {
|
||||
return false
|
||||
}
|
||||
|
||||
if condition.Reason != expectedReason {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user