github/open-cluster-management
1
0
Fork 0
mirror of https://github.com/open-cluster-management-io/ocm.git synced 2026-05-22 00:54:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add filter plugin taint toleration (#63)

Browse Source 
* add plugin tainittoleration based on PR #61

Signed-off-by: haoqing0110 <qhao@redhat.com>

* cover PreferNoSelect and NoSelectIfNew case

Signed-off-by: haoqing0110 <qhao@redhat.com>
This commit is contained in:
Qing Hao
2022-03-03 00:04:30 +08:00
committed by GitHub
parent 9898dbd56b
commit 2152581aea
6 changed files with 605 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/controllers/scheduling/schedule.go
View File

@@ -18,6 +18,7 @@ import (
"open-cluster-management.io/placement/pkg/plugins/predicate"
"open-cluster-management.io/placement/pkg/plugins/resource"
"open-cluster-management.io/placement/pkg/plugins/steady"
"open-cluster-management.io/placement/pkg/plugins/tainttoleration"
)
const (
@@ -139,6 +140,7 @@ func NewPluginScheduler(handle plugins.Handle) *pluginScheduler {
handle: handle,
filters: []plugins.Filter{
predicate.New(handle),
tainttoleration.New(handle),
},
prioritizerWeights: defaultPrioritizerConfig,
}
@@ -324,10 +326,21 @@ func getPrioritizers(weights map[clusterapiv1beta1.ScoreCoordinate]int32, handle
func (r *scheduleResult) FilterResults() []FilterResult {
results := []FilterResult{}
for name, r := range r.filteredRecords {
// order the FilterResults by key length
filteredRecordsKey := []string{}
for name := range r.filteredRecords {
filteredRecordsKey = append(filteredRecordsKey, name)
}
sort.SliceStable(filteredRecordsKey, func(i, j int) bool {
return len(filteredRecordsKey[i]) < len(filteredRecordsKey[j])
})
// go through the FilterResults by key length
for _, name := range filteredRecordsKey {
result := FilterResult{Name: name, FilteredClusters: []string{}}
for _, c := range r {
for _, c := range r.filteredRecords[name] {
result.FilteredClusters = append(result.FilteredClusters, c.Name)
}
results = append(results, result)

106
pkg/controllers/scheduling/schedule_test.go
View File

@@ -10,6 +10,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterfake "open-cluster-management.io/api/client/cluster/clientset/versioned/fake"
clusterapiv1 "open-cluster-management.io/api/cluster/v1"
clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
@@ -49,6 +50,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -86,6 +91,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -130,6 +139,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -161,6 +174,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -179,6 +196,67 @@ func TestSchedule(t *testing.T) {
},
expectedUnScheduled: 0,
},
{
name: "placement with taint and toleration",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(3).AddToleration(
&clusterapiv1beta1.Toleration{
Key: "key1",
Value: "value1",
Operator: clusterapiv1beta1.TolerationOpEqual,
}).Build(),
initObjs: []runtime.Object{
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
testinghelpers.NewAddOnPlacementScore("cluster1", "demo").WithScore("demo", 30).Build(),
testinghelpers.NewAddOnPlacementScore("cluster2", "demo").WithScore("demo", 40).Build(),
testinghelpers.NewAddOnPlacementScore("cluster3", "demo").WithScore("demo", 50).Build(),
},
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterSetLabel, clusterSetName).Build(),
},
decisions: []runtime.Object{},
expectedDecisions: []clusterapiv1beta1.ClusterDecision{
{ClusterName: "cluster1"},
{ClusterName: "cluster3"},
},
expectedFilterResult: []FilterResult{
{
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1", "cluster3"},
},
},
expectedScoreResult: []PrioritizerResult{
{
Name: "Balance",
Weight: 1,
Scores: PrioritizerScore{"cluster1": 100, "cluster3": 100},
},
{
Name: "Steady",
Weight: 1,
Scores: PrioritizerScore{"cluster1": 0, "cluster3": 0},
},
},
expectedUnScheduled: 1,
},
{
name: "placement with additive Prioritizer Policy",
placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(2).WithPrioritizerPolicy("Additive").WithPrioritizerConfig("Balance", 3).WithPrioritizerConfig("ResourceAllocatableMemory", 1).WithScoreCoordinateAddOn("demo", "demo", 1).Build(),
@@ -204,6 +282,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -251,6 +333,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -294,6 +380,10 @@ func TestSchedule(t *testing.T) {
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster1", "cluster2"},
},
},
expectedScoreResult: []PrioritizerResult{
{
@@ -333,6 +423,10 @@ func TestSchedule(t *testing.T) {
expectedFilterResult: []FilterResult{
{
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster3", "cluster1", "cluster2"},
},
},
@@ -357,18 +451,18 @@ func TestSchedule(t *testing.T) {
testinghelpers.NewClusterSet(clusterSetName),
testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)).
WithDecisions("cluster3", "cluster2").Build(),
WithDecisions("cluster2", "cluster3").Build(),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 2)).
WithDecisions("cluster2", "cluster1").Build(),
WithDecisions("cluster1", "cluster2").Build(),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)).
WithLabel(placementLabel, placementName).
WithDecisions("cluster3").Build(),
},
decisions: []runtime.Object{
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)).
WithDecisions("cluster3", "cluster2").Build(),
WithDecisions("cluster2", "cluster3").Build(),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 2)).
WithDecisions("cluster2", "cluster1").Build(),
WithDecisions("cluster1", "cluster2").Build(),
testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)).
WithLabel(placementLabel, placementName).
WithDecisions("cluster3").Build(),
@@ -384,6 +478,10 @@ func TestSchedule(t *testing.T) {
expectedFilterResult: []FilterResult{
{
Name: "Predicate",
FilteredClusters: []string{"cluster1", "cluster2", "cluster3"},
},
{
Name: "Predicate,TaintToleration",
FilteredClusters: []string{"cluster3", "cluster1", "cluster2"},
},
},

16
pkg/helpers/testing/builders.go
View File

@@ -97,6 +97,14 @@ func (b *placementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, cla
return b
}
func (b *placementBuilder) AddToleration(toleration *clusterapiv1beta1.Toleration) *placementBuilder {
if b.placement.Spec.Tolerations == nil {
b.placement.Spec.Tolerations = []clusterapiv1beta1.Toleration{}
}
b.placement.Spec.Tolerations = append(b.placement.Spec.Tolerations, *toleration)
return b
}
func (b *placementBuilder) WithNumOfSelectedClusters(nosc int) *placementBuilder {
b.placement.Status.NumberOfSelectedClusters = int32(nosc)
return b
@@ -247,6 +255,14 @@ func (b *managedClusterBuilder) WithResource(resourceName clusterapiv1.ResourceN
return b
}
func (b *managedClusterBuilder) WithTaint(taint *clusterapiv1.Taint) *managedClusterBuilder {
if b.cluster.Spec.Taints == nil {
b.cluster.Spec.Taints = []clusterapiv1.Taint{}
}
b.cluster.Spec.Taints = append(b.cluster.Spec.Taints, *taint)
return b
}
func (b *managedClusterBuilder) Build() *clusterapiv1.ManagedCluster {
return b.cluster
}

136
pkg/plugins/tainttoleration/taint_toleration.go Normal file
View File

@@ -0,0 +1,136 @@
package tainttoleration
import (
"context"
"errors"
"reflect"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/util/sets"
clusterapiv1 "open-cluster-management.io/api/cluster/v1"
clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
"open-cluster-management.io/placement/pkg/plugins"
)
var _ plugins.Filter = &TaintToleration{}
const (
placementLabel = "cluster.open-cluster-management.io/placement"
description = "TaintToleration is a plugin that checks if a placement tolerates a managed cluster's taints"
)
type TaintToleration struct {
handle plugins.Handle
}
func New(handle plugins.Handle) *TaintToleration {
return &TaintToleration{
handle: handle,
}
}
func (p *TaintToleration) Name() string {
return reflect.TypeOf(*p).Name()
}
func (pl *TaintToleration) Description() string {
return description
}
func (pl *TaintToleration) Filter(ctx context.Context, placement *clusterapiv1beta1.Placement, clusters []*clusterapiv1.ManagedCluster) ([]*clusterapiv1.ManagedCluster, error) {
if len(clusters) == 0 {
return clusters, nil
}
// do validation on each toleration and return error if necessary
for _, toleration := range placement.Spec.Tolerations {
if len(toleration.Key) == 0 && toleration.Operator != clusterapiv1beta1.TolerationOpExists {
return nil, errors.New("If the key is empty, operator must be Exists.\n")
}
if toleration.Operator == clusterapiv1beta1.TolerationOpExists && len(toleration.Value) > 0 {
return nil, errors.New("If the operator is Exists, the value should be empty.\n")
}
}
existingDecisions := getDecisions(pl.handle, placement)
// filter the clusters
matched := []*clusterapiv1.ManagedCluster{}
for _, cluster := range clusters {
if isClusterTolerated(cluster, placement.Spec.Tolerations, existingDecisions.Has(cluster.Name)) {
matched = append(matched, cluster)
}
}
return matched, nil
}
// isClusterTolerated returns true if a cluster is tolerated by the given toleration array
func isClusterTolerated(cluster *clusterapiv1.ManagedCluster, tolerations []clusterapiv1beta1.Toleration, inDecision bool) bool {
for _, taint := range cluster.Spec.Taints {
if !isTaintTolerated(taint, tolerations, inDecision) {
return false
}
}
return true
}
// isTaintTolerated returns true if a taint is tolerated by the given toleration array
func isTaintTolerated(taint clusterapiv1.Taint, tolerations []clusterapiv1beta1.Toleration, inDecision bool) bool {
if (taint.Effect == clusterapiv1.TaintEffectPreferNoSelect) || (taint.Effect == clusterapiv1.TaintEffectNoSelectIfNew && inDecision) {
return true
}
for _, toleration := range tolerations {
if isTolerated(taint, toleration) {
return true
}
}
return false
}
// isTolerated returns true if a taint is tolerated by the given toleration
func isTolerated(taint clusterapiv1.Taint, toleration clusterapiv1beta1.Toleration) bool {
if len(toleration.Effect) > 0 && toleration.Effect != taint.Effect {
return false
}
if len(toleration.Key) > 0 && toleration.Key != taint.Key {
return false
}
switch toleration.Operator {
// empty operator means Equal
case "", clusterapiv1beta1.TolerationOpEqual:
return toleration.Value == taint.Value
case clusterapiv1beta1.TolerationOpExists:
return true
default:
return false
}
}
func getDecisions(handle plugins.Handle, placement *clusterapiv1beta1.Placement) sets.String {
existingDecisions := sets.String{}
// query placementdecisions with label selector
requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name})
if err != nil {
return existingDecisions
}
labelSelector := labels.NewSelector().Add(*requirement)
decisions, err := handle.DecisionLister().PlacementDecisions(placement.Namespace).List(labelSelector)
if err != nil {
return existingDecisions
}
for _, decision := range decisions {
for _, d := range decision.Status.Decisions {
existingDecisions.Insert(d.ClusterName)
}
}
return existingDecisions
}

259
pkg/plugins/tainttoleration/taint_toleration_test.go Normal file
View File

@@ -0,0 +1,259 @@
package tainttoleration
import (
"context"
"strings"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
clusterapiv1 "open-cluster-management.io/api/cluster/v1"
clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
testinghelpers "open-cluster-management.io/placement/pkg/helpers/testing"
)
func TestMatchWithClusterTaintToleration(t *testing.T) {
cases := []struct {
name string
placement *clusterapiv1beta1.Placement
clusters []*clusterapiv1.ManagedCluster
existingDecisions []runtime.Object
expectedClusterNames []string
}{
{
name: "taint.Effect is NoSelect and tolerations is empty",
placement: testinghelpers.NewPlacement("test", "test").Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{},
expectedClusterNames: []string{},
},
{
name: "taint.Effect is NoSelect and tolerations.Operator is Equal",
placement: testinghelpers.NewPlacement("test", "test").AddToleration(
&clusterapiv1beta1.Toleration{
Key: "key1",
Value: "value1",
Operator: clusterapiv1beta1.TolerationOpEqual,
}).Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{},
expectedClusterNames: []string{"cluster1"},
},
{
name: "taint.Effect is NoSelect and tolerations.Operator is Exist",
placement: testinghelpers.NewPlacement("test", "test").AddToleration(
&clusterapiv1beta1.Toleration{
Key: "key1",
Operator: clusterapiv1beta1.TolerationOpExists,
}).Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{},
expectedClusterNames: []string{"cluster1"},
},
{
name: "taint.Effect is NoSelectIfNew and tolerations is empty",
placement: testinghelpers.NewPlacement("test", "test").Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelectIfNew,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelectIfNew,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster3").WithTaint(
&clusterapiv1.Taint{
Key: "key3",
Value: "value3",
Effect: clusterapiv1.TaintEffectNoSelectIfNew,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{
testinghelpers.NewPlacementDecision("test", "test").WithLabel(placementLabel, "test").WithDecisions("cluster2").Build(),
},
expectedClusterNames: []string{"cluster2"},
},
{
name: "taint.Effect is NoSelectIfNew and tolerations is Exist",
placement: testinghelpers.NewPlacement("test", "test").AddToleration(
&clusterapiv1beta1.Toleration{
Key: "key1",
Operator: clusterapiv1beta1.TolerationOpExists,
}).Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelectIfNew,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelectIfNew,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster3").WithTaint(
&clusterapiv1.Taint{
Key: "key3",
Value: "value3",
Effect: clusterapiv1.TaintEffectNoSelectIfNew,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{
testinghelpers.NewPlacementDecision("test", "test").WithLabel(placementLabel, "test").WithDecisions("cluster2").Build(),
},
expectedClusterNames: []string{"cluster1", "cluster2"},
},
{
name: "taint.Effect is PreferNoSelect and tolerations is Empty",
placement: testinghelpers.NewPlacement("test", "test").Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectPreferNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key1",
Value: "value2",
Effect: clusterapiv1.TaintEffectPreferNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{},
expectedClusterNames: []string{"cluster1", "cluster2"},
},
{
name: "taint.Effect is PreferNoSelect and tolerations is Exist",
placement: testinghelpers.NewPlacement("test", "test").AddToleration(
&clusterapiv1beta1.Toleration{
Key: "key1",
Operator: clusterapiv1beta1.TolerationOpExists,
}).Build(),
clusters: []*clusterapiv1.ManagedCluster{
testinghelpers.NewManagedCluster("cluster1").WithTaint(
&clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectPreferNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
testinghelpers.NewManagedCluster("cluster2").WithTaint(
&clusterapiv1.Taint{
Key: "key3",
Value: "value3",
Effect: clusterapiv1.TaintEffectPreferNoSelect,
TimeAdded: metav1.Time{},
}).Build(),
},
existingDecisions: []runtime.Object{},
expectedClusterNames: []string{"cluster1", "cluster2"},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
p := &TaintToleration{
handle: testinghelpers.NewFakePluginHandle(t, nil, c.existingDecisions...),
}
clusters, err := p.Filter(context.TODO(), c.placement, c.clusters)
if err != nil {
t.Errorf("unexpected err: %v", err)
}
expectedClusterNames := sets.NewString(c.expectedClusterNames...)
if len(clusters) != expectedClusterNames.Len() {
t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters))
}
for _, cluster := range clusters {
expectedClusterNames.Delete(cluster.Name)
}
if expectedClusterNames.Len() > 0 {
t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ","))
}
})
}
}

94
test/integration/placement_test.go
View File

@@ -344,6 +344,21 @@ var _ = ginkgo.Describe("Placement", func() {
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
assertUpdatingClusterWithClusterTaint := func(managedClusterName string, taint *clusterapiv1.Taint) {
ginkgo.By(fmt.Sprintf("Updating ManagedClusters %s taint", managedClusterName))
if taint == nil {
return
}
mc, err := clusterClient.ClusterV1().ManagedClusters().Get(context.Background(), managedClusterName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
mc.Spec.Taints = append(mc.Spec.Taints, *taint)
_, err = clusterClient.ClusterV1().ManagedClusters().Update(context.Background(), mc, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
assertDeletingCluster := func(clusterName string) {
ginkgo.By(fmt.Sprintf("Delete cluster %s", clusterName))
err = clusterClient.ClusterV1().ManagedClusters().Delete(context.Background(), clusterName, metav1.DeleteOptions{})
@@ -359,7 +374,7 @@ var _ = ginkgo.Describe("Placement", func() {
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
assertCreatingPlacement := func(name string, noc *int32, nod int, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy) {
assertCreatingPlacement := func(name string, noc *int32, nod int, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy, tolerations []clusterapiv1beta1.Toleration) {
ginkgo.By("Create placement")
placement := &clusterapiv1beta1.Placement{
ObjectMeta: metav1.ObjectMeta{
@@ -369,6 +384,7 @@ var _ = ginkgo.Describe("Placement", func() {
Spec: clusterapiv1beta1.PlacementSpec{
NumberOfClusters: noc,
PrioritizerPolicy: prioritizerPolicy,
Tolerations: tolerations,
},
}
placement, err = clusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
@@ -432,7 +448,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should re-create placementdecisions successfully once placementdecisions are deleted", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
ginkgo.By("Delete placementdecisions")
placementDecisions, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
@@ -456,7 +472,7 @@ var _ = ginkgo.Describe("Placement", func() {
assertBindingClusterSet(clusterSet2Name)
assertCreatingClusters(clusterSet1Name, 2)
assertCreatingClusters(clusterSet2Name, 3)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
// update ClusterSets
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
@@ -472,7 +488,7 @@ var _ = ginkgo.Describe("Placement", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 2)
assertCreatingClusters(clusterSet1Name, 3, "cloud", "Amazon")
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
// add a predicates
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
@@ -497,7 +513,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should schedule successfully once spec.NumberOfClusters is reduced", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
ginkgo.By("Reduce NOC of the placement")
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
@@ -515,7 +531,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should schedule successfully once spec.NumberOfClusters is increased", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 10)
assertCreatingPlacement(placementName, noc(5), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(5), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
ginkgo.By("Increase NOC of the placement")
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
@@ -533,7 +549,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should be satisfied once new clusters are added", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
// add more clusters
assertCreatingClusters(clusterSet1Name, 5)
@@ -546,7 +562,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should schedule successfully once new clusterset is bound", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
ginkgo.By("Bind one more clusterset to the placement namespace")
assertBindingClusterSet(clusterSet2Name)
@@ -560,13 +576,57 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should create multiple placementdecisions once scheduled", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 101)
assertCreatingPlacement(placementName, nil, 101, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, nil, 101, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
nod := 101
assertNumberOfDecisions(placementName, nod)
assertPlacementStatus(placementName, nod, true)
})
//TODO: should requeue when expire TolerationSeconds
ginkgo.It("Should schedule successfully with taint/toleration", func() {
addedTime := metav1.Now()
// cluster settings
clusterNames := []string{
clusterName + "-1",
clusterName + "-2",
clusterName + "-3",
clusterName + "-4",
}
assertBindingClusterSet(clusterSet1Name)
assertCreatingClustersWithNames(clusterSet1Name, clusterNames)
assertUpdatingClusterWithClusterTaint(clusterNames[0], &clusterapiv1.Taint{
Key: "key1",
Value: "value1",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: addedTime,
})
assertUpdatingClusterWithClusterTaint(clusterNames[1], &clusterapiv1.Taint{
Key: "key2",
Value: "value2",
Effect: clusterapiv1.TaintEffectNoSelect,
TimeAdded: addedTime,
})
assertUpdatingClusterWithClusterTaint(clusterNames[2], &clusterapiv1.Taint{
Key: "key3",
Value: "value3",
Effect: clusterapiv1.TaintEffectPreferNoSelect,
TimeAdded: addedTime,
})
//Checking the result of the placement
assertCreatingPlacement(placementName, noc(4), 3, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{
{
Key: "key1",
Operator: clusterapiv1beta1.TolerationOpExists,
},
})
assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2], clusterNames[3]})
})
ginkgo.It("Should schedule successfully with default SchedulePolicy", func() {
// cluster settings
clusterNames := []string{
@@ -590,7 +650,7 @@ var _ = ginkgo.Describe("Placement", func() {
}
//Checking the result of the placement
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy)
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{})
assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[1]})
})
@@ -635,7 +695,7 @@ var _ = ginkgo.Describe("Placement", func() {
}
//Checking the result of the placement
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy)
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{})
assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2]})
})
@@ -673,7 +733,7 @@ var _ = ginkgo.Describe("Placement", func() {
assertCreatingAddOnPlacementScores(clusterNames[2], "demo", "demo", 100)
//Checking the result of the placement
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy)
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{})
assertClusterNamesOfDecisions(placementName, []string{clusterNames[1], clusterNames[2]})
})
@@ -707,7 +767,7 @@ var _ = ginkgo.Describe("Placement", func() {
assertCreatingClustersWithNames(clusterSet1Name, clusterNames)
//Creating the placement
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy)
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{})
//Checking the result of the placement when no AddOnPlacementScores
assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[1]})
@@ -777,7 +837,7 @@ var _ = ginkgo.Describe("Placement", func() {
}
//Checking the result of the placement
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy)
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{})
assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2]})
ginkgo.By("Adding fake placement decisions")
@@ -833,7 +893,7 @@ var _ = ginkgo.Describe("Placement", func() {
}
//Checking the result of the placement
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy)
assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{})
assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2]})
ginkgo.By("Adding a new cluster with resources")
@@ -855,7 +915,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should re-schedule successfully once a clusterset deleted/added", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
assertNumberOfDecisions(placementName, 5)
assertPlacementStatus(placementName, 5, false)
@@ -875,7 +935,7 @@ var _ = ginkgo.Describe("Placement", func() {
ginkgo.It("Should re-schedule successfully once a clustersetbinding deleted/added", func() {
assertBindingClusterSet(clusterSet1Name)
assertCreatingClusters(clusterSet1Name, 5)
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{})
assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
assertNumberOfDecisions(placementName, 5)
assertPlacementStatus(placementName, 5, false)