From 2152581aea98edca9e3e7f96edece40f6575e5ed Mon Sep 17 00:00:00 2001 From: Qing Hao Date: Thu, 3 Mar 2022 00:04:30 +0800 Subject: [PATCH] Add filter plugin taint toleration (#63) * add plugin tainittoleration based on PR #61 Signed-off-by: haoqing0110 * cover PreferNoSelect and NoSelectIfNew case Signed-off-by: haoqing0110 --- pkg/controllers/scheduling/schedule.go | 17 +- pkg/controllers/scheduling/schedule_test.go | 106 ++++++- pkg/helpers/testing/builders.go | 16 ++ .../tainttoleration/taint_toleration.go | 136 +++++++++ .../tainttoleration/taint_toleration_test.go | 259 ++++++++++++++++++ test/integration/placement_test.go | 94 +++++-- 6 files changed, 605 insertions(+), 23 deletions(-) create mode 100644 pkg/plugins/tainttoleration/taint_toleration.go create mode 100644 pkg/plugins/tainttoleration/taint_toleration_test.go diff --git a/pkg/controllers/scheduling/schedule.go b/pkg/controllers/scheduling/schedule.go index 58632d348..99d7a56a3 100644 --- a/pkg/controllers/scheduling/schedule.go +++ b/pkg/controllers/scheduling/schedule.go @@ -18,6 +18,7 @@ import ( "open-cluster-management.io/placement/pkg/plugins/predicate" "open-cluster-management.io/placement/pkg/plugins/resource" "open-cluster-management.io/placement/pkg/plugins/steady" + "open-cluster-management.io/placement/pkg/plugins/tainttoleration" ) const ( @@ -139,6 +140,7 @@ func NewPluginScheduler(handle plugins.Handle) *pluginScheduler { handle: handle, filters: []plugins.Filter{ predicate.New(handle), + tainttoleration.New(handle), }, prioritizerWeights: defaultPrioritizerConfig, } @@ -324,10 +326,21 @@ func getPrioritizers(weights map[clusterapiv1beta1.ScoreCoordinate]int32, handle func (r *scheduleResult) FilterResults() []FilterResult { results := []FilterResult{} - for name, r := range r.filteredRecords { + + // order the FilterResults by key length + filteredRecordsKey := []string{} + for name := range r.filteredRecords { + filteredRecordsKey = append(filteredRecordsKey, name) + } + sort.SliceStable(filteredRecordsKey, func(i, j int) bool { + return len(filteredRecordsKey[i]) < len(filteredRecordsKey[j]) + }) + + // go through the FilterResults by key length + for _, name := range filteredRecordsKey { result := FilterResult{Name: name, FilteredClusters: []string{}} - for _, c := range r { + for _, c := range r.filteredRecords[name] { result.FilteredClusters = append(result.FilteredClusters, c.Name) } results = append(results, result) diff --git a/pkg/controllers/scheduling/schedule_test.go b/pkg/controllers/scheduling/schedule_test.go index d7427d133..631972356 100644 --- a/pkg/controllers/scheduling/schedule_test.go +++ b/pkg/controllers/scheduling/schedule_test.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterfake "open-cluster-management.io/api/client/cluster/clientset/versioned/fake" clusterapiv1 "open-cluster-management.io/api/cluster/v1" clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1" @@ -49,6 +50,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -86,6 +91,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -130,6 +139,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -161,6 +174,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -179,6 +196,67 @@ func TestSchedule(t *testing.T) { }, expectedUnScheduled: 0, }, + { + name: "placement with taint and toleration", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(3).AddToleration( + &clusterapiv1beta1.Toleration{ + Key: "key1", + Value: "value1", + Operator: clusterapiv1beta1.TolerationOpEqual, + }).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + testinghelpers.NewAddOnPlacementScore("cluster1", "demo").WithScore("demo", 30).Build(), + testinghelpers.NewAddOnPlacementScore("cluster2", "demo").WithScore("demo", 40).Build(), + testinghelpers.NewAddOnPlacementScore("cluster3", "demo").WithScore("demo", 50).Build(), + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterSetLabel, clusterSetName).WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + decisions: []runtime.Object{}, + expectedDecisions: []clusterapiv1beta1.ClusterDecision{ + {ClusterName: "cluster1"}, + {ClusterName: "cluster3"}, + }, + expectedFilterResult: []FilterResult{ + { + Name: "Predicate", + FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, + }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1", "cluster3"}, + }, + }, + expectedScoreResult: []PrioritizerResult{ + { + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100, "cluster3": 100}, + }, + { + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0, "cluster3": 0}, + }, + }, + expectedUnScheduled: 1, + }, { name: "placement with additive Prioritizer Policy", placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(2).WithPrioritizerPolicy("Additive").WithPrioritizerConfig("Balance", 3).WithPrioritizerConfig("ResourceAllocatableMemory", 1).WithScoreCoordinateAddOn("demo", "demo", 1).Build(), @@ -204,6 +282,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -251,6 +333,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -294,6 +380,10 @@ func TestSchedule(t *testing.T) { Name: "Predicate", FilteredClusters: []string{"cluster1", "cluster2"}, }, + { + Name: "Predicate,TaintToleration", + FilteredClusters: []string{"cluster1", "cluster2"}, + }, }, expectedScoreResult: []PrioritizerResult{ { @@ -333,6 +423,10 @@ func TestSchedule(t *testing.T) { expectedFilterResult: []FilterResult{ { Name: "Predicate", + FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, + }, + { + Name: "Predicate,TaintToleration", FilteredClusters: []string{"cluster3", "cluster1", "cluster2"}, }, }, @@ -357,18 +451,18 @@ func TestSchedule(t *testing.T) { testinghelpers.NewClusterSet(clusterSetName), testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)). - WithDecisions("cluster3", "cluster2").Build(), + WithDecisions("cluster2", "cluster3").Build(), testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 2)). - WithDecisions("cluster2", "cluster1").Build(), + WithDecisions("cluster1", "cluster2").Build(), testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). WithLabel(placementLabel, placementName). WithDecisions("cluster3").Build(), }, decisions: []runtime.Object{ testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)). - WithDecisions("cluster3", "cluster2").Build(), + WithDecisions("cluster2", "cluster3").Build(), testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 2)). - WithDecisions("cluster2", "cluster1").Build(), + WithDecisions("cluster1", "cluster2").Build(), testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). WithLabel(placementLabel, placementName). WithDecisions("cluster3").Build(), @@ -384,6 +478,10 @@ func TestSchedule(t *testing.T) { expectedFilterResult: []FilterResult{ { Name: "Predicate", + FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, + }, + { + Name: "Predicate,TaintToleration", FilteredClusters: []string{"cluster3", "cluster1", "cluster2"}, }, }, diff --git a/pkg/helpers/testing/builders.go b/pkg/helpers/testing/builders.go index 79ff98003..f7f2b93b2 100644 --- a/pkg/helpers/testing/builders.go +++ b/pkg/helpers/testing/builders.go @@ -97,6 +97,14 @@ func (b *placementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, cla return b } +func (b *placementBuilder) AddToleration(toleration *clusterapiv1beta1.Toleration) *placementBuilder { + if b.placement.Spec.Tolerations == nil { + b.placement.Spec.Tolerations = []clusterapiv1beta1.Toleration{} + } + b.placement.Spec.Tolerations = append(b.placement.Spec.Tolerations, *toleration) + return b +} + func (b *placementBuilder) WithNumOfSelectedClusters(nosc int) *placementBuilder { b.placement.Status.NumberOfSelectedClusters = int32(nosc) return b @@ -247,6 +255,14 @@ func (b *managedClusterBuilder) WithResource(resourceName clusterapiv1.ResourceN return b } +func (b *managedClusterBuilder) WithTaint(taint *clusterapiv1.Taint) *managedClusterBuilder { + if b.cluster.Spec.Taints == nil { + b.cluster.Spec.Taints = []clusterapiv1.Taint{} + } + b.cluster.Spec.Taints = append(b.cluster.Spec.Taints, *taint) + return b +} + func (b *managedClusterBuilder) Build() *clusterapiv1.ManagedCluster { return b.cluster } diff --git a/pkg/plugins/tainttoleration/taint_toleration.go b/pkg/plugins/tainttoleration/taint_toleration.go new file mode 100644 index 000000000..4c74054e3 --- /dev/null +++ b/pkg/plugins/tainttoleration/taint_toleration.go @@ -0,0 +1,136 @@ +package tainttoleration + +import ( + "context" + "errors" + "reflect" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/sets" + clusterapiv1 "open-cluster-management.io/api/cluster/v1" + clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1" + "open-cluster-management.io/placement/pkg/plugins" +) + +var _ plugins.Filter = &TaintToleration{} + +const ( + placementLabel = "cluster.open-cluster-management.io/placement" + description = "TaintToleration is a plugin that checks if a placement tolerates a managed cluster's taints" +) + +type TaintToleration struct { + handle plugins.Handle +} + +func New(handle plugins.Handle) *TaintToleration { + return &TaintToleration{ + handle: handle, + } +} + +func (p *TaintToleration) Name() string { + return reflect.TypeOf(*p).Name() +} + +func (pl *TaintToleration) Description() string { + return description +} + +func (pl *TaintToleration) Filter(ctx context.Context, placement *clusterapiv1beta1.Placement, clusters []*clusterapiv1.ManagedCluster) ([]*clusterapiv1.ManagedCluster, error) { + if len(clusters) == 0 { + return clusters, nil + } + + // do validation on each toleration and return error if necessary + for _, toleration := range placement.Spec.Tolerations { + if len(toleration.Key) == 0 && toleration.Operator != clusterapiv1beta1.TolerationOpExists { + return nil, errors.New("If the key is empty, operator must be Exists.\n") + } + if toleration.Operator == clusterapiv1beta1.TolerationOpExists && len(toleration.Value) > 0 { + return nil, errors.New("If the operator is Exists, the value should be empty.\n") + } + } + + existingDecisions := getDecisions(pl.handle, placement) + + // filter the clusters + matched := []*clusterapiv1.ManagedCluster{} + for _, cluster := range clusters { + if isClusterTolerated(cluster, placement.Spec.Tolerations, existingDecisions.Has(cluster.Name)) { + matched = append(matched, cluster) + } + } + + return matched, nil +} + +// isClusterTolerated returns true if a cluster is tolerated by the given toleration array +func isClusterTolerated(cluster *clusterapiv1.ManagedCluster, tolerations []clusterapiv1beta1.Toleration, inDecision bool) bool { + for _, taint := range cluster.Spec.Taints { + if !isTaintTolerated(taint, tolerations, inDecision) { + return false + } + } + return true +} + +// isTaintTolerated returns true if a taint is tolerated by the given toleration array +func isTaintTolerated(taint clusterapiv1.Taint, tolerations []clusterapiv1beta1.Toleration, inDecision bool) bool { + if (taint.Effect == clusterapiv1.TaintEffectPreferNoSelect) || (taint.Effect == clusterapiv1.TaintEffectNoSelectIfNew && inDecision) { + return true + } + + for _, toleration := range tolerations { + if isTolerated(taint, toleration) { + return true + } + } + return false +} + +// isTolerated returns true if a taint is tolerated by the given toleration +func isTolerated(taint clusterapiv1.Taint, toleration clusterapiv1beta1.Toleration) bool { + if len(toleration.Effect) > 0 && toleration.Effect != taint.Effect { + return false + } + + if len(toleration.Key) > 0 && toleration.Key != taint.Key { + return false + } + + switch toleration.Operator { + // empty operator means Equal + case "", clusterapiv1beta1.TolerationOpEqual: + return toleration.Value == taint.Value + case clusterapiv1beta1.TolerationOpExists: + return true + default: + return false + } +} + +func getDecisions(handle plugins.Handle, placement *clusterapiv1beta1.Placement) sets.String { + existingDecisions := sets.String{} + + // query placementdecisions with label selector + requirement, err := labels.NewRequirement(placementLabel, selection.Equals, []string{placement.Name}) + if err != nil { + return existingDecisions + } + + labelSelector := labels.NewSelector().Add(*requirement) + decisions, err := handle.DecisionLister().PlacementDecisions(placement.Namespace).List(labelSelector) + if err != nil { + return existingDecisions + } + + for _, decision := range decisions { + for _, d := range decision.Status.Decisions { + existingDecisions.Insert(d.ClusterName) + } + } + + return existingDecisions +} diff --git a/pkg/plugins/tainttoleration/taint_toleration_test.go b/pkg/plugins/tainttoleration/taint_toleration_test.go new file mode 100644 index 000000000..786210958 --- /dev/null +++ b/pkg/plugins/tainttoleration/taint_toleration_test.go @@ -0,0 +1,259 @@ +package tainttoleration + +import ( + "context" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + clusterapiv1 "open-cluster-management.io/api/cluster/v1" + clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1" + testinghelpers "open-cluster-management.io/placement/pkg/helpers/testing" +) + +func TestMatchWithClusterTaintToleration(t *testing.T) { + + cases := []struct { + name string + placement *clusterapiv1beta1.Placement + clusters []*clusterapiv1.ManagedCluster + existingDecisions []runtime.Object + expectedClusterNames []string + }{ + { + name: "taint.Effect is NoSelect and tolerations is empty", + placement: testinghelpers.NewPlacement("test", "test").Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{}, + expectedClusterNames: []string{}, + }, + { + name: "taint.Effect is NoSelect and tolerations.Operator is Equal", + placement: testinghelpers.NewPlacement("test", "test").AddToleration( + &clusterapiv1beta1.Toleration{ + Key: "key1", + Value: "value1", + Operator: clusterapiv1beta1.TolerationOpEqual, + }).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{}, + expectedClusterNames: []string{"cluster1"}, + }, + { + name: "taint.Effect is NoSelect and tolerations.Operator is Exist", + placement: testinghelpers.NewPlacement("test", "test").AddToleration( + &clusterapiv1beta1.Toleration{ + Key: "key1", + Operator: clusterapiv1beta1.TolerationOpExists, + }).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{}, + expectedClusterNames: []string{"cluster1"}, + }, + { + name: "taint.Effect is NoSelectIfNew and tolerations is empty", + placement: testinghelpers.NewPlacement("test", "test").Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelectIfNew, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelectIfNew, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster3").WithTaint( + &clusterapiv1.Taint{ + Key: "key3", + Value: "value3", + Effect: clusterapiv1.TaintEffectNoSelectIfNew, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{ + testinghelpers.NewPlacementDecision("test", "test").WithLabel(placementLabel, "test").WithDecisions("cluster2").Build(), + }, + expectedClusterNames: []string{"cluster2"}, + }, + { + name: "taint.Effect is NoSelectIfNew and tolerations is Exist", + placement: testinghelpers.NewPlacement("test", "test").AddToleration( + &clusterapiv1beta1.Toleration{ + Key: "key1", + Operator: clusterapiv1beta1.TolerationOpExists, + }).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelectIfNew, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelectIfNew, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster3").WithTaint( + &clusterapiv1.Taint{ + Key: "key3", + Value: "value3", + Effect: clusterapiv1.TaintEffectNoSelectIfNew, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{ + testinghelpers.NewPlacementDecision("test", "test").WithLabel(placementLabel, "test").WithDecisions("cluster2").Build(), + }, + expectedClusterNames: []string{"cluster1", "cluster2"}, + }, + { + name: "taint.Effect is PreferNoSelect and tolerations is Empty", + placement: testinghelpers.NewPlacement("test", "test").Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectPreferNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value2", + Effect: clusterapiv1.TaintEffectPreferNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{}, + expectedClusterNames: []string{"cluster1", "cluster2"}, + }, + { + name: "taint.Effect is PreferNoSelect and tolerations is Exist", + placement: testinghelpers.NewPlacement("test", "test").AddToleration( + &clusterapiv1beta1.Toleration{ + Key: "key1", + Operator: clusterapiv1beta1.TolerationOpExists, + }).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithTaint( + &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectPreferNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster2").WithTaint( + &clusterapiv1.Taint{ + Key: "key3", + Value: "value3", + Effect: clusterapiv1.TaintEffectPreferNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + }, + existingDecisions: []runtime.Object{}, + expectedClusterNames: []string{"cluster1", "cluster2"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + p := &TaintToleration{ + handle: testinghelpers.NewFakePluginHandle(t, nil, c.existingDecisions...), + } + clusters, err := p.Filter(context.TODO(), c.placement, c.clusters) + + if err != nil { + t.Errorf("unexpected err: %v", err) + } + + expectedClusterNames := sets.NewString(c.expectedClusterNames...) + if len(clusters) != expectedClusterNames.Len() { + t.Errorf("expected %d clusters but got %d", expectedClusterNames.Len(), len(clusters)) + } + for _, cluster := range clusters { + expectedClusterNames.Delete(cluster.Name) + } + if expectedClusterNames.Len() > 0 { + t.Errorf("expected clusters not selected: %s", strings.Join(expectedClusterNames.List(), ",")) + } + + }) + } + +} diff --git a/test/integration/placement_test.go b/test/integration/placement_test.go index 6bb46ecdf..e839723fb 100644 --- a/test/integration/placement_test.go +++ b/test/integration/placement_test.go @@ -344,6 +344,21 @@ var _ = ginkgo.Describe("Placement", func() { gomega.Expect(err).ToNot(gomega.HaveOccurred()) } + assertUpdatingClusterWithClusterTaint := func(managedClusterName string, taint *clusterapiv1.Taint) { + ginkgo.By(fmt.Sprintf("Updating ManagedClusters %s taint", managedClusterName)) + if taint == nil { + return + } + + mc, err := clusterClient.ClusterV1().ManagedClusters().Get(context.Background(), managedClusterName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + mc.Spec.Taints = append(mc.Spec.Taints, *taint) + + _, err = clusterClient.ClusterV1().ManagedClusters().Update(context.Background(), mc, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + assertDeletingCluster := func(clusterName string) { ginkgo.By(fmt.Sprintf("Delete cluster %s", clusterName)) err = clusterClient.ClusterV1().ManagedClusters().Delete(context.Background(), clusterName, metav1.DeleteOptions{}) @@ -359,7 +374,7 @@ var _ = ginkgo.Describe("Placement", func() { }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) } - assertCreatingPlacement := func(name string, noc *int32, nod int, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy) { + assertCreatingPlacement := func(name string, noc *int32, nod int, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy, tolerations []clusterapiv1beta1.Toleration) { ginkgo.By("Create placement") placement := &clusterapiv1beta1.Placement{ ObjectMeta: metav1.ObjectMeta{ @@ -369,6 +384,7 @@ var _ = ginkgo.Describe("Placement", func() { Spec: clusterapiv1beta1.PlacementSpec{ NumberOfClusters: noc, PrioritizerPolicy: prioritizerPolicy, + Tolerations: tolerations, }, } placement, err = clusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) @@ -432,7 +448,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should re-create placementdecisions successfully once placementdecisions are deleted", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) ginkgo.By("Delete placementdecisions") placementDecisions, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{ @@ -456,7 +472,7 @@ var _ = ginkgo.Describe("Placement", func() { assertBindingClusterSet(clusterSet2Name) assertCreatingClusters(clusterSet1Name, 2) assertCreatingClusters(clusterSet2Name, 3) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) // update ClusterSets placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) @@ -472,7 +488,7 @@ var _ = ginkgo.Describe("Placement", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 2) assertCreatingClusters(clusterSet1Name, 3, "cloud", "Amazon") - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) // add a predicates placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) @@ -497,7 +513,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should schedule successfully once spec.NumberOfClusters is reduced", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) ginkgo.By("Reduce NOC of the placement") placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) @@ -515,7 +531,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should schedule successfully once spec.NumberOfClusters is increased", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 10) - assertCreatingPlacement(placementName, noc(5), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(5), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) ginkgo.By("Increase NOC of the placement") placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) @@ -533,7 +549,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should be satisfied once new clusters are added", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) // add more clusters assertCreatingClusters(clusterSet1Name, 5) @@ -546,7 +562,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should schedule successfully once new clusterset is bound", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) ginkgo.By("Bind one more clusterset to the placement namespace") assertBindingClusterSet(clusterSet2Name) @@ -560,13 +576,57 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should create multiple placementdecisions once scheduled", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 101) - assertCreatingPlacement(placementName, nil, 101, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, nil, 101, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) nod := 101 assertNumberOfDecisions(placementName, nod) assertPlacementStatus(placementName, nod, true) }) + //TODO: should requeue when expire TolerationSeconds + ginkgo.It("Should schedule successfully with taint/toleration", func() { + addedTime := metav1.Now() + + // cluster settings + clusterNames := []string{ + clusterName + "-1", + clusterName + "-2", + clusterName + "-3", + clusterName + "-4", + } + + assertBindingClusterSet(clusterSet1Name) + assertCreatingClustersWithNames(clusterSet1Name, clusterNames) + + assertUpdatingClusterWithClusterTaint(clusterNames[0], &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: addedTime, + }) + assertUpdatingClusterWithClusterTaint(clusterNames[1], &clusterapiv1.Taint{ + Key: "key2", + Value: "value2", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: addedTime, + }) + assertUpdatingClusterWithClusterTaint(clusterNames[2], &clusterapiv1.Taint{ + Key: "key3", + Value: "value3", + Effect: clusterapiv1.TaintEffectPreferNoSelect, + TimeAdded: addedTime, + }) + + //Checking the result of the placement + assertCreatingPlacement(placementName, noc(4), 3, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ + { + Key: "key1", + Operator: clusterapiv1beta1.TolerationOpExists, + }, + }) + assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2], clusterNames[3]}) + }) + ginkgo.It("Should schedule successfully with default SchedulePolicy", func() { // cluster settings clusterNames := []string{ @@ -590,7 +650,7 @@ var _ = ginkgo.Describe("Placement", func() { } //Checking the result of the placement - assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy) + assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[1]}) }) @@ -635,7 +695,7 @@ var _ = ginkgo.Describe("Placement", func() { } //Checking the result of the placement - assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy) + assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2]}) }) @@ -673,7 +733,7 @@ var _ = ginkgo.Describe("Placement", func() { assertCreatingAddOnPlacementScores(clusterNames[2], "demo", "demo", 100) //Checking the result of the placement - assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy) + assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) assertClusterNamesOfDecisions(placementName, []string{clusterNames[1], clusterNames[2]}) }) @@ -707,7 +767,7 @@ var _ = ginkgo.Describe("Placement", func() { assertCreatingClustersWithNames(clusterSet1Name, clusterNames) //Creating the placement - assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy) + assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) //Checking the result of the placement when no AddOnPlacementScores assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[1]}) @@ -777,7 +837,7 @@ var _ = ginkgo.Describe("Placement", func() { } //Checking the result of the placement - assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy) + assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2]}) ginkgo.By("Adding fake placement decisions") @@ -833,7 +893,7 @@ var _ = ginkgo.Describe("Placement", func() { } //Checking the result of the placement - assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy) + assertCreatingPlacement(placementName, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) assertClusterNamesOfDecisions(placementName, []string{clusterNames[0], clusterNames[2]}) ginkgo.By("Adding a new cluster with resources") @@ -855,7 +915,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should re-schedule successfully once a clusterset deleted/added", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) assertNumberOfDecisions(placementName, 5) assertPlacementStatus(placementName, 5, false) @@ -875,7 +935,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should re-schedule successfully once a clustersetbinding deleted/added", func() { assertBindingClusterSet(clusterSet1Name) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}) + assertCreatingPlacement(placementName, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) assertNumberOfDecisions(placementName, 5) assertPlacementStatus(placementName, 5, false)