From 2d1f7c301a212e43b830c13748ad089a243be904 Mon Sep 17 00:00:00 2001 From: haoqing0110 Date: Mon, 30 Aug 2021 06:26:43 +0000 Subject: [PATCH] add prioritizer with weight Signed-off-by: haoqing0110 --- deploy/hub/placements.crd.yaml | 63 +++++- go.mod | 2 +- go.sum | 4 +- pkg/controllers/scheduling/schedule.go | 101 ++++++++-- pkg/controllers/scheduling/schedule_test.go | 190 ++++++++++++++---- .../scheduling/scheduling_controller.go | 16 +- .../scheduling/scheduling_controller_test.go | 1 + pkg/debugger/debugger.go | 8 +- pkg/debugger/debugger_test.go | 13 +- pkg/helpers/testing/builders.go | 17 ++ pkg/plugins/balance/balance.go | 7 +- pkg/plugins/predicate/predicate.go | 3 +- pkg/plugins/steady/steady.go | 7 +- vendor/modules.txt | 2 +- ...-cluster-management.io_placements.crd.yaml | 63 +++++- .../api/cluster/v1alpha1/types.go | 63 ++++++ .../cluster/v1alpha1/zz_generated.deepcopy.go | 38 ++++ .../zz_generated.swagger_doc_generated.go | 28 ++- 18 files changed, 544 insertions(+), 82 deletions(-) diff --git a/deploy/hub/placements.crd.yaml b/deploy/hub/placements.crd.yaml index 619a15a89..f119d84c2 100644 --- a/deploy/hub/placements.crd.yaml +++ b/deploy/hub/placements.crd.yaml @@ -12,7 +12,17 @@ spec: scope: Namespaced preserveUnknownFields: false versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="PlacementSatisfied")].status + name: Succeeded + type: string + - jsonPath: .status.conditions[?(@.type=="PlacementSatisfied")].reason + name: Reason + type: string + - jsonPath: .status.numberOfSelectedClusters + name: SelectedClusters + type: integer + name: v1alpha1 schema: openAPIV3Schema: description: "Placement defines a rule to select a set of ManagedClusters @@ -182,6 +192,57 @@ spec: type: object additionalProperties: type: string + prioritizerPolicy: + description: PrioritizerPolicy defines the policy of the prioritizers. + If this field is unset, then default prioritizer mode and configurations + are used. Referring to PrioritizerPolicy to see more description + about Mode and Configurations. + type: object + properties: + configurations: + type: array + items: + description: PrioritizerConfig represents the configuration + of prioritizer + type: object + required: + - name + properties: + name: + description: 'Name is the name of a prioritizer. Below are + the valid names: 1) Balance: balance the decisions among + the clusters. 2) Steady: ensure the existing decision + is stabilized. 3) ResourceRatioCPU & ResourceRatioMemory: + sort clusters based on the allocatable to capacity ratio. + 4) ResourceAllocatableCPU & ResourceAllocatableMemory: + sort clusters based on the allocatable.' + type: string + weight: + description: Weight defines the weight of prioritizer. The + value must be ranged in [0,10]. Each prioritizer will + calculate an integer score of a cluster in the range of + [-100, 100]. The final score of a cluster will be sum(weight + * prioritizer_score). A higher weight indicates that the + prioritizer weights more in the cluster selection, while + 0 weight indicate thats the prioritizer is disabled. + type: integer + format: int32 + default: 1 + maximum: 10 + minimum: 0 + mode: + description: Mode is either Exact, Additive, "" where "" is Additive + by default. In Additive mode, any prioritizer not explicitly + enumerated is enabled in its default Configurations, in which + Steady and Balance prioritizers have the weight of 1 while other + prioritizers have the weight of 0. Additive doesn't require + configuring all prioritizers. The default Configurations may + change in the future, and additional prioritization will happen. + In Exact mode, any prioritizer not explicitly enumerated is + weighted as zero. Exact requires knowing the full set of prioritizers + you want, but avoids behavior changes between releases. + type: string + default: Additive status: description: Status represents the current status of the Placement type: object diff --git a/go.mod b/go.mod index 4d0e402cb..aee14dcac 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,6 @@ require ( k8s.io/client-go v0.21.1 k8s.io/component-base v0.21.0 k8s.io/klog/v2 v2.8.0 - open-cluster-management.io/api v0.0.0-20210610125115-f57c747b84aa + open-cluster-management.io/api v0.0.0-20210908005819-815ac23c7308 sigs.k8s.io/controller-runtime v0.8.3 ) diff --git a/go.sum b/go.sum index 73f6e7c3c..06e07be58 100644 --- a/go.sum +++ b/go.sum @@ -959,8 +959,8 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -open-cluster-management.io/api v0.0.0-20210610125115-f57c747b84aa h1:Jw/Jo3Om1xXgdcn6xDfyREkc48nUcW/DqhMHffLCCww= -open-cluster-management.io/api v0.0.0-20210610125115-f57c747b84aa/go.mod h1:9qiA5h/8kvPQnJEOlAPHVjRO9a1jCmDhGzvgMBvXEaE= +open-cluster-management.io/api v0.0.0-20210908005819-815ac23c7308 h1:itWWudWVTqviZ2H2Arb1yTQ7NMUSPVDmWiVOpXiblOM= +open-cluster-management.io/api v0.0.0-20210908005819-815ac23c7308/go.mod h1:9qiA5h/8kvPQnJEOlAPHVjRO9a1jCmDhGzvgMBvXEaE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/controllers/scheduling/schedule.go b/pkg/controllers/scheduling/schedule.go index 451476344..24a174d13 100644 --- a/pkg/controllers/scheduling/schedule.go +++ b/pkg/controllers/scheduling/schedule.go @@ -2,6 +2,7 @@ package scheduling import ( "context" + "fmt" "sort" "strings" @@ -16,7 +17,8 @@ import ( "open-cluster-management.io/placement/pkg/plugins/steady" ) -type PrioritizeSore map[string]int64 +// PrioritizerScore defines the score for each cluster +type PrioritizerScore map[string]int64 // Scheduler is an interface for scheduler, it returs the scheduler results type Scheduler interface { @@ -31,8 +33,11 @@ type ScheduleResult interface { // FilterResults returns results for each filter FilterResults() []FilterResult - // PriorizeResults returns results for each prioritizer - PriorizeResults() []PriorizeResult + // PrioritizerResults returns results for each prioritizer + PrioritizerResults() []PrioritizerResult + + // PrioritizerScores returns total score for each cluster + PrioritizerScores() PrioritizerScore // Decisions returns the decisions of the schedule Decisions() []clusterapiv1alpha1.ClusterDecision @@ -46,9 +51,12 @@ type FilterResult struct { FilteredClusters []string `json:"filteredClusters"` } -type PriorizeResult struct { - Name string `json:"name"` - Scores PrioritizeSore `json:"scores"` +// PrioritizerResult defines the result of one prioritizer, +// include name, weight, and score of each cluster. +type PrioritizerResult struct { + Name string `json:"name"` + Weight int32 `json:"weight"` + Scores PrioritizerScore `json:"scores"` } // ScheduleResult is the result for a certain schedule. @@ -58,7 +66,8 @@ type scheduleResult struct { unscheduledDecisions int filteredRecords map[string][]*clusterapiv1.ManagedCluster - scoreRecords []PriorizeResult + scoreRecords []PrioritizerResult + scoreSum PrioritizerScore } type schedulerHandler struct { @@ -89,9 +98,18 @@ func (s *schedulerHandler) ClusterClient() clusterclient.Interface { return s.clusterClient } +// Initialize the default prioritizer weight. +// Balane and Steady weight 1, others weight 0. +// The default weight can be replaced by each placement's PrioritizerConfigs. +var defaultPrioritizerConfig = map[string]int32{ + "Balance": 1, + "Steady": 1, +} + type pluginScheduler struct { - filters []plugins.Filter - prioritizers []plugins.Prioritizer + filters []plugins.Filter + prioritizers []plugins.Prioritizer + prioritizerWeights map[string]int32 } func NewPluginScheduler(handle plugins.Handle) *pluginScheduler { @@ -103,6 +121,7 @@ func NewPluginScheduler(handle plugins.Handle) *pluginScheduler { balance.New(handle), steady.New(handle), }, + prioritizerWeights: defaultPrioritizerConfig, } } @@ -116,7 +135,7 @@ func (s *pluginScheduler) Schedule( results := &scheduleResult{ filteredRecords: map[string][]*clusterapiv1.ManagedCluster{}, - scoreRecords: []PriorizeResult{}, + scoreRecords: []PrioritizerResult{}, } // filter clusters @@ -134,27 +153,38 @@ func (s *pluginScheduler) Schedule( results.filteredRecords[strings.Join(filterPipline, ",")] = filtered } + // get weight for each prioritizers + weights, err := getWeights(s.prioritizerWeights, placement) + if err != nil { + return nil, err + } // score clusters - scoreSum := PrioritizeSore{} + scoreSum := PrioritizerScore{} for _, cluster := range filtered { scoreSum[cluster.Name] = 0 } for _, p := range s.prioritizers { + // If weight is 0 (set to 0 or not defined in map), skip Score(). + weight := weights[p.Name()] + if weight == 0 { + results.scoreRecords = append(results.scoreRecords, PrioritizerResult{Name: p.Name(), Weight: weight, Scores: nil}) + continue + } + score, err := p.Score(ctx, placement, filtered) if err != nil { return nil, err } - results.scoreRecords = append(results.scoreRecords, PriorizeResult{Name: p.Name(), Scores: score}) + results.scoreRecords = append(results.scoreRecords, PrioritizerResult{Name: p.Name(), Weight: weight, Scores: score}) - // TODO we currently weigh each prioritizer as equal. We should consider - // importance factor for each priotizer when caculating the final score. - // Since currently balance plugin has a score range of +/- 100 while the score range of - // balacne is 0/100, the balance plugin will trigger the reschedule for rebalancing when - // a cluster's decision count is larger than average. + // The final score is a sum of each prioritizer score * weight. + // A higher weight indicates that the prioritizer weights more in the cluster selection, + // while 0 weight indicate thats the prioritizer is disabled. for name, val := range score { - scoreSum[name] = scoreSum[name] + val + scoreSum[name] = scoreSum[name] + val*int64(weight) } + } // Sort cluster by score, if score is equal, sort by name @@ -167,6 +197,7 @@ func (s *pluginScheduler) Schedule( }) results.feasibleClusters = filtered + results.scoreSum = scoreSum // select clusters and generate cluster decisions // TODO: sort the feasible clusters and make sure the selection stable @@ -204,6 +235,34 @@ func selectClusters(placement *clusterapiv1alpha1.Placement, clusters []*cluster return decisions } +// Get prioritizer weight for the placement. +// In Additive and "" mode, will override defaultWeight with what placement has defined and return. +// In Exact mode, will return the name and weight defined in placement. +func getWeights(defaultWeight map[string]int32, placement *clusterapiv1alpha1.Placement) (map[string]int32, error) { + mode := placement.Spec.PrioritizerPolicy.Mode + switch { + case mode == clusterapiv1alpha1.PrioritizerPolicyModeExact: + return mergeWeights(nil, placement.Spec.PrioritizerPolicy.Configurations), nil + case mode == clusterapiv1alpha1.PrioritizerPolicyModeAdditive || mode == "": + return mergeWeights(defaultWeight, placement.Spec.PrioritizerPolicy.Configurations), nil + default: + return nil, fmt.Errorf("incorrect prioritizer policy mode: %s", mode) + } +} + +func mergeWeights(defaultWeight map[string]int32, customizedWeight []clusterapiv1alpha1.PrioritizerConfig) map[string]int32 { + weights := make(map[string]int32) + // copy the default weight + for k, v := range defaultWeight { + weights[k] = v + } + // override the default weight + for _, c := range customizedWeight { + weights[c.Name] = c.Weight + } + return weights +} + func (r *scheduleResult) FilterResults() []FilterResult { results := []FilterResult{} for name, r := range r.filteredRecords { @@ -217,10 +276,14 @@ func (r *scheduleResult) FilterResults() []FilterResult { return results } -func (r *scheduleResult) PriorizeResults() []PriorizeResult { +func (r *scheduleResult) PrioritizerResults() []PrioritizerResult { return r.scoreRecords } +func (r *scheduleResult) PrioritizerScores() PrioritizerScore { + return r.scoreSum +} + func (r *scheduleResult) Decisions() []clusterapiv1alpha1.ClusterDecision { return r.scheduledDecisions } diff --git a/pkg/controllers/scheduling/schedule_test.go b/pkg/controllers/scheduling/schedule_test.go index cfc9a3ce8..11b193c48 100644 --- a/pkg/controllers/scheduling/schedule_test.go +++ b/pkg/controllers/scheduling/schedule_test.go @@ -27,7 +27,7 @@ func TestSchedule(t *testing.T) { clusters []*clusterapiv1.ManagedCluster decisions []runtime.Object expectedFilterResult []FilterResult - expectedScoreResult []PriorizeResult + expectedScoreResult []PrioritizerResult expectedDecisions []clusterapiv1alpha1.ClusterDecision expectedUnScheduled int }{ @@ -44,18 +44,20 @@ func TestSchedule(t *testing.T) { }, expectedFilterResult: []FilterResult{ { - Name: "predicate", + Name: "Predicate", FilteredClusters: []string{"cluster1"}, }, }, - expectedScoreResult: []PriorizeResult{ + expectedScoreResult: []PrioritizerResult{ { - Name: "balance", - Scores: PrioritizeSore{"cluster1": 100}, + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100}, }, { - Name: "steady", - Scores: PrioritizeSore{"cluster1": 0}, + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0}, }, }, clusters: []*clusterapiv1.ManagedCluster{ @@ -79,18 +81,20 @@ func TestSchedule(t *testing.T) { }, expectedFilterResult: []FilterResult{ { - Name: "predicate", + Name: "Predicate", FilteredClusters: []string{"cluster1"}, }, }, - expectedScoreResult: []PriorizeResult{ + expectedScoreResult: []PrioritizerResult{ { - Name: "balance", - Scores: PrioritizeSore{"cluster1": 100}, + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100}, }, { - Name: "steady", - Scores: PrioritizeSore{"cluster1": 0}, + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0}, }, }, expectedUnScheduled: 2, @@ -121,22 +125,126 @@ func TestSchedule(t *testing.T) { }, expectedFilterResult: []FilterResult{ { - Name: "predicate", + Name: "Predicate", FilteredClusters: []string{"cluster1", "cluster2", "cluster3"}, }, }, - expectedScoreResult: []PriorizeResult{ + expectedScoreResult: []PrioritizerResult{ { - Name: "balance", - Scores: PrioritizeSore{"cluster1": 100, "cluster2": 100, "cluster3": 100}, + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100, "cluster2": 100, "cluster3": 100}, }, { - Name: "steady", - Scores: PrioritizeSore{"cluster1": 100, "cluster2": 100, "cluster3": 0}, + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100, "cluster2": 100, "cluster3": 0}, }, }, expectedUnScheduled: 0, }, + { + name: "placement with empty Prioritizer Policy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithPrioritizerPolicy("").Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + }, + decisions: []runtime.Object{}, + expectedDecisions: []clusterapiv1alpha1.ClusterDecision{ + {ClusterName: "cluster1"}, + }, + expectedFilterResult: []FilterResult{ + { + Name: "Predicate", + FilteredClusters: []string{"cluster1"}, + }, + }, + expectedScoreResult: []PrioritizerResult{ + { + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100}, + }, + { + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0}, + }, + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + expectedUnScheduled: 0, + }, + { + name: "placement with additive Prioritizer Policy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithPrioritizerPolicy("Additive").WithPrioritizerConfig("Steady", 3).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + }, + decisions: []runtime.Object{}, + expectedDecisions: []clusterapiv1alpha1.ClusterDecision{ + {ClusterName: "cluster1"}, + }, + expectedFilterResult: []FilterResult{ + { + Name: "Predicate", + FilteredClusters: []string{"cluster1"}, + }, + }, + expectedScoreResult: []PrioritizerResult{ + { + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100}, + }, + { + Name: "Steady", + Weight: 3, + Scores: PrioritizerScore{"cluster1": 0}, + }, + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + expectedUnScheduled: 0, + }, + { + name: "placement with exact Prioritizer Policy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithPrioritizerPolicy("Exact").WithPrioritizerConfig("Steady", 3).Build(), + initObjs: []runtime.Object{ + testinghelpers.NewClusterSet(clusterSetName), + testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), + }, + decisions: []runtime.Object{}, + expectedDecisions: []clusterapiv1alpha1.ClusterDecision{ + {ClusterName: "cluster1"}, + }, + expectedFilterResult: []FilterResult{ + { + Name: "Predicate", + FilteredClusters: []string{"cluster1"}, + }, + }, + expectedScoreResult: []PrioritizerResult{ + { + Name: "Balance", + Weight: 0, + Scores: nil, + }, + { + Name: "Steady", + Weight: 3, + Scores: PrioritizerScore{"cluster1": 0}, + }, + }, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, clusterSetName).Build(), + }, + expectedUnScheduled: 0, + }, { name: "placement with part of decisions scheduled", placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithNOC(4).Build(), @@ -162,18 +270,20 @@ func TestSchedule(t *testing.T) { }, expectedFilterResult: []FilterResult{ { - Name: "predicate", + Name: "Predicate", FilteredClusters: []string{"cluster1", "cluster2"}, }, }, - expectedScoreResult: []PriorizeResult{ + expectedScoreResult: []PrioritizerResult{ { - Name: "balance", - Scores: PrioritizeSore{"cluster1": 100, "cluster2": 100}, + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100, "cluster2": 100}, }, { - Name: "steady", - Scores: PrioritizeSore{"cluster1": 100, "cluster2": 0}, + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 100, "cluster2": 0}, }, }, expectedUnScheduled: 2, @@ -201,18 +311,20 @@ func TestSchedule(t *testing.T) { }, expectedFilterResult: []FilterResult{ { - Name: "predicate", + Name: "Predicate", FilteredClusters: []string{"cluster3", "cluster1", "cluster2"}, }, }, - expectedScoreResult: []PriorizeResult{ + expectedScoreResult: []PrioritizerResult{ { - Name: "balance", - Scores: PrioritizeSore{"cluster1": -100, "cluster2": -100, "cluster3": 100}, + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": -100, "cluster2": -100, "cluster3": 100}, }, { - Name: "steady", - Scores: PrioritizeSore{"cluster1": 0, "cluster2": 0, "cluster3": 0}, + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0, "cluster2": 0, "cluster3": 0}, }, }, expectedUnScheduled: 0, @@ -250,18 +362,20 @@ func TestSchedule(t *testing.T) { }, expectedFilterResult: []FilterResult{ { - Name: "predicate", + Name: "Predicate", FilteredClusters: []string{"cluster3", "cluster1", "cluster2"}, }, }, - expectedScoreResult: []PriorizeResult{ + expectedScoreResult: []PrioritizerResult{ { - Name: "balance", - Scores: PrioritizeSore{"cluster1": 0, "cluster2": -100, "cluster3": 0}, + Name: "Balance", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0, "cluster2": -100, "cluster3": 0}, }, { - Name: "steady", - Scores: PrioritizeSore{"cluster1": 0, "cluster2": 0, "cluster3": 100}, + Name: "Steady", + Weight: 1, + Scores: PrioritizerScore{"cluster1": 0, "cluster2": 0, "cluster3": 100}, }, }, expectedUnScheduled: 0, @@ -294,7 +408,7 @@ func TestSchedule(t *testing.T) { t.Errorf("expected filter results %v, but got %v", string(expected), string(actual)) } - actual, _ = json.Marshal(result.PriorizeResults()) + actual, _ = json.Marshal(result.PrioritizerResults()) expected, _ = json.Marshal(c.expectedScoreResult) if !reflect.DeepEqual(actual, expected) { t.Errorf("expected score results %v, but got %v", string(expected), string(actual)) diff --git a/pkg/controllers/scheduling/scheduling_controller.go b/pkg/controllers/scheduling/scheduling_controller.go index c7ba12de5..6d869d4f4 100644 --- a/pkg/controllers/scheduling/scheduling_controller.go +++ b/pkg/controllers/scheduling/scheduling_controller.go @@ -192,7 +192,7 @@ func (c *schedulingController) sync(ctx context.Context, syncCtx factory.SyncCon return err } - err = c.bind(ctx, placement, scheduleResult.Decisions()) + err = c.bind(ctx, placement, scheduleResult.Decisions(), scheduleResult.PrioritizerScores()) if err != nil { return err } @@ -337,6 +337,7 @@ func (c *schedulingController) bind( ctx context.Context, placement *clusterapiv1alpha1.Placement, clusterDecisions []clusterapiv1alpha1.ClusterDecision, + clusterScores PrioritizerScore, ) error { // sort clusterdecisions by cluster name sort.SliceStable(clusterDecisions, func(i, j int) bool { @@ -368,7 +369,7 @@ func (c *schedulingController) bind( placementDecisionName := fmt.Sprintf("%s-decision-%d", placement.Name, index+1) placementDecisionNames.Insert(placementDecisionName) err := c.createOrUpdatePlacementDecision( - ctx, placement, placementDecisionName, decisionSlice) + ctx, placement, placementDecisionName, decisionSlice, clusterScores) if err != nil { errs = append(errs, err) } @@ -417,6 +418,7 @@ func (c *schedulingController) createOrUpdatePlacementDecision( placement *clusterapiv1alpha1.Placement, placementDecisionName string, clusterDecisions []clusterapiv1alpha1.ClusterDecision, + clusterScores PrioritizerScore, ) error { if len(clusterDecisions) > maxNumOfClusterDecisions { return fmt.Errorf("the number of clusterdecisions %q exceeds the max limitation %q", len(clusterDecisions), maxNumOfClusterDecisions) @@ -470,5 +472,15 @@ func (c *schedulingController) createOrUpdatePlacementDecision( "DecisionUpdate", "DecisionUpdated", "Decision %s is updated with placement %s in namespace %s", placementDecision.Name, placement.Name, placement.Namespace) + // update the event with prioritizer score. + scoreStr := "" + for k, v := range clusterScores { + scoreStr += fmt.Sprintf("%s:%d ", k, v) + } + c.recorder.Eventf( + placement, placementDecision, corev1.EventTypeNormal, + "ScoreUpdate", "ScoreUpdated", + scoreStr) + return nil } diff --git a/pkg/controllers/scheduling/scheduling_controller_test.go b/pkg/controllers/scheduling/scheduling_controller_test.go index 1d78596f2..442405421 100644 --- a/pkg/controllers/scheduling/scheduling_controller_test.go +++ b/pkg/controllers/scheduling/scheduling_controller_test.go @@ -560,6 +560,7 @@ func TestBind(t *testing.T) { context.TODO(), testinghelpers.NewPlacement(placementNamespace, placementName).Build(), c.clusterDecisions, + nil, ) if err != nil { t.Errorf("unexpected err: %v", err) diff --git a/pkg/debugger/debugger.go b/pkg/debugger/debugger.go index a15dfee3b..05ff49566 100644 --- a/pkg/debugger/debugger.go +++ b/pkg/debugger/debugger.go @@ -25,9 +25,9 @@ type Debugger struct { // DebugResult is the result returned by debugger type DebugResult struct { - FilterResults []scheduling.FilterResult `json:"filteredPiplieResults,omitempty"` - PrioritizeResults []scheduling.PriorizeResult `json:"prioritizeResults,omitempty"` - Error string `json:"error,omitempty"` + FilterResults []scheduling.FilterResult `json:"filteredPiplieResults,omitempty"` + PrioritizeResults []scheduling.PrioritizerResult `json:"prioritizeResults,omitempty"` + Error string `json:"error,omitempty"` } func NewDebugger( @@ -62,7 +62,7 @@ func (d *Debugger) Handler(w http.ResponseWriter, r *http.Request) { scheduleResults, _ := d.scheduler.Schedule(r.Context(), placement, clusters) - result := DebugResult{FilterResults: scheduleResults.FilterResults(), PrioritizeResults: scheduleResults.PriorizeResults()} + result := DebugResult{FilterResults: scheduleResults.FilterResults(), PrioritizeResults: scheduleResults.PrioritizerResults()} resultByte, _ := json.Marshal(result) diff --git a/pkg/debugger/debugger_test.go b/pkg/debugger/debugger_test.go index 33d1ab5c2..6acffa6b2 100644 --- a/pkg/debugger/debugger_test.go +++ b/pkg/debugger/debugger_test.go @@ -24,17 +24,22 @@ type testScheduler struct { type testResult struct { filterResults []scheduling.FilterResult - prioritizeResults []scheduling.PriorizeResult + prioritizeResults []scheduling.PrioritizerResult + scoreSum scheduling.PrioritizerScore } func (r *testResult) FilterResults() []scheduling.FilterResult { return r.filterResults } -func (r *testResult) PriorizeResults() []scheduling.PriorizeResult { +func (r *testResult) PrioritizerResults() []scheduling.PrioritizerResult { return r.prioritizeResults } +func (r *testResult) PrioritizerScores() scheduling.PrioritizerScore { + return r.scoreSum +} + func (r *testResult) Decisions() []clusterapiv1alpha1.ClusterDecision { return []clusterapiv1alpha1.ClusterDecision{} } @@ -59,7 +64,7 @@ func TestDebugger(t *testing.T) { name string initObjs []runtime.Object filterResults []scheduling.FilterResult - prioritizeResults []scheduling.PriorizeResult + prioritizeResults []scheduling.PrioritizerResult key string }{ { @@ -70,7 +75,7 @@ func TestDebugger(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").Build(), }, filterResults: []scheduling.FilterResult{{Name: "filter1", FilteredClusters: []string{"cluster1", "cluster2"}}}, - prioritizeResults: []scheduling.PriorizeResult{{Name: "prioritize1", Scores: map[string]int64{"cluster1": 100, "cluster2": 0}}}, + prioritizeResults: []scheduling.PrioritizerResult{{Name: "prioritize1", Scores: map[string]int64{"cluster1": 100, "cluster2": 0}}}, key: placementNamespace + "/" + placementName, }, } diff --git a/pkg/helpers/testing/builders.go b/pkg/helpers/testing/builders.go index 9bb8f07b2..ab7e70913 100644 --- a/pkg/helpers/testing/builders.go +++ b/pkg/helpers/testing/builders.go @@ -36,6 +36,23 @@ func (b *placementBuilder) WithNOC(noc int32) *placementBuilder { return b } +func (b *placementBuilder) WithPrioritizerPolicy(mode clusterapiv1alpha1.PrioritizerPolicyModeType) *placementBuilder { + b.placement.Spec.PrioritizerPolicy = clusterapiv1alpha1.PrioritizerPolicy{ + Mode: mode, + } + return b +} + +func (b *placementBuilder) WithPrioritizerConfig(name string, weight int32) *placementBuilder { + if b.placement.Spec.PrioritizerPolicy.Configurations == nil { + b.placement.Spec.PrioritizerPolicy.Configurations = []clusterapiv1alpha1.PrioritizerConfig{} + } + if len(name) > 0 { + b.placement.Spec.PrioritizerPolicy.Configurations = append(b.placement.Spec.PrioritizerPolicy.Configurations, clusterapiv1alpha1.PrioritizerConfig{Name: name, Weight: weight}) + } + return b +} + func (b *placementBuilder) WithClusterSets(clusterSets ...string) *placementBuilder { b.placement.Spec.ClusterSets = clusterSets return b diff --git a/pkg/plugins/balance/balance.go b/pkg/plugins/balance/balance.go index bd9f9cd99..a4ef576ca 100644 --- a/pkg/plugins/balance/balance.go +++ b/pkg/plugins/balance/balance.go @@ -2,6 +2,7 @@ package balance import ( "context" + "reflect" "k8s.io/apimachinery/pkg/labels" clusterapiv1 "open-cluster-management.io/api/cluster/v1" @@ -25,11 +26,13 @@ type Balance struct { } func New(handle plugins.Handle) *Balance { - return &Balance{handle: handle} + return &Balance{ + handle: handle, + } } func (b *Balance) Name() string { - return "balance" + return reflect.TypeOf(*b).Name() } func (b *Balance) Description() string { diff --git a/pkg/plugins/predicate/predicate.go b/pkg/plugins/predicate/predicate.go index da6e5c436..2eac48f04 100644 --- a/pkg/plugins/predicate/predicate.go +++ b/pkg/plugins/predicate/predicate.go @@ -2,6 +2,7 @@ package predicate import ( "context" + "reflect" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -27,7 +28,7 @@ func New(handle plugins.Handle) *Predicate { } func (p *Predicate) Name() string { - return "predicate" + return reflect.TypeOf(*p).Name() } func (p *Predicate) Description() string { diff --git a/pkg/plugins/steady/steady.go b/pkg/plugins/steady/steady.go index 1ca8f0e25..2241c2a20 100644 --- a/pkg/plugins/steady/steady.go +++ b/pkg/plugins/steady/steady.go @@ -2,6 +2,7 @@ package steady import ( "context" + "reflect" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" @@ -27,11 +28,13 @@ type Steady struct { } func New(handle plugins.Handle) *Steady { - return &Steady{handle: handle} + return &Steady{ + handle: handle, + } } func (s *Steady) Name() string { - return "steady" + return reflect.TypeOf(*s).Name() } func (s *Steady) Description() string { diff --git a/vendor/modules.txt b/vendor/modules.txt index 3246fec28..31ac1b76c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -860,7 +860,7 @@ k8s.io/utils/net k8s.io/utils/path k8s.io/utils/pointer k8s.io/utils/trace -# open-cluster-management.io/api v0.0.0-20210610125115-f57c747b84aa +# open-cluster-management.io/api v0.0.0-20210908005819-815ac23c7308 ## explicit open-cluster-management.io/api/client/cluster/clientset/versioned open-cluster-management.io/api/client/cluster/clientset/versioned/fake diff --git a/vendor/open-cluster-management.io/api/cluster/v1alpha1/0000_03_clusters.open-cluster-management.io_placements.crd.yaml b/vendor/open-cluster-management.io/api/cluster/v1alpha1/0000_03_clusters.open-cluster-management.io_placements.crd.yaml index 619a15a89..f119d84c2 100644 --- a/vendor/open-cluster-management.io/api/cluster/v1alpha1/0000_03_clusters.open-cluster-management.io_placements.crd.yaml +++ b/vendor/open-cluster-management.io/api/cluster/v1alpha1/0000_03_clusters.open-cluster-management.io_placements.crd.yaml @@ -12,7 +12,17 @@ spec: scope: Namespaced preserveUnknownFields: false versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="PlacementSatisfied")].status + name: Succeeded + type: string + - jsonPath: .status.conditions[?(@.type=="PlacementSatisfied")].reason + name: Reason + type: string + - jsonPath: .status.numberOfSelectedClusters + name: SelectedClusters + type: integer + name: v1alpha1 schema: openAPIV3Schema: description: "Placement defines a rule to select a set of ManagedClusters @@ -182,6 +192,57 @@ spec: type: object additionalProperties: type: string + prioritizerPolicy: + description: PrioritizerPolicy defines the policy of the prioritizers. + If this field is unset, then default prioritizer mode and configurations + are used. Referring to PrioritizerPolicy to see more description + about Mode and Configurations. + type: object + properties: + configurations: + type: array + items: + description: PrioritizerConfig represents the configuration + of prioritizer + type: object + required: + - name + properties: + name: + description: 'Name is the name of a prioritizer. Below are + the valid names: 1) Balance: balance the decisions among + the clusters. 2) Steady: ensure the existing decision + is stabilized. 3) ResourceRatioCPU & ResourceRatioMemory: + sort clusters based on the allocatable to capacity ratio. + 4) ResourceAllocatableCPU & ResourceAllocatableMemory: + sort clusters based on the allocatable.' + type: string + weight: + description: Weight defines the weight of prioritizer. The + value must be ranged in [0,10]. Each prioritizer will + calculate an integer score of a cluster in the range of + [-100, 100]. The final score of a cluster will be sum(weight + * prioritizer_score). A higher weight indicates that the + prioritizer weights more in the cluster selection, while + 0 weight indicate thats the prioritizer is disabled. + type: integer + format: int32 + default: 1 + maximum: 10 + minimum: 0 + mode: + description: Mode is either Exact, Additive, "" where "" is Additive + by default. In Additive mode, any prioritizer not explicitly + enumerated is enabled in its default Configurations, in which + Steady and Balance prioritizers have the weight of 1 while other + prioritizers have the weight of 0. Additive doesn't require + configuring all prioritizers. The default Configurations may + change in the future, and additional prioritization will happen. + In Exact mode, any prioritizer not explicitly enumerated is + weighted as zero. Exact requires knowing the full set of prioritizers + you want, but avoids behavior changes between releases. + type: string + default: Additive status: description: Status represents the current status of the Placement type: object diff --git a/vendor/open-cluster-management.io/api/cluster/v1alpha1/types.go b/vendor/open-cluster-management.io/api/cluster/v1alpha1/types.go index 9c9dc0a0d..789208987 100644 --- a/vendor/open-cluster-management.io/api/cluster/v1alpha1/types.go +++ b/vendor/open-cluster-management.io/api/cluster/v1alpha1/types.go @@ -164,6 +164,9 @@ var ReservedClusterClaimNames = [...]string{ // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:resource:scope="Namespaced" // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Succeeded",type="string",JSONPath=".status.conditions[?(@.type==\"PlacementSatisfied\")].status" +// +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=".status.conditions[?(@.type==\"PlacementSatisfied\")].reason" +// +kubebuilder:printcolumn:name="SelectedClusters",type="integer",JSONPath=".status.numberOfSelectedClusters" // Placement defines a rule to select a set of ManagedClusters from the ManagedClusterSets bound // to the placement namespace. @@ -230,6 +233,12 @@ type PlacementSpec struct { // Predicates represent a slice of predicates to select ManagedClusters. The predicates are ORed. // +optional Predicates []ClusterPredicate `json:"predicates,omitempty"` + + // PrioritizerPolicy defines the policy of the prioritizers. + // If this field is unset, then default prioritizer mode and configurations are used. + // Referring to PrioritizerPolicy to see more description about Mode and Configurations. + // +optional + PrioritizerPolicy PrioritizerPolicy `json:"prioritizerPolicy"` } // ClusterPredicate represents a predicate to select ManagedClusters. @@ -264,6 +273,55 @@ type ClusterClaimSelector struct { MatchExpressions []metav1.LabelSelectorRequirement `json:"matchExpressions,omitempty"` } +// PrioritizerPolicy represents the policy of prioritizer +type PrioritizerPolicy struct { + // Mode is either Exact, Additive, "" where "" is Additive by default. + // In Additive mode, any prioritizer not explicitly enumerated is enabled in its default Configurations, + // in which Steady and Balance prioritizers have the weight of 1 while other prioritizers have the weight of 0. + // Additive doesn't require configuring all prioritizers. The default Configurations may change in the future, + // and additional prioritization will happen. + // In Exact mode, any prioritizer not explicitly enumerated is weighted as zero. + // Exact requires knowing the full set of prioritizers you want, but avoids behavior changes between releases. + // +kubebuilder:default:=Additive + // +optional + Mode PrioritizerPolicyModeType `json:"mode,omitempty"` + + // +optional + Configurations []PrioritizerConfig `json:"configurations,omitempty"` +} + +// PrioritizerPolicyModeType represents the type of PrioritizerPolicy.Mode +type PrioritizerPolicyModeType string + +const ( + // Valid PrioritizerPolicyModeType value is Exact, Additive. + PrioritizerPolicyModeAdditive PrioritizerPolicyModeType = "Additive" + PrioritizerPolicyModeExact PrioritizerPolicyModeType = "Exact" +) + +// PrioritizerConfig represents the configuration of prioritizer +type PrioritizerConfig struct { + // Name is the name of a prioritizer. Below are the valid names: + // 1) Balance: balance the decisions among the clusters. + // 2) Steady: ensure the existing decision is stabilized. + // 3) ResourceRatioCPU & ResourceRatioMemory: sort clusters based on the allocatable to capacity ratio. + // 4) ResourceAllocatableCPU & ResourceAllocatableMemory: sort clusters based on the allocatable. + // +kubebuilder:validation:Required + // +required + Name string `json:"name"` + + // Weight defines the weight of prioritizer. The value must be ranged in [0,10]. + // Each prioritizer will calculate an integer score of a cluster in the range of [-100, 100]. + // The final score of a cluster will be sum(weight * prioritizer_score). + // A higher weight indicates that the prioritizer weights more in the cluster selection, + // while 0 weight indicate thats the prioritizer is disabled. + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=10 + // +kubebuilder:default:=1 + // +optional + Weight int32 `json:"weight,omitempty"` +} + type PlacementStatus struct { // NumberOfSelectedClusters represents the number of selected ManagedClusters // +optional @@ -319,6 +377,11 @@ type PlacementDecision struct { Status PlacementDecisionStatus `json:"status,omitempty"` } +//The placementDecsion label name holding the placement name +const ( + PlacementLabel string = "cluster.open-cluster-management.io/placement" +) + // PlacementDecisionStatus represents the current status of the PlacementDecision. type PlacementDecisionStatus struct { // Decisions is a slice of decisions according to a placement diff --git a/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.deepcopy.go b/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.deepcopy.go index 7344a96e6..b39e35467 100644 --- a/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.deepcopy.go @@ -497,6 +497,7 @@ func (in *PlacementSpec) DeepCopyInto(out *PlacementSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.PrioritizerPolicy.DeepCopyInto(&out.PrioritizerPolicy) return } @@ -532,3 +533,40 @@ func (in *PlacementStatus) DeepCopy() *PlacementStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrioritizerConfig) DeepCopyInto(out *PrioritizerConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrioritizerConfig. +func (in *PrioritizerConfig) DeepCopy() *PrioritizerConfig { + if in == nil { + return nil + } + out := new(PrioritizerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrioritizerPolicy) DeepCopyInto(out *PrioritizerPolicy) { + *out = *in + if in.Configurations != nil { + in, out := &in.Configurations, &out.Configurations + *out = make([]PrioritizerConfig, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrioritizerPolicy. +func (in *PrioritizerPolicy) DeepCopy() *PrioritizerPolicy { + if in == nil { + return nil + } + out := new(PrioritizerPolicy) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.swagger_doc_generated.go b/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.swagger_doc_generated.go index 79653cb61..52c79b5ae 100644 --- a/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/vendor/open-cluster-management.io/api/cluster/v1alpha1/zz_generated.swagger_doc_generated.go @@ -190,10 +190,11 @@ func (PlacementList) SwaggerDoc() map[string]string { } var map_PlacementSpec = map[string]string{ - "": "PlacementSpec defines the attributes of Placement. An empty PlacementSpec selects all ManagedClusters from the ManagedClusterSets bound to the placement namespace. The containing fields are ANDed.", - "clusterSets": "ClusterSets represent the ManagedClusterSets from which the ManagedClusters are selected. If the slice is empty, ManagedClusters will be selected from the ManagedClusterSets bound to the placement namespace, otherwise ManagedClusters will be selected from the intersection of this slice and the ManagedClusterSets bound to the placement namespace.", - "numberOfClusters": "NumberOfClusters represents the desired number of ManagedClusters to be selected which meet the placement requirements. 1) If not specified, all ManagedClusters which meet the placement requirements (including ClusterSets,\n and Predicates) will be selected;\n2) Otherwise if the nubmer of ManagedClusters meet the placement requirements is larger than\n NumberOfClusters, a random subset with desired number of ManagedClusters will be selected;\n3) If the nubmer of ManagedClusters meet the placement requirements is equal to NumberOfClusters,\n all of them will be selected;\n4) If the nubmer of ManagedClusters meet the placement requirements is less than NumberOfClusters,\n all of them will be selected, and the status of condition `PlacementConditionSatisfied` will be\n set to false;", - "predicates": "Predicates represent a slice of predicates to select ManagedClusters. The predicates are ORed.", + "": "PlacementSpec defines the attributes of Placement. An empty PlacementSpec selects all ManagedClusters from the ManagedClusterSets bound to the placement namespace. The containing fields are ANDed.", + "clusterSets": "ClusterSets represent the ManagedClusterSets from which the ManagedClusters are selected. If the slice is empty, ManagedClusters will be selected from the ManagedClusterSets bound to the placement namespace, otherwise ManagedClusters will be selected from the intersection of this slice and the ManagedClusterSets bound to the placement namespace.", + "numberOfClusters": "NumberOfClusters represents the desired number of ManagedClusters to be selected which meet the placement requirements. 1) If not specified, all ManagedClusters which meet the placement requirements (including ClusterSets,\n and Predicates) will be selected;\n2) Otherwise if the nubmer of ManagedClusters meet the placement requirements is larger than\n NumberOfClusters, a random subset with desired number of ManagedClusters will be selected;\n3) If the nubmer of ManagedClusters meet the placement requirements is equal to NumberOfClusters,\n all of them will be selected;\n4) If the nubmer of ManagedClusters meet the placement requirements is less than NumberOfClusters,\n all of them will be selected, and the status of condition `PlacementConditionSatisfied` will be\n set to false;", + "predicates": "Predicates represent a slice of predicates to select ManagedClusters. The predicates are ORed.", + "prioritizerPolicy": "PrioritizerPolicy defines the policy of the prioritizers. If this field is unset, then default prioritizer mode and configurations are used. Referring to PrioritizerPolicy to see more description about Mode and Configurations.", } func (PlacementSpec) SwaggerDoc() map[string]string { @@ -209,4 +210,23 @@ func (PlacementStatus) SwaggerDoc() map[string]string { return map_PlacementStatus } +var map_PrioritizerConfig = map[string]string{ + "": "PrioritizerConfig represents the configuration of prioritizer", + "name": "Name is the name of a prioritizer. Below are the valid names: 1) Balance: balance the decisions among the clusters. 2) Steady: ensure the existing decision is stabilized. 3) ResourceRatioCPU & ResourceRatioMemory: sort clusters based on the allocatable to capacity ratio. 4) ResourceAllocatableCPU & ResourceAllocatableMemory: sort clusters based on the allocatable.", + "weight": "Weight defines the weight of prioritizer. The value must be ranged in [0,10]. Each prioritizer will calculate an integer score of a cluster in the range of [-100, 100]. The final score of a cluster will be sum(weight * prioritizer_score). A higher weight indicates that the prioritizer weights more in the cluster selection, while 0 weight indicate thats the prioritizer is disabled.", +} + +func (PrioritizerConfig) SwaggerDoc() map[string]string { + return map_PrioritizerConfig +} + +var map_PrioritizerPolicy = map[string]string{ + "": "PrioritizerPolicy represents the policy of prioritizer", + "mode": "Mode is either Exact, Additive, \"\" where \"\" is Additive by default. In Additive mode, any prioritizer not explicitly enumerated is enabled in its default Configurations, in which Steady and Balance prioritizers have the weight of 1 while other prioritizers have the weight of 0. Additive doesn't require configuring all prioritizers. The default Configurations may change in the future, and additional prioritization will happen. In Exact mode, any prioritizer not explicitly enumerated is weighted as zero. Exact requires knowing the full set of prioritizers you want, but avoids behavior changes between releases.", +} + +func (PrioritizerPolicy) SwaggerDoc() map[string]string { + return map_PrioritizerPolicy +} + // AUTO-GENERATED FUNCTIONS END HERE