package addonconfiguration import ( "fmt" "sort" "time" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" clusterlisterv1beta1 "open-cluster-management.io/api/client/cluster/listers/cluster/v1beta1" clusterv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" clustersdkv1alpha1 "open-cluster-management.io/sdk-go/pkg/apis/cluster/v1alpha1" clustersdkv1beta1 "open-cluster-management.io/sdk-go/pkg/apis/cluster/v1beta1" "open-cluster-management.io/ocm/pkg/common/helpers" ) // configurationTree is a 2 level snapshot tree on the configuration of addons // the first level is a list of nodes that represents a install strategy and a desired configuration for this install // strategy. The second level is a list of nodes that represent each mca and its desired configuration type configurationGraph struct { // nodes maintains a list between a installStrategy and its related mcas nodes []*installStrategyNode // defaults is the nodes with no install strategy defaults *installStrategyNode } // installStrategyNode is a node in configurationGraph defined by a install strategy type installStrategyNode struct { placementRef addonv1alpha1.PlacementRef pdTracker *clustersdkv1beta1.PlacementDecisionClustersTracker rolloutStrategy clusterv1alpha1.RolloutStrategy rolloutResult clustersdkv1alpha1.RolloutResult desiredConfigs addonConfigMap // children keeps a map of addons node as the children of this node children map[string]*addonNode clusters sets.Set[string] } // addonNode is node as a child of installStrategy node represting a mca // addonnode type addonNode struct { desiredConfigs addonConfigMap mca *addonv1alpha1.ManagedClusterAddOn status *clustersdkv1alpha1.ClusterRolloutStatus } type addonConfigMap map[addonv1alpha1.ConfigGroupResource][]addonv1alpha1.ConfigReference // set addon rollout status func (n *addonNode) setRolloutStatus() { n.status = &clustersdkv1alpha1.ClusterRolloutStatus{ClusterName: n.mca.Namespace} // desired configs doesn't match actual configs, set to ToApply if len(n.mca.Status.ConfigReferences) != n.desiredConfigs.len() { n.status.Status = clustersdkv1alpha1.ToApply return } var progressingCond metav1.Condition for _, cond := range n.mca.Status.Conditions { if cond.Type == addonv1alpha1.ManagedClusterAddOnConditionProgressing { progressingCond = cond break } } for _, actual := range n.mca.Status.ConfigReferences { if index, exist := n.desiredConfigs.containsConfig(actual.ConfigGroupResource, actual.DesiredConfig.ConfigReferent); exist { // desired config spec hash doesn't match actual, set to ToApply if !equality.Semantic.DeepEqual(n.desiredConfigs[actual.ConfigGroupResource][index].DesiredConfig, actual.DesiredConfig) { n.status.Status = clustersdkv1alpha1.ToApply return // desired config spec hash matches actual, but last applied config spec hash doesn't match actual } else if !equality.Semantic.DeepEqual(actual.LastAppliedConfig, actual.DesiredConfig) { switch progressingCond.Reason { case addonv1alpha1.ProgressingReasonFailed: n.status.Status = clustersdkv1alpha1.Failed n.status.LastTransitionTime = &progressingCond.LastTransitionTime case addonv1alpha1.ProgressingReasonProgressing: n.status.Status = clustersdkv1alpha1.Progressing n.status.LastTransitionTime = &progressingCond.LastTransitionTime default: n.status.Status = clustersdkv1alpha1.Progressing } return } } else { n.status.Status = clustersdkv1alpha1.ToApply return } } // succeed n.status.Status = clustersdkv1alpha1.Succeeded if progressingCond.Reason == addonv1alpha1.ProgressingReasonCompleted { n.status.LastTransitionTime = &progressingCond.LastTransitionTime } } func (m addonConfigMap) copy() addonConfigMap { output := make(addonConfigMap, len(m)) for k, v := range m { // copy the key ConfigGroupResource newk := k.DeepCopy() // copy the slice of ConfigReference newv := make([]addonv1alpha1.ConfigReference, len(v)) for i, cr := range v { newv[i] = *cr.DeepCopy() } output[*newk] = newv } return output } func (m addonConfigMap) len() int { totalLength := 0 for _, v := range m { totalLength += len(v) } return totalLength } func (m addonConfigMap) containsConfig(expectConfigGr addonv1alpha1.ConfigGroupResource, expectConfigRef addonv1alpha1.ConfigReferent) (int, bool) { if existArray, ok := m[expectConfigGr]; ok { for i, e := range existArray { if expectConfigRef == e.DesiredConfig.ConfigReferent { return i, true } } } return -1, false } func (m addonConfigMap) orderedKeys() []addonv1alpha1.ConfigGroupResource { gvks := []addonv1alpha1.ConfigGroupResource{} for gvk := range m { gvks = append(gvks, gvk) } sort.Slice(gvks, func(i, j int) bool { if gvks[i].Group == gvks[j].Group { return gvks[i].Resource < gvks[j].Resource } return gvks[i].Group < gvks[j].Group }) return gvks } func newGraph(supportedConfigs []addonv1alpha1.ConfigMeta, defaultConfigReferences []addonv1alpha1.DefaultConfigReference) *configurationGraph { graph := &configurationGraph{ nodes: []*installStrategyNode{}, defaults: &installStrategyNode{ desiredConfigs: addonConfigMap{}, children: map[string]*addonNode{}, }, } // init graph.defaults.desiredConfigs with supportedConfigs for _, config := range supportedConfigs { if config.DefaultConfig != nil { graph.defaults.desiredConfigs[config.ConfigGroupResource] = []addonv1alpha1.ConfigReference{ { ConfigGroupResource: config.ConfigGroupResource, ConfigReferent: *config.DefaultConfig, DesiredConfig: &addonv1alpha1.ConfigSpecHash{ ConfigReferent: *config.DefaultConfig, }, }, } } } // copy the spechash from cma status defaultConfigReferences for _, configRef := range defaultConfigReferences { if configRef.DesiredConfig == nil { continue } if defaultsDesiredConfigArray, ok := graph.defaults.desiredConfigs[configRef.ConfigGroupResource]; ok { for i, defaultsDesiredConfig := range defaultsDesiredConfigArray { if defaultsDesiredConfig.DesiredConfig.ConfigReferent == configRef.DesiredConfig.ConfigReferent { graph.defaults.desiredConfigs[configRef.ConfigGroupResource][i].DesiredConfig.SpecHash = configRef.DesiredConfig.SpecHash } } } } return graph } // addAddonNode to the graph, starting from placement with the highest order func (g *configurationGraph) addAddonNode(mca *addonv1alpha1.ManagedClusterAddOn) { for i := len(g.nodes) - 1; i >= 0; i-- { if g.nodes[i].clusters.Has(mca.Namespace) { g.nodes[i].addNode(mca) return } } g.defaults.addNode(mca) } // addNode delete clusters on existing graph so the new configuration overrides the previous func (g *configurationGraph) addPlacementNode( installStrategy addonv1alpha1.PlacementStrategy, installProgression addonv1alpha1.InstallProgression, placementLister clusterlisterv1beta1.PlacementLister, placementDecisionGetter helpers.PlacementDecisionGetter, ) error { placementRef := installProgression.PlacementRef installConfigReference := installProgression.ConfigReferences // get placement if placementLister == nil { return fmt.Errorf("invalid placement lister %v", placementLister) } placement, err := placementLister.Placements(placementRef.Namespace).Get(placementRef.Name) if err != nil { return err } // new decision tracker pdTracker := clustersdkv1beta1.NewPlacementDecisionClustersTracker(placement, placementDecisionGetter, nil) // refresh and get existing decision clusters err = pdTracker.Refresh() if err != nil { return err } clusters := pdTracker.ExistingClusterGroupsBesides().GetClusters() node := &installStrategyNode{ placementRef: placementRef, pdTracker: pdTracker, rolloutStrategy: installStrategy.RolloutStrategy, desiredConfigs: g.defaults.desiredConfigs, children: map[string]*addonNode{}, clusters: clusters, } // Set MaxConcurrency // If progressive strategy is not initialized or MaxConcurrency is not specified, set MaxConcurrency to the default value if node.rolloutStrategy.Type == clusterv1alpha1.Progressive { progressiveStrategy := node.rolloutStrategy.Progressive if progressiveStrategy == nil { progressiveStrategy = &clusterv1alpha1.RolloutProgressive{} } if progressiveStrategy.MaxConcurrency.StrVal == "" && progressiveStrategy.MaxConcurrency.IntVal == 0 { progressiveStrategy.MaxConcurrency = placement.Spec.DecisionStrategy.GroupStrategy.ClustersPerDecisionGroup } node.rolloutStrategy.Progressive = progressiveStrategy } // overrides configuration by install strategy if len(installConfigReference) > 0 { node.desiredConfigs = node.desiredConfigs.copy() overrideConfigMapByInstallConfigRef(node.desiredConfigs, installConfigReference) } // remove addon in defaults and other placements. for _, cluster := range node.clusters.UnsortedList() { if _, ok := g.defaults.children[cluster]; ok { node.addNode(g.defaults.children[cluster].mca) delete(g.defaults.children, cluster) } for _, placementNode := range g.nodes { if _, ok := placementNode.children[cluster]; ok { node.addNode(placementNode.children[cluster].mca) delete(placementNode.children, cluster) } } } g.nodes = append(g.nodes, node) return nil } func (g *configurationGraph) generateRolloutResult() error { for _, node := range g.nodes { if err := node.generateRolloutResult(); err != nil { return err } } if err := g.defaults.generateRolloutResult(); err != nil { return err } return nil } func (g *configurationGraph) getPlacementNodes() map[addonv1alpha1.PlacementRef]*installStrategyNode { placementNodeMap := map[addonv1alpha1.PlacementRef]*installStrategyNode{} for _, node := range g.nodes { placementNodeMap[node.placementRef] = node } return placementNodeMap } // getAddonsToUpdate returns the list of addons to be updated based on the rollout strategy. // It is a subset of the addons returned by getAddonsToApply. // For example, if there are 10 addons whose desired config has not yet been synced to status, // all 10 addons will be returned by getAddonsToApply(). // Given a Progressive rollout strategy with a maxConcurrency of 3, only 3 of these addons // will be returned by getAddonsToUpdate() for update. func (g *configurationGraph) getAddonsToUpdate() []*addonNode { var addons []*addonNode for _, node := range g.nodes { addons = append(addons, node.getAddonsToUpdate()...) } addons = append(addons, g.defaults.getAddonsToUpdate()...) return addons } // getAddonsToApply returns the list of addons that need their configurations synchronized. // ToApply indicates that the resource's desired status has not been applied yet. func (g *configurationGraph) getAddonsToApply() []*addonNode { var addons []*addonNode for _, node := range g.nodes { addons = append(addons, node.getAddonsToApply()...) } addons = append(addons, g.defaults.getAddonsToApply()...) return addons } // getAddonsSucceeded returns the list of addons that their configurations desired status is applied // and last applied status is successful. func (g *configurationGraph) getAddonsSucceeded() []*addonNode { var addons []*addonNode for _, node := range g.nodes { addons = append(addons, node.getAddonsSucceeded()...) } addons = append(addons, g.defaults.getAddonsSucceeded()...) return addons } func (g *configurationGraph) getRequeueTime() time.Duration { minRequeue := maxRequeueTime for _, node := range g.nodes { nodeRecheckAfter := node.rolloutResult.RecheckAfter if nodeRecheckAfter != nil && *nodeRecheckAfter < minRequeue { minRequeue = *nodeRecheckAfter } } return minRequeue } func (n *installStrategyNode) addNode(addon *addonv1alpha1.ManagedClusterAddOn) { n.children[addon.Namespace] = &addonNode{ mca: addon, desiredConfigs: n.desiredConfigs, } // override configuration by mca spec if len(addon.Spec.Configs) > 0 { n.children[addon.Namespace].desiredConfigs = n.children[addon.Namespace].desiredConfigs.copy() // TODO we should also filter out the configs which are not supported configs. overrideConfigMapByAddOnConfigs(n.children[addon.Namespace].desiredConfigs, addon.Spec.Configs) // go through mca spec configs and copy the specHash from status if they match for _, config := range addon.Spec.Configs { for _, configRef := range addon.Status.ConfigReferences { if configRef.DesiredConfig == nil { continue } // compare the ConfigGroupResource and ConfigReferent if configRef.ConfigGroupResource != config.ConfigGroupResource || configRef.DesiredConfig.ConfigReferent != config.ConfigReferent { continue } // copy the spec hash to desired configs nodeDesiredConfigArray, ok := n.children[addon.Namespace].desiredConfigs[configRef.ConfigGroupResource] if !ok { continue } for i, nodeDesiredConfig := range nodeDesiredConfigArray { if nodeDesiredConfig.DesiredConfig.ConfigReferent == configRef.DesiredConfig.ConfigReferent { n.children[addon.Namespace].desiredConfigs[configRef.ConfigGroupResource][i].DesiredConfig.SpecHash = configRef.DesiredConfig.SpecHash } } } } } // set addon node rollout status n.children[addon.Namespace].setRolloutStatus() } func (n *installStrategyNode) generateRolloutResult() error { if n.placementRef.Name == "" { // default addons rolloutResult := clustersdkv1alpha1.RolloutResult{} rolloutResult.ClustersToRollout = []clustersdkv1alpha1.ClusterRolloutStatus{} for name, addon := range n.children { if addon.status == nil { return fmt.Errorf("failed to get rollout status on cluster %v", name) } if addon.status.Status != clustersdkv1alpha1.Succeeded { rolloutResult.ClustersToRollout = append(rolloutResult.ClustersToRollout, *addon.status) } } n.rolloutResult = rolloutResult } else { // placement addons rolloutHandler, err := clustersdkv1alpha1.NewRolloutHandler(n.pdTracker, getClusterRolloutStatus) if err != nil { return err } // get existing addons existingRolloutClusters := []clustersdkv1alpha1.ClusterRolloutStatus{} for name, addon := range n.children { clsRolloutStatus, err := getClusterRolloutStatus(name, addon) if err != nil { return err } existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) } // sort by cluster name sort.SliceStable(existingRolloutClusters, func(i, j int) bool { return existingRolloutClusters[i].ClusterName < existingRolloutClusters[j].ClusterName }) _, rolloutResult, err := rolloutHandler.GetRolloutCluster(n.rolloutStrategy, existingRolloutClusters) if err != nil { return err } n.rolloutResult = rolloutResult } return nil } // addonToUpdate finds the addons to be updated by placement func (n *installStrategyNode) getAddonsToUpdate() []*addonNode { var addons []*addonNode var clusters []string // get addon to update from rollout result for _, c := range n.rolloutResult.ClustersToRollout { if _, exist := n.children[c.ClusterName]; exist { clusters = append(clusters, c.ClusterName) } } // sort addons by name sort.Strings(clusters) for _, k := range clusters { addons = append(addons, n.children[k]) } return addons } // getAddonsToApply return the addons to sync configurations // ToApply indicates that the resource's desired status has not been applied yet. func (n *installStrategyNode) getAddonsToApply() []*addonNode { var addons []*addonNode for i, addon := range n.children { if addon.status.Status == clustersdkv1alpha1.ToApply { addons = append(addons, n.children[i]) } } return addons } // getAddonsSucceeded return the addons already rollout successfully or has no configurations func (n *installStrategyNode) getAddonsSucceeded() []*addonNode { var addons []*addonNode for i, addon := range n.children { if addon.status.Status == clustersdkv1alpha1.Succeeded { addons = append(addons, n.children[i]) } } return addons } // Return the number of succeed addons. // Including the addons with status Succeed after MinSuccessTime. func (n *installStrategyNode) countAddonUpgradeSucceed() int { count := 0 for _, addon := range n.children { if desiredConfigsEqual(addon.desiredConfigs, n.desiredConfigs) && addon.status.Status == clustersdkv1alpha1.Succeeded && !rolloutStatusHasCluster(n.rolloutResult.ClustersToRollout, addon.mca.Namespace) { count += 1 } } return count } // Return the number of failed addons after ProgressDeadline. func (n *installStrategyNode) countAddonUpgradeFailed() int { count := 0 for _, addon := range n.children { if desiredConfigsEqual(addon.desiredConfigs, n.desiredConfigs) && addon.status.Status == clustersdkv1alpha1.Failed && !rolloutStatusHasCluster(n.rolloutResult.ClustersToRollout, addon.mca.Namespace) { count += 1 } } return count } // Return the number of exiting addons in rolloutResult.ClustersToRollout. // Including the addons with status ToApply, Progressing within ProgressDeadline, Failed within ProgressDeadline and Succeed within MinSuccessTime. func (n *installStrategyNode) countAddonUpgrading() int { count := 0 for _, addon := range n.children { if rolloutStatusHasCluster(n.rolloutResult.ClustersToRollout, addon.mca.Namespace) { count += 1 } } return count } // Return the number of addons in rolloutResult.ClustersTimeOut. // Including the addons with status Progressing after ProgressDeadline, Failed after ProgressDeadline. func (n *installStrategyNode) countAddonTimeOut() int { return len(n.rolloutResult.ClustersTimeOut) } func getClusterRolloutStatus(clusterName string, addonNode *addonNode) (clustersdkv1alpha1.ClusterRolloutStatus, error) { if addonNode.status == nil { return clustersdkv1alpha1.ClusterRolloutStatus{}, fmt.Errorf("failed to get rollout status on cluster %v", clusterName) } return *addonNode.status, nil } func desiredConfigsEqual(a, b addonConfigMap) bool { if len(a) != len(b) { return false } for configgrA := range a { if len(a[configgrA]) != len(b[configgrA]) { return false } for i := range a[configgrA] { if a[configgrA][i] != b[configgrA][i] { return false } } } return true } func rolloutStatusHasCluster(clusterRolloutStatus []clustersdkv1alpha1.ClusterRolloutStatus, clusterName string) bool { for _, s := range clusterRolloutStatus { if s.ClusterName == clusterName { return true } } return false } // Override the desired addonConfigMap by a slice of InstallConfigReference (from cma status), func overrideConfigMapByInstallConfigRef( desiredConfigs addonConfigMap, installConfigRefs []addonv1alpha1.InstallConfigReference, ) { gvkOverwritten := sets.New[addonv1alpha1.ConfigGroupResource]() // Go through the cma installConfigReferences, // for a group of configs with same gvk, cma installConfigReferences override the desiredConfigs. for _, configRef := range installConfigRefs { if configRef.DesiredConfig == nil { continue } gr := configRef.ConfigGroupResource referent := configRef.DesiredConfig.ConfigReferent if !gvkOverwritten.Has(gr) { desiredConfigs[gr] = []addonv1alpha1.ConfigReference{} gvkOverwritten.Insert(gr) } // If a config not exist in the desiredConfigs, append it. // This is to avoid adding duplicate configs (name + namespace). if _, exist := desiredConfigs.containsConfig(gr, referent); !exist { desiredConfigs[gr] = append(desiredConfigs[gr], addonv1alpha1.ConfigReference{ ConfigGroupResource: gr, ConfigReferent: referent, DesiredConfig: configRef.DesiredConfig.DeepCopy(), }) } } } // Override the desired addonConfigMap by a slice of InstallConfigReference (from mca spec), func overrideConfigMapByAddOnConfigs( desiredConfigs addonConfigMap, addOnConfigs []addonv1alpha1.AddOnConfig, ) { gvkOverwritten := sets.New[addonv1alpha1.ConfigGroupResource]() // Go through the mca addOnConfigs, // for a group of configs with same gvk, mca addOnConfigs override the desiredConfigs. for _, config := range addOnConfigs { gr := config.ConfigGroupResource referent := config.ConfigReferent if !gvkOverwritten.Has(gr) { desiredConfigs[gr] = []addonv1alpha1.ConfigReference{} gvkOverwritten.Insert(gr) } // If a config not exist in the desiredConfigs, append it. // This is to avoid adding duplicate configs (name + namespace). if _, exist := desiredConfigs.containsConfig(gr, referent); !exist { desiredConfigs[gr] = append(desiredConfigs[gr], addonv1alpha1.ConfigReference{ ConfigGroupResource: gr, ConfigReferent: referent, DesiredConfig: &addonv1alpha1.ConfigSpecHash{ ConfigReferent: referent, }, }) } } }