open-cluster-management/pkg/addon/controllers/addonprogressing/controller.go

package addonprogressing

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/openshift/library-go/pkg/controller/factory"
	"github.com/openshift/library-go/pkg/operator/events"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/selection"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"

	"open-cluster-management.io/addon-framework/pkg/addonmanager/constants"
	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy"
	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
	addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
	addoninformerv1alpha1 "open-cluster-management.io/api/client/addon/informers/externalversions/addon/v1alpha1"
	addonlisterv1alpha1 "open-cluster-management.io/api/client/addon/listers/addon/v1alpha1"
	workinformers "open-cluster-management.io/api/client/work/informers/externalversions/work/v1"
	worklister "open-cluster-management.io/api/client/work/listers/work/v1"
	workapiv1 "open-cluster-management.io/api/work/v1"
	"open-cluster-management.io/sdk-go/pkg/patcher"

	"open-cluster-management.io/ocm/pkg/common/queue"
)

// addonProgressingController reconciles instances of managedclusteraddon on the hub
// based to update the status progressing condition and last applied config
type addonProgressingController struct {
	addonClient                  addonv1alpha1client.Interface
	managedClusterAddonLister    addonlisterv1alpha1.ManagedClusterAddOnLister
	clusterManagementAddonLister addonlisterv1alpha1.ClusterManagementAddOnLister
	workLister                   worklister.ManifestWorkLister
	addonFilterFunc              factory.EventFilterFunc
}

func NewAddonProgressingController(
	addonClient addonv1alpha1client.Interface,
	addonInformers addoninformerv1alpha1.ManagedClusterAddOnInformer,
	clusterManagementAddonInformers addoninformerv1alpha1.ClusterManagementAddOnInformer,
	workInformers workinformers.ManifestWorkInformer,
	addonFilterFunc factory.EventFilterFunc,
	recorder events.Recorder,
) factory.Controller {
	c := &addonProgressingController{
		addonClient:                  addonClient,
		managedClusterAddonLister:    addonInformers.Lister(),
		clusterManagementAddonLister: clusterManagementAddonInformers.Lister(),
		workLister:                   workInformers.Lister(),
		addonFilterFunc:              addonFilterFunc,
	}

	return factory.New().WithInformersQueueKeysFunc(
		queue.QueueKeyByMetaNamespaceName,
		addonInformers.Informer(), clusterManagementAddonInformers.Informer()).
		WithFilteredEventsInformersQueueKeysFunc(
			func(obj runtime.Object) []string {
				accessor, _ := meta.Accessor(obj)
				namespace := accessor.GetNamespace()
				if len(accessor.GetLabels()[addonapiv1alpha1.AddonNamespaceLabelKey]) > 0 {
					namespace = accessor.GetLabels()[addonapiv1alpha1.AddonNamespaceLabelKey]
				}
				return []string{fmt.Sprintf("%s/%s", namespace, accessor.GetLabels()[addonapiv1alpha1.AddonLabelKey])}
			},
			func(obj interface{}) bool {
				accessor, _ := meta.Accessor(obj)
				return len(accessor.GetLabels()) > 0 && len(accessor.GetLabels()[addonapiv1alpha1.AddonLabelKey]) > 0
			},
			workInformers.Informer()).
		WithSync(c.sync).ToController("addon-progressing-controller", recorder)
}

func (c *addonProgressingController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
	logger := klog.FromContext(ctx)
	key := syncCtx.QueueKey()
	logger.V(4).Info("Reconciling addon", "addon", key)

	namespace, addonName, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		// ignore addon whose key is invalid
		return nil
	}

	addon, err := c.managedClusterAddonLister.ManagedClusterAddOns(namespace).Get(addonName)
	switch {
	case errors.IsNotFound(err):
		return nil
	case err != nil:
		return err
	}

	clusterManagementAddon, err := c.clusterManagementAddonLister.Get(addonName)
	if errors.IsNotFound(err) {
		return nil
	}

	if err != nil {
		return err
	}

	if !c.addonFilterFunc(clusterManagementAddon) {
		return nil
	}

	// update progressing condition and last applied config
	_, err = c.updateAddonProgressingAndLastApplied(ctx, addon.DeepCopy(), addon)
	return err
}

func (c *addonProgressingController) updateAddonProgressingAndLastApplied(
	ctx context.Context, newaddon, oldaddon *addonapiv1alpha1.ManagedClusterAddOn) (bool, error) {
	patcher := patcher.NewPatcher[
		*addonapiv1alpha1.ManagedClusterAddOn,
		addonapiv1alpha1.ManagedClusterAddOnSpec,
		addonapiv1alpha1.ManagedClusterAddOnStatus](
		c.addonClient.AddonV1alpha1().ManagedClusterAddOns(newaddon.Namespace))
	// check config references
	if supported, config := isConfigurationSupported(newaddon); !supported {
		meta.SetStatusCondition(&newaddon.Status.Conditions, metav1.Condition{
			Type:   addonapiv1alpha1.ManagedClusterAddOnConditionProgressing,
			Status: metav1.ConditionFalse,
			Reason: addonapiv1alpha1.ProgressingReasonConfigurationUnsupported,
			Message: fmt.Sprintf("Configuration with gvr %s/%s is not supported for this addon",
				config.Group, config.Resource),
		})

		return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
	}

	// wait until addon has ManifestApplied condition
	if cond := meta.FindStatusCondition(
		newaddon.Status.Conditions, addonapiv1alpha1.ManagedClusterAddOnManifestApplied); cond == nil {
		meta.SetStatusCondition(&newaddon.Status.Conditions, metav1.Condition{
			Type:    addonapiv1alpha1.ManagedClusterAddOnConditionProgressing,
			Status:  metav1.ConditionFalse,
			Reason:  "WaitingForManifestApplied",
			Message: "Waiting for ManagedClusterAddOn ManifestApplied condition",
		})
		return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
	}

	var hostingClusterName string
	if newaddon.Annotations != nil && len(newaddon.Annotations[addonapiv1alpha1.HostingClusterNameAnnotationKey]) > 0 {
		hostingClusterName = newaddon.Annotations[addonapiv1alpha1.HostingClusterNameAnnotationKey]
	}

	if len(hostingClusterName) > 0 {
		// wait until addon has HostingManifestApplied condition
		if cond := meta.FindStatusCondition(
			newaddon.Status.Conditions, addonapiv1alpha1.ManagedClusterAddOnHostingManifestApplied); cond == nil {
			meta.SetStatusCondition(&newaddon.Status.Conditions, metav1.Condition{
				Type:    addonapiv1alpha1.ManagedClusterAddOnConditionProgressing,
				Status:  metav1.ConditionFalse,
				Reason:  "WaitingForHostingManifestApplied",
				Message: "Waiting for ManagedClusterAddOn HostingManifestApplied condition",
			})
			return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
		}
	}

	// get addon works
	// first get all works for addon in default mode.
	requirement, _ := labels.NewRequirement(addonapiv1alpha1.AddonLabelKey, selection.Equals, []string{newaddon.Name})
	namespaceNotExistRequirement, _ := labels.NewRequirement(addonapiv1alpha1.AddonNamespaceLabelKey, selection.DoesNotExist, []string{})
	selector := labels.NewSelector().Add(*requirement).Add(*namespaceNotExistRequirement)
	addonWorks, err := c.workLister.ManifestWorks(newaddon.Namespace).List(selector)
	if err != nil {
		setAddOnProgressingAndLastApplied(addonapiv1alpha1.ProgressingReasonFailed, err.Error(), newaddon)
		return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
	}

	// next get all works for addon in hosted mode
	if len(hostingClusterName) > 0 {
		namespaceRequirement, _ := labels.NewRequirement(addonapiv1alpha1.AddonNamespaceLabelKey, selection.Equals, []string{newaddon.Namespace})
		selector = labels.NewSelector().Add(*requirement).Add(*namespaceRequirement)
		hostedAddonWorks, err := c.workLister.ManifestWorks(hostingClusterName).List(selector)
		if err != nil {
			setAddOnProgressingAndLastApplied(addonapiv1alpha1.ProgressingReasonFailed, err.Error(), newaddon)
			return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
		}
		addonWorks = append(addonWorks, hostedAddonWorks...)
	}

	if len(addonWorks) == 0 {
		setAddOnProgressingAndLastApplied(addonapiv1alpha1.ProgressingReasonProgressing, "no addon works", newaddon)
		return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
	}

	// check addon manifestworks
	for _, work := range addonWorks {
		// skip pre-delete manifestwork
		if strings.HasPrefix(work.Name, constants.PreDeleteHookWorkName(newaddon.Name)) {
			continue
		}

		// check if work configs matches addon configs
		if !workConfigsMatchesAddon(klog.FromContext(ctx), work, newaddon) {
			setAddOnProgressingAndLastApplied(addonapiv1alpha1.ProgressingReasonProgressing, "mca and work configs mismatch", newaddon)
			return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
		}

		// check if work is ready
		if !workIsReady(work) {
			setAddOnProgressingAndLastApplied(addonapiv1alpha1.ProgressingReasonProgressing, "work is not ready", newaddon)
			return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
		}
	}

	// set lastAppliedConfig when all the work matches addon and are ready.
	setAddOnProgressingAndLastApplied(addonapiv1alpha1.ProgressingReasonCompleted, "", newaddon)
	return patcher.PatchStatus(ctx, newaddon, newaddon.Status, oldaddon.Status)
}

func isConfigurationSupported(addon *addonapiv1alpha1.ManagedClusterAddOn) (bool, addonapiv1alpha1.ConfigGroupResource) {
	supportedConfigSet := map[addonapiv1alpha1.ConfigGroupResource]bool{}
	for _, config := range addon.Status.SupportedConfigs {
		supportedConfigSet[config] = true
	}

	for _, config := range addon.Spec.Configs {
		if _, ok := supportedConfigSet[config.ConfigGroupResource]; !ok {
			return false, config.ConfigGroupResource
		}
	}

	return true, addonapiv1alpha1.ConfigGroupResource{}
}

func workConfigsMatchesAddon(logger klog.Logger, work *workapiv1.ManifestWork, addon *addonapiv1alpha1.ManagedClusterAddOn) bool {
	// get work spec hash
	if _, ok := work.Annotations[workapiv1.ManifestConfigSpecHashAnnotationKey]; !ok {
		return len(addon.Status.ConfigReferences) == 0
	}

	// parse work spec hash
	workSpecHashMap := make(map[string]string)
	if err := json.Unmarshal([]byte(work.Annotations[workapiv1.ManifestConfigSpecHashAnnotationKey]), &workSpecHashMap); err != nil {
		logger.Info("Failed to parse work spec hash", "error", err)
		return false
	}

	// check work spec hash, all the config should have spec hash
	for _, v := range workSpecHashMap {
		if v == "" {
			return false
		}
	}

	// check addon desired config
	for _, configReference := range addon.Status.ConfigReferences {
		if configReference.DesiredConfig == nil || configReference.DesiredConfig.SpecHash == "" {
			return false
		}
	}
	addonSpecHashMap := agentdeploy.ConfigsToMap(addon.Status.ConfigReferences)

	// compare work and addon configs
	return equality.Semantic.DeepEqual(workSpecHashMap, addonSpecHashMap)
}

// work is ready when
// 1) condition Available status is true.
// 2) condition Available observedGeneration equals to generation.
// 3) If it is a fresh install since one addon can have multiple ManifestWorks, the ManifestWork condition ManifestApplied must also be true.
func workIsReady(work *workapiv1.ManifestWork) bool {
	cond := meta.FindStatusCondition(work.Status.Conditions, workapiv1.WorkAvailable)
	if cond == nil || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != work.Generation {
		return false
	}
	cond = meta.FindStatusCondition(work.Status.Conditions, workapiv1.WorkApplied)
	if cond == nil || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != work.Generation {
		return false
	}

	return true
}

// Set addon progressing condition and last applied
// Once completed, the LastAppliedConfig will set to DesiredConfig. Even the condition status
// changes later, eg, spoke cluster becomes unhealthy, status becomes progressing or failed after completed,
// the LastAppliedConfig and DesiredConfig won't change. this is to avoid back and forth in rollout.
// The setRolloutStatus() func in graph.go only look at the LastAppliedConfig and DesiredConfig.
func setAddOnProgressingAndLastApplied(reason string, message string, addon *addonapiv1alpha1.ManagedClusterAddOn) {
	condition := metav1.Condition{
		Type:   addonapiv1alpha1.ManagedClusterAddOnConditionProgressing,
		Reason: reason,
	}
	switch reason {
	case addonapiv1alpha1.ProgressingReasonProgressing:
		condition.Status = metav1.ConditionTrue
		condition.Message = fmt.Sprintf("progressing... %v", message)
	case addonapiv1alpha1.ProgressingReasonCompleted:
		condition.Status = metav1.ConditionFalse
		for i, configReference := range addon.Status.ConfigReferences {
			addon.Status.ConfigReferences[i].LastAppliedConfig = configReference.DesiredConfig.DeepCopy()
		}
		condition.Message = "completed with no errors."
	case addonapiv1alpha1.ProgressingReasonFailed:
		condition.Status = metav1.ConditionFalse
		condition.Message = message
	}
	meta.SetStatusCondition(&addon.Status.Conditions, condition)
}