package addontokeninfra import ( "context" "fmt" "github.com/openshift/library-go/pkg/assets" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" coreinformers "k8s.io/client-go/informers/core/v1" rbacinformers "k8s.io/client-go/informers/rbac/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1" addonclient "open-cluster-management.io/api/client/addon/clientset/versioned" addoninformerv1beta1 "open-cluster-management.io/api/client/addon/informers/externalversions/addon/v1beta1" addonlisterv1beta1 "open-cluster-management.io/api/client/addon/listers/addon/v1beta1" "open-cluster-management.io/sdk-go/pkg/basecontroller/events" "open-cluster-management.io/sdk-go/pkg/basecontroller/factory" "open-cluster-management.io/sdk-go/pkg/patcher" "open-cluster-management.io/ocm/manifests" "open-cluster-management.io/ocm/pkg/common/queue" commonrecorder "open-cluster-management.io/ocm/pkg/common/recorder" ) var ( tokenInfraManifests = []string{ "cluster-manager/hub/addon-manager/token-serviceaccount.yaml", "cluster-manager/hub/addon-manager/token-role.yaml", "cluster-manager/hub/addon-manager/token-rolebinding.yaml", } ) const ( // TokenInfrastructureReadyCondition is the condition type indicating token infrastructure is ready TokenInfrastructureReadyCondition = "TokenInfrastructureReady" ) // TokenInfraConfig holds configuration for rendering token infrastructure manifests type TokenInfraConfig struct { ClusterName string AddonName string } // tokenInfrastructureController reconciles ManagedClusterAddOn resources // to create token-based authentication infrastructure type tokenInfrastructureController struct { kubeClient kubernetes.Interface addonClient addonclient.Interface addonLister addonlisterv1beta1.ManagedClusterAddOnLister cache resourceapply.ResourceCache recorder events.Recorder } // usesTokenAuth checks if the addon uses token-based authentication // by checking if it has kubeClient registration and kubeClientDriver is "token" func usesTokenAuth(addon *addonapiv1beta1.ManagedClusterAddOn) bool { for _, reg := range addon.Status.Registrations { if reg.Type != addonapiv1beta1.KubeClient { continue } if reg.KubeClient == nil { continue } return reg.KubeClient.Driver == "token" } return false } // addonFilter filters addons that use token-based kubeClient registration // or have TokenInfrastructureReady condition (for cleanup) func addonFilter(obj interface{}) bool { addon, ok := obj.(*addonapiv1beta1.ManagedClusterAddOn) if !ok { return false } // Check if addon uses token authentication if usesTokenAuth(addon) { return true } // Check if addon has TokenInfrastructureReady condition (needs cleanup) if meta.FindStatusCondition(addon.Status.Conditions, TokenInfrastructureReadyCondition) != nil { return true } return false } // tokenInfraResourceToAddonKey extracts the addon namespace/name from token infrastructure resource labels // Returns empty string if resource doesn't have the required labels func tokenInfraResourceToAddonKey(obj runtime.Object) string { metaObj, err := meta.Accessor(obj) if err != nil { return "" } labels := metaObj.GetLabels() if labels == nil { return "" } // Only process resources with token-infrastructure label if labels["addon.open-cluster-management.io/token-infrastructure"] != "true" { return "" } // Extract addon name from label addonName := labels["addon.open-cluster-management.io/name"] if addonName == "" { return "" } // Namespace is the cluster name namespace := metaObj.GetNamespace() if namespace == "" { return "" } // Return namespace/addonName as queue key return namespace + "/" + addonName } // newTokenInfraEventHandler creates an event handler for token infrastructure resources // that enqueues the associated addon for reconciliation on Update and Delete events func newTokenInfraEventHandler(syncCtx factory.SyncContext, queueKeyFn func(runtime.Object) string) cache.ResourceEventHandler { return &cache.ResourceEventHandlerFuncs{ UpdateFunc: func(old, new interface{}) { newObj, ok := new.(runtime.Object) if !ok { utilruntime.HandleError(fmt.Errorf("error decoding object, invalid type")) return } if key := queueKeyFn(newObj); key != "" { syncCtx.Queue().Add(key) } }, DeleteFunc: func(obj interface{}) { var runtimeObj runtime.Object var ok bool if tombstone, isTombstone := obj.(cache.DeletedFinalStateUnknown); isTombstone { runtimeObj, ok = tombstone.Obj.(runtime.Object) if !ok { utilruntime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type")) return } } else { runtimeObj, ok = obj.(runtime.Object) if !ok { utilruntime.HandleError(fmt.Errorf("error decoding object, invalid type")) return } } if key := queueKeyFn(runtimeObj); key != "" { syncCtx.Queue().Add(key) } }, } } func NewTokenInfrastructureController( kubeClient kubernetes.Interface, addonClient addonclient.Interface, addonInformers addoninformerv1beta1.ManagedClusterAddOnInformer, serviceAccountInformer coreinformers.ServiceAccountInformer, roleInformer rbacinformers.RoleInformer, roleBindingInformer rbacinformers.RoleBindingInformer, ) factory.Controller { c := &tokenInfrastructureController{ kubeClient: kubeClient, addonClient: addonClient, addonLister: addonInformers.Lister(), cache: resourceapply.NewResourceCache(), recorder: events.NewContextualLoggingEventRecorder("addon-token-infrastructure-controller"), } syncCtx := factory.NewSyncContext("addon-token-infrastructure-controller") // Register custom event handlers on infrastructure informers // Only handle Update and Delete, NOT Add (creation is triggered by addon watch) eventHandler := newTokenInfraEventHandler(syncCtx, tokenInfraResourceToAddonKey) // Register the same event handler for all infrastructure informers saInformer := serviceAccountInformer.Informer() _, err := saInformer.AddEventHandler(eventHandler) if err != nil { utilruntime.HandleError(err) } rInformer := roleInformer.Informer() _, err = rInformer.AddEventHandler(eventHandler) if err != nil { utilruntime.HandleError(err) } rbInformer := roleBindingInformer.Informer() _, err = rbInformer.AddEventHandler(eventHandler) if err != nil { utilruntime.HandleError(err) } return factory.New(). WithSyncContext(syncCtx). // Primary watch: ManagedClusterAddOns with token registration WithFilteredEventsInformersQueueKeysFunc( queue.QueueKeyByMetaNamespaceName, addonFilter, addonInformers.Informer(), ). // Bare informers with custom handlers (already registered above) WithBareInformers( saInformer, rInformer, rbInformer, ). WithSync(c.sync). ToController("addon-token-infrastructure-controller") } func (c *tokenInfrastructureController) sync(ctx context.Context, syncCtx factory.SyncContext, key string) error { logger := klog.FromContext(ctx).WithValues("addon", key) logger.V(4).Info("Reconciling addon token authentication") clusterName, addonName, err := cache.SplitMetaNamespaceKey(key) if err != nil { // ignore addon whose key is invalid return nil } addon, err := c.addonLister.ManagedClusterAddOns(clusterName).Get(addonName) if errors.IsNotFound(err) { // Addon is deleted, attempt to clean up any remaining token infrastructure logger.Info("Addon not found, cleaning up any remaining token infrastructure") return c.cleanupTokenInfrastructure(ctx, clusterName, addonName) } if err != nil { return err } // If addon is being deleted, clean up token infrastructure if addon.DeletionTimestamp != nil { logger.Info("Addon is being deleted, cleaning up token infrastructure") // Check if TokenInfrastructureReady condition exists infraReady := meta.FindStatusCondition(addon.Status.Conditions, TokenInfrastructureReadyCondition) if infraReady != nil { // Clean up token infrastructure resources // No need to remove the condition - the addon is being deleted anyway return c.cleanupTokenInfrastructure(ctx, clusterName, addonName) } return nil } // Check if addon uses token authentication if !usesTokenAuth(addon) { // Check if TokenInfrastructureReady condition exists infraReady := meta.FindStatusCondition(addon.Status.Conditions, TokenInfrastructureReadyCondition) if infraReady == nil { // No condition, nothing to clean up logger.V(4).Info("No token-based kubeClient authentication found and no condition exists, skipping") return nil } // Clean up token infrastructure resources logger.Info("Addon no longer uses token-based authentication, cleaning up token infrastructure") if err := c.cleanupTokenInfrastructure(ctx, clusterName, addonName); err != nil { return err } // Remove TokenInfrastructureReady condition return c.removeCondition(ctx, addon) } // Ensure token infrastructure is created and ready return c.ensureTokenInfrastructure(ctx, addon, clusterName, addonName) } // ensureTokenInfrastructure creates and maintains token authentication infrastructure func (c *tokenInfrastructureController) ensureTokenInfrastructure( ctx context.Context, addon *addonapiv1beta1.ManagedClusterAddOn, clusterName, addonName string) error { config := TokenInfraConfig{ ClusterName: clusterName, AddonName: addonName, } // Apply manifests resourceResults := c.applyManifests(ctx, config) var saUID string var errs []error for _, result := range resourceResults { if result.Error != nil { errs = append(errs, fmt.Errorf("%q (%T): %v", result.File, result.Type, result.Error)) } // Extract ServiceAccount UID if sa, ok := result.Result.(*corev1.ServiceAccount); ok { saUID = string(sa.UID) } } if len(errs) > 0 { updateErr := c.updateCondition(ctx, addon, metav1.ConditionFalse, "TokenInfrastructureApplyFailed", fmt.Sprintf("Failed to apply token infrastructure: %v", utilerrors.NewAggregate(errs))) // Append updateErr to errs and return aggregate (NewAggregate filters out nil errors) return utilerrors.NewAggregate(append(errs, updateErr)) } // Set TokenInfrastructureReady condition with ServiceAccount UID serviceAccountName := fmt.Sprintf("%s-agent", addonName) message := fmt.Sprintf("ServiceAccount %s/%s (UID: %s) is ready", clusterName, serviceAccountName, saUID) return c.updateCondition(ctx, addon, metav1.ConditionTrue, "TokenInfrastructureReady", message) } // applyManifests applies token infrastructure manifests func (c *tokenInfrastructureController) applyManifests(ctx context.Context, config TokenInfraConfig) []resourceapply.ApplyResult { recorderWrapper := commonrecorder.NewEventsRecorderWrapper(ctx, c.recorder) return resourceapply.ApplyDirectly( ctx, resourceapply.NewKubeClientHolder(c.kubeClient), recorderWrapper, c.cache, func(name string) ([]byte, error) { template, err := manifests.ClusterManagerManifestFiles.ReadFile(name) if err != nil { return nil, err } return assets.MustCreateAssetFromTemplate(name, template, config).Data, nil }, tokenInfraManifests..., ) } // updateCondition updates the TokenInfrastructureReady condition on the addon func (c *tokenInfrastructureController) updateCondition(ctx context.Context, addon *addonapiv1beta1.ManagedClusterAddOn, status metav1.ConditionStatus, reason, message string) error { addonPatcher := patcher.NewPatcher[ *addonapiv1beta1.ManagedClusterAddOn, addonapiv1beta1.ManagedClusterAddOnSpec, addonapiv1beta1.ManagedClusterAddOnStatus]( c.addonClient.AddonV1beta1().ManagedClusterAddOns(addon.Namespace)) addonCopy := addon.DeepCopy() condition := metav1.Condition{ Type: TokenInfrastructureReadyCondition, Status: status, Reason: reason, Message: message, } meta.SetStatusCondition(&addonCopy.Status.Conditions, condition) _, err := addonPatcher.PatchStatus(ctx, addonCopy, addonCopy.Status, addon.Status) return err } // removeCondition removes the TokenInfrastructureReady condition from the addon func (c *tokenInfrastructureController) removeCondition(ctx context.Context, addon *addonapiv1beta1.ManagedClusterAddOn) error { addonPatcher := patcher.NewPatcher[ *addonapiv1beta1.ManagedClusterAddOn, addonapiv1beta1.ManagedClusterAddOnSpec, addonapiv1beta1.ManagedClusterAddOnStatus]( c.addonClient.AddonV1beta1().ManagedClusterAddOns(addon.Namespace)) addonCopy := addon.DeepCopy() // Remove the TokenInfrastructureReady condition meta.RemoveStatusCondition(&addonCopy.Status.Conditions, TokenInfrastructureReadyCondition) _, err := addonPatcher.PatchStatus(ctx, addonCopy, addonCopy.Status, addon.Status) return err } // cleanupTokenInfrastructure removes token authentication infrastructure resources func (c *tokenInfrastructureController) cleanupTokenInfrastructure(ctx context.Context, clusterName, addonName string) error { logger := klog.FromContext(ctx) config := TokenInfraConfig{ ClusterName: clusterName, AddonName: addonName, } recorderWrapper := commonrecorder.NewEventsRecorderWrapper(ctx, c.recorder) resourceResults := resourceapply.DeleteAll( ctx, resourceapply.NewKubeClientHolder(c.kubeClient), recorderWrapper, func(name string) ([]byte, error) { template, err := manifests.ClusterManagerManifestFiles.ReadFile(name) if err != nil { return nil, err } return assets.MustCreateAssetFromTemplate(name, template, config).Data, nil }, tokenInfraManifests..., ) var errs []error for _, result := range resourceResults { if result.Error != nil { errs = append(errs, fmt.Errorf("%q (%T): %v", result.File, result.Type, result.Error)) } } if len(errs) > 0 { return utilerrors.NewAggregate(errs) } logger.Info("Successfully cleaned up token infrastructure", "addon", addonName) return nil }