From 5cde8b5021eedb964b42e49ea0433cc58cb1b71a Mon Sep 17 00:00:00 2001 From: Yang Le Date: Wed, 21 Apr 2021 13:40:55 +0800 Subject: [PATCH] add resync logic for addon Signed-off-by: Yang Le --- pkg/spoke/addon/registration_controller.go | 35 ++- .../addon/registration_controller_test.go | 208 ++++++++++++++++++ 2 files changed, 241 insertions(+), 2 deletions(-) diff --git a/pkg/spoke/addon/registration_controller.go b/pkg/spoke/addon/registration_controller.go index 592facc24..1cef42649 100644 --- a/pkg/spoke/addon/registration_controller.go +++ b/pkg/spoke/addon/registration_controller.go @@ -38,6 +38,8 @@ type addOnRegistrationController struct { hubCSRClient csrclient.CertificateSigningRequestInterface recorder events.Recorder + startRegistrationFunc func(ctx context.Context, config registrationConfig) context.CancelFunc + // registrationConfigs maps the addon name to a map of registrationConfigs whose key is the hash of // the registrationConfig addOnRegistrationConfigs map[string]map[string]registrationConfig @@ -66,6 +68,8 @@ func NewAddOnRegistrationController( addOnRegistrationConfigs: map[string]map[string]registrationConfig{}, } + c.startRegistrationFunc = c.startRegistration + return factory.New(). WithInformersQueueKeyFunc( func(obj runtime.Object) string { @@ -74,11 +78,38 @@ func NewAddOnRegistrationController( }, hubAddOnInformers.Informer()). WithSync(c.sync). + ResyncEvery(10*time.Minute). ToController("AddOnRegistrationController", recorder) } func (c *addOnRegistrationController) sync(ctx context.Context, syncCtx factory.SyncContext) error { - addOnName := syncCtx.QueueKey() + queueKey := syncCtx.QueueKey() + if queueKey != factory.DefaultQueueKey { + // sync a particular addOn + return c.syncAddOn(ctx, syncCtx, queueKey) + } + + // handle resync + errs := []error{} + for addOnName := range c.addOnRegistrationConfigs { + _, err := c.hubAddOnLister.ManagedClusterAddOns(c.clusterName).Get(addOnName) + if err == nil { + syncCtx.Queue().Add(addOnName) + continue + } + if errors.IsNotFound(err) { + // clean up if the addOn no longer exists + err = c.cleanup(ctx, addOnName) + } + if err != nil { + errs = append(errs, err) + } + } + + return operatorhelpers.NewMultiLineAggregate(errs) +} + +func (c *addOnRegistrationController) syncAddOn(ctx context.Context, syncCtx factory.SyncContext, addOnName string) error { klog.V(4).Infof("Reconciling addOn %q", addOnName) addOn, err := c.hubAddOnLister.ManagedClusterAddOns(c.clusterName).Get(addOnName) @@ -126,7 +157,7 @@ func (c *addOnRegistrationController) sync(ctx context.Context, syncCtx factory. } // start registration for the new added configs - config.stopFunc = c.startRegistration(ctx, config) + config.stopFunc = c.startRegistrationFunc(ctx, config) syncedConfigs[hash] = config } diff --git a/pkg/spoke/addon/registration_controller_test.go b/pkg/spoke/addon/registration_controller_test.go index a252618b2..cf1e6ca02 100644 --- a/pkg/spoke/addon/registration_controller_test.go +++ b/pkg/spoke/addon/registration_controller_test.go @@ -1,12 +1,26 @@ package addon import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" "testing" + "time" + "github.com/openshift/library-go/pkg/controller/factory" certificates "k8s.io/api/certificates/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + addonv1alpha1 "github.com/open-cluster-management/api/addon/v1alpha1" + addonfake "github.com/open-cluster-management/api/client/addon/clientset/versioned/fake" + addoninformers "github.com/open-cluster-management/api/client/addon/informers/externalversions" "github.com/open-cluster-management/registration/pkg/clientcert" + testinghelpers "github.com/open-cluster-management/registration/pkg/helpers/testing" + "github.com/openshift/library-go/pkg/operator/events/eventstesting" ) func TestFilterCSREvents(t *testing.T) { @@ -59,3 +73,197 @@ func TestFilterCSREvents(t *testing.T) { }) } } + +func TestRegistrationSync(t *testing.T) { + clusterName := "cluster1" + addonName := "addon1" + signerName := "signer1" + + config1 := addonv1alpha1.RegistrationConfig{ + SignerName: signerName, + } + + config2 := addonv1alpha1.RegistrationConfig{ + SignerName: signerName, + Subject: addonv1alpha1.Subject{ + User: addonName, + }, + } + + cases := []struct { + name string + queueKey string + addOn *addonv1alpha1.ManagedClusterAddOn + addOnRegistrationConfigs map[string]map[string]registrationConfig + expectedAddOnRegistrationConfigHashs map[string][]string + validateActions func(t *testing.T, actions []clienttesting.Action) + }{ + { + name: "addon registration not enabled", + queueKey: addonName, + addOn: newManagedClusterAddOn(clusterName, addonName, nil), + validateActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 0 { + t.Errorf("expect 0 actions but got %d", len(actions)) + } + }, + }, + { + name: "addon registration enabled", + queueKey: addonName, + addOn: newManagedClusterAddOn(clusterName, addonName, []addonv1alpha1.RegistrationConfig{config1}), + expectedAddOnRegistrationConfigHashs: map[string][]string{ + addonName: {hash(config1)}, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 0 { + t.Errorf("expect 0 actions but got %d", len(actions)) + } + }, + }, + { + name: "addon registration updated", + queueKey: addonName, + addOn: newManagedClusterAddOn(clusterName, addonName, []addonv1alpha1.RegistrationConfig{config2}), + addOnRegistrationConfigs: map[string]map[string]registrationConfig{ + addonName: { + hash(config1): { + secretName: "secret1", + installationNamespace: addonName, + }, + }, + }, + expectedAddOnRegistrationConfigHashs: map[string][]string{ + addonName: {hash(config2)}, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 1 { + t.Errorf("expect 1 actions but got %d", len(actions)) + } + testinghelpers.AssertActions(t, actions, "delete") + }, + }, + { + name: "addon is deleted", + queueKey: addonName, + addOnRegistrationConfigs: map[string]map[string]registrationConfig{ + addonName: { + hash(config1): { + secretName: "secret1", + installationNamespace: addonName, + }, + }, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 1 { + t.Errorf("expect 1 actions but got %d", len(actions)) + } + testinghelpers.AssertActions(t, actions, "delete") + }, + }, + { + name: "resync", + queueKey: factory.DefaultQueueKey, + addOn: newManagedClusterAddOn(clusterName, addonName, []addonv1alpha1.RegistrationConfig{config1}), + addOnRegistrationConfigs: map[string]map[string]registrationConfig{ + addonName: { + hash(config1): { + secretName: "secret1", + installationNamespace: addonName, + }, + }, + "addon2": { + hash(config1): { + secretName: "secret2", + installationNamespace: "addon2", + }, + }, + }, + expectedAddOnRegistrationConfigHashs: map[string][]string{ + addonName: {hash(config1)}, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 1 { + t.Errorf("expect 1 actions but got %d", len(actions)) + } + testinghelpers.AssertActions(t, actions, "delete") + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + kubeClient := kubefake.NewSimpleClientset() + addons := []runtime.Object{} + if c.addOn != nil { + addons = append(addons, c.addOn) + } + addonClient := addonfake.NewSimpleClientset(addons...) + addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, time.Minute*10) + addonStore := addonInformerFactory.Addon().V1alpha1().ManagedClusterAddOns().Informer().GetStore() + if c.addOn != nil { + addonStore.Add(c.addOn) + } + + if c.addOnRegistrationConfigs == nil { + c.addOnRegistrationConfigs = map[string]map[string]registrationConfig{} + } + + controller := addOnRegistrationController{ + clusterName: clusterName, + kubeClient: kubeClient, + hubAddOnLister: addonInformerFactory.Addon().V1alpha1().ManagedClusterAddOns().Lister(), + recorder: eventstesting.NewTestingEventRecorder(t), + startRegistrationFunc: func(ctx context.Context, config registrationConfig) context.CancelFunc { + _, cancel := context.WithCancel(context.Background()) + return cancel + }, + addOnRegistrationConfigs: c.addOnRegistrationConfigs, + } + + err := controller.sync(context.Background(), testinghelpers.NewFakeSyncContext(t, c.queueKey)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(c.expectedAddOnRegistrationConfigHashs) != len(controller.addOnRegistrationConfigs) { + t.Errorf("expected %d addOns, but got %d", len(c.expectedAddOnRegistrationConfigHashs), len(controller.addOnRegistrationConfigs)) + } + + for addOnName, hashs := range c.expectedAddOnRegistrationConfigHashs { + addonRegistrationConfigs := controller.addOnRegistrationConfigs[addOnName] + if len(addonRegistrationConfigs) != len(hashs) { + t.Errorf("expected %d config items for addOn %q, but got %d", len(hashs), addOnName, len(addonRegistrationConfigs)) + } + for _, hash := range hashs { + if _, ok := addonRegistrationConfigs[hash]; !ok { + t.Errorf("registration config with hash %q is not found for addOn %q", hash, addOnName) + } + } + } + + if c.validateActions != nil { + c.validateActions(t, kubeClient.Actions()) + } + }) + } +} + +func newManagedClusterAddOn(namespace, name string, registrations []addonv1alpha1.RegistrationConfig) *addonv1alpha1.ManagedClusterAddOn { + return &addonv1alpha1.ManagedClusterAddOn{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Status: addonv1alpha1.ManagedClusterAddOnStatus{ + Registrations: registrations, + }, + } +} + +func hash(registration addonv1alpha1.RegistrationConfig) string { + data, _ := json.Marshal(registration) + h := sha256.New() + h.Write(data) + return fmt.Sprintf("%x", h.Sum(nil)) +}