From 3d4e3377fa5515ec725ec19d55165d1308bdfb5d Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Fri, 28 Oct 2022 22:02:21 +0800 Subject: [PATCH] Avoid frequent CSR creation (#277) * Avoid frequent CSR creation Signed-off-by: Jian Qiu * Add todo for threshold value Signed-off-by: Jian Qiu Signed-off-by: Jian Qiu --- pkg/clientcert/cert_controller.go | 72 ++++++--------------- pkg/clientcert/certficate_beta.go | 4 +- pkg/clientcert/certificate.go | 47 +++++++++++--- pkg/clientcert/controller_test.go | 9 +-- pkg/spoke/addon/registration_controller.go | 72 +++++++++++++++++---- pkg/spoke/managedcluster/registration.go | 67 +++++++++++++++++-- pkg/spoke/spokeagent.go | 26 ++++---- test/integration/addon_registration_test.go | 47 ++++++++++++++ 8 files changed, 245 insertions(+), 99 deletions(-) diff --git a/pkg/clientcert/cert_controller.go b/pkg/clientcert/cert_controller.go index 16d47f7d4..c786403b1 100644 --- a/pkg/clientcert/cert_controller.go +++ b/pkg/clientcert/cert_controller.go @@ -6,28 +6,22 @@ import ( "crypto/x509/pkix" "fmt" "math/rand" - ocmfeature "open-cluster-management.io/api/feature" "reflect" "time" "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - certificatesinformers "k8s.io/client-go/informers/certificates" corev1informers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" certutil "k8s.io/client-go/util/cert" "k8s.io/client-go/util/keyutil" "k8s.io/klog/v2" - "open-cluster-management.io/registration/pkg/features" - "open-cluster-management.io/registration/pkg/helpers" ) const ( @@ -73,6 +67,9 @@ type CSROption struct { // EventFilterFunc matches csrs created with above options EventFilterFunc factory.EventFilterFunc + + // HaltCSRCreation halt the csr creation + HaltCSRCreation func() bool } // ClientCertOption includes options that is used to create client certificate @@ -99,7 +96,7 @@ type StatusUpdateFunc func(ctx context.Context, cond metav1.Condition) error type clientCertificateController struct { ClientCertOption CSROption - csrControl + csrControl CSRControl // managementCoreClient is used to create/delete hub kubeconfig secret on the management cluster managementCoreClient corev1client.CoreV1Interface controllerName string @@ -123,51 +120,7 @@ type clientCertificateController struct { func NewClientCertificateController( clientCertOption ClientCertOption, csrOption CSROption, - hubCSRInformer certificatesinformers.Interface, - hubKubeClient kubernetes.Interface, - managementSecretInformer corev1informers.SecretInformer, - managementKubeClient kubernetes.Interface, - statusUpdater StatusUpdateFunc, - recorder events.Recorder, - controllerName string, -) (factory.Controller, error) { - var csrCtrl csrControl = nil - if features.DefaultSpokeMutableFeatureGate.Enabled(ocmfeature.V1beta1CSRAPICompatibility) { - v1CSRSupported, v1beta1CSRSupported, err := helpers.IsCSRSupported(hubKubeClient) - if err != nil { - return nil, errors.Wrapf(err, "failed CSR api discovery") - } - if !v1CSRSupported && v1beta1CSRSupported { - csrCtrl = &v1beta1CSRControl{ - hubCSRInformer: hubCSRInformer.V1beta1().CertificateSigningRequests(), - hubCSRLister: hubCSRInformer.V1beta1().CertificateSigningRequests().Lister(), - hubCSRClient: hubKubeClient.CertificatesV1beta1().CertificateSigningRequests(), - } - klog.Info("Using v1beta1 CSR api to manage spoke client certificate") - } - } - if csrCtrl == nil { - csrCtrl = &v1CSRControl{ - hubCSRInformer: hubCSRInformer.V1().CertificateSigningRequests(), - hubCSRLister: hubCSRInformer.V1().CertificateSigningRequests().Lister(), - hubCSRClient: hubKubeClient.CertificatesV1().CertificateSigningRequests(), - } - } - return newClientCertificateController( - clientCertOption, - csrOption, - csrCtrl, - managementSecretInformer, - managementKubeClient.CoreV1(), - statusUpdater, - recorder, - controllerName), nil -} - -func newClientCertificateController( - clientCertOption ClientCertOption, - csrOption CSROption, - csrControl csrControl, + csrControl CSRControl, managementSecretInformer corev1informers.SecretInformer, managementCoreClient corev1client.CoreV1Interface, statusUpdater StatusUpdateFunc, @@ -199,7 +152,7 @@ func newClientCertificateController( }, managementSecretInformer.Informer()). WithFilteredEventsInformersQueueKeyFunc(func(obj runtime.Object) string { return factory.DefaultQueueKey - }, c.EventFilterFunc, csrControl.informer()). + }, c.EventFilterFunc, csrControl.Informer()). WithSync(c.sync). ResyncEvery(ControllerResyncInterval). ToController(controllerName, recorder) @@ -347,6 +300,19 @@ func (c *clientCertificateController) sync(ctx context.Context, syncCtx factory. return nil } + shouldHalt := c.CSROption.HaltCSRCreation() + if shouldHalt { + if updateErr := c.statusUpdater(ctx, metav1.Condition{ + Type: "ClusterCertificateRotated", + Status: metav1.ConditionFalse, + Reason: "ClientCertificateUpdateFailed", + Message: "Stop creating csr since there are too many csr created already on hub", + }); updateErr != nil { + return updateErr + } + return nil + } + // create a new private key keyData, err := keyutil.MakeEllipticPrivateKeyPEM() if err != nil { diff --git a/pkg/clientcert/certficate_beta.go b/pkg/clientcert/certficate_beta.go index 606b99e83..d012cb84d 100644 --- a/pkg/clientcert/certficate_beta.go +++ b/pkg/clientcert/certficate_beta.go @@ -14,7 +14,7 @@ import ( "k8s.io/client-go/tools/cache" ) -var _ csrControl = &v1beta1CSRControl{} +var _ CSRControl = &v1beta1CSRControl{} type v1beta1CSRControl struct { hubCSRInformer certificatesinformers.CertificateSigningRequestInformer @@ -74,7 +74,7 @@ func (v *v1beta1CSRControl) create(ctx context.Context, recorder events.Recorder return req.Name, nil } -func (v *v1beta1CSRControl) informer() cache.SharedIndexInformer { +func (v *v1beta1CSRControl) Informer() cache.SharedIndexInformer { return v.hubCSRInformer.Informer() } diff --git a/pkg/clientcert/certificate.go b/pkg/clientcert/certificate.go index 1326ef289..69e57efeb 100644 --- a/pkg/clientcert/certificate.go +++ b/pkg/clientcert/certificate.go @@ -12,14 +12,19 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - certificatesinformers "k8s.io/client-go/informers/certificates/v1" + certificatesinformers "k8s.io/client-go/informers/certificates" + certificatesv1informers "k8s.io/client-go/informers/certificates/v1" + "k8s.io/client-go/kubernetes" csrclient "k8s.io/client-go/kubernetes/typed/certificates/v1" - certificateslisters "k8s.io/client-go/listers/certificates/v1" + certificatesv1listers "k8s.io/client-go/listers/certificates/v1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" "k8s.io/klog/v2" + ocmfeature "open-cluster-management.io/api/feature" + "open-cluster-management.io/registration/pkg/features" + "open-cluster-management.io/registration/pkg/helpers" ) // HasValidClientCertificate checks if there exists a valid client certificate in the given secret @@ -165,18 +170,20 @@ func BuildKubeconfig(clientConfig *restclient.Config, certPath, keyPath string) return kubeconfig } -type csrControl interface { +type CSRControl interface { create(ctx context.Context, recorder events.Recorder, objMeta metav1.ObjectMeta, csrData []byte, signerName string) (string, error) isApproved(name string) (bool, error) getIssuedCertificate(name string) ([]byte, error) - informer() cache.SharedIndexInformer + + // public so we can add indexer outside + Informer() cache.SharedIndexInformer } -var _ csrControl = &v1CSRControl{} +var _ CSRControl = &v1CSRControl{} type v1CSRControl struct { - hubCSRInformer certificatesinformers.CertificateSigningRequestInformer - hubCSRLister certificateslisters.CertificateSigningRequestLister + hubCSRInformer certificatesv1informers.CertificateSigningRequestInformer + hubCSRLister certificatesv1listers.CertificateSigningRequestLister hubCSRClient csrclient.CertificateSigningRequestInterface } @@ -228,7 +235,7 @@ func (v *v1CSRControl) create(ctx context.Context, recorder events.Recorder, obj return req.Name, nil } -func (v *v1CSRControl) informer() cache.SharedIndexInformer { +func (v *v1CSRControl) Informer() cache.SharedIndexInformer { return v.hubCSRInformer.Informer() } @@ -246,3 +253,27 @@ func (v *v1CSRControl) get(name string) (metav1.Object, error) { } return csr, nil } + +func NewCSRControl(hubCSRInformer certificatesinformers.Interface, hubKubeClient kubernetes.Interface) (CSRControl, error) { + if features.DefaultSpokeMutableFeatureGate.Enabled(ocmfeature.V1beta1CSRAPICompatibility) { + v1CSRSupported, v1beta1CSRSupported, err := helpers.IsCSRSupported(hubKubeClient) + if err != nil { + return nil, err + } + if !v1CSRSupported && v1beta1CSRSupported { + csrCtrl := &v1beta1CSRControl{ + hubCSRInformer: hubCSRInformer.V1beta1().CertificateSigningRequests(), + hubCSRLister: hubCSRInformer.V1beta1().CertificateSigningRequests().Lister(), + hubCSRClient: hubKubeClient.CertificatesV1beta1().CertificateSigningRequests(), + } + klog.Info("Using v1beta1 CSR api to manage spoke client certificate") + return csrCtrl, nil + } + } + + return &v1CSRControl{ + hubCSRInformer: hubCSRInformer.V1().CertificateSigningRequests(), + hubCSRLister: hubCSRInformer.V1().CertificateSigningRequests().Lister(), + hubCSRClient: hubKubeClient.CertificatesV1().CertificateSigningRequests(), + }, nil +} diff --git a/pkg/clientcert/controller_test.go b/pkg/clientcert/controller_test.go index e64bc2b42..a421df444 100644 --- a/pkg/clientcert/controller_test.go +++ b/pkg/clientcert/controller_test.go @@ -186,8 +186,9 @@ func TestSync(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-", }, - Subject: testSubject, - SignerName: certificates.KubeAPIServerClientSignerName, + Subject: testSubject, + SignerName: certificates.KubeAPIServerClientSignerName, + HaltCSRCreation: func() bool { return false }, } updater := &fakeStatusUpdater{} @@ -230,7 +231,7 @@ func TestSync(t *testing.T) { } } -var _ csrControl = &mockCSRControl{} +var _ CSRControl = &mockCSRControl{} func conditionEqual(expected, actual *metav1.Condition) bool { if expected == nil && actual == nil { @@ -301,6 +302,6 @@ func (m *mockCSRControl) getIssuedCertificate(name string) ([]byte, error) { return m.issuedCertData, err } -func (m *mockCSRControl) informer() cache.SharedIndexInformer { +func (m *mockCSRControl) Informer() cache.SharedIndexInformer { panic("implement me") } diff --git a/pkg/spoke/addon/registration_controller.go b/pkg/spoke/addon/registration_controller.go index 5085ce422..382ad55eb 100644 --- a/pkg/spoke/addon/registration_controller.go +++ b/pkg/spoke/addon/registration_controller.go @@ -15,8 +15,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" - certificatesinformers "k8s.io/client-go/informers/certificates" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" addonclient "open-cluster-management.io/api/client/addon/clientset/versioned" addoninformerv1alpha1 "open-cluster-management.io/api/client/addon/informers/externalversions/addon/v1alpha1" @@ -25,6 +25,13 @@ import ( "open-cluster-management.io/registration/pkg/helpers" ) +const ( + indexByAddon = "indexByAddon" + + // TODO(qiujian16) expose it if necessary in the future. + addonCSRThreshold = 10 +) + // addOnRegistrationController monitors ManagedClusterAddOns on hub and starts addOn registration // according to the registrationConfigs read from annotations of ManagedClusterAddOns. Echo addOn // may have multiple registrationConfigs. A clientcert.NewClientCertificateController will be started @@ -36,10 +43,10 @@ type addOnRegistrationController struct { managementKubeClient kubernetes.Interface // in-cluster local management kubeClient spokeKubeClient kubernetes.Interface hubAddOnLister addonlisterv1alpha1.ManagedClusterAddOnLister - hubCSRInformer certificatesinformers.Interface - hubKubeClient kubernetes.Interface addOnClient addonclient.Interface + csrControl clientcert.CSRControl recorder events.Recorder + csrIndexer cache.Indexer startRegistrationFunc func(ctx context.Context, config registrationConfig) context.CancelFunc @@ -56,9 +63,8 @@ func NewAddOnRegistrationController( addOnClient addonclient.Interface, managementKubeClient kubernetes.Interface, managedKubeClient kubernetes.Interface, - hubCSRInformer certificatesinformers.Interface, + csrControl clientcert.CSRControl, hubAddOnInformers addoninformerv1alpha1.ManagedClusterAddOnInformer, - hubCSRClient kubernetes.Interface, recorder events.Recorder, ) factory.Controller { c := &addOnRegistrationController{ @@ -68,13 +74,20 @@ func NewAddOnRegistrationController( managementKubeClient: managementKubeClient, spokeKubeClient: managedKubeClient, hubAddOnLister: hubAddOnInformers.Lister(), - hubCSRInformer: hubCSRInformer, - hubKubeClient: hubCSRClient, + csrControl: csrControl, addOnClient: addOnClient, recorder: recorder, + csrIndexer: csrControl.Informer().GetIndexer(), addOnRegistrationConfigs: map[string]map[string]registrationConfig{}, } + err := csrControl.Informer().AddIndexers(cache.Indexers{ + indexByAddon: indexByAddonFunc, + }) + if err != nil { + utilruntime.HandleError(err) + } + c.startRegistrationFunc = c.startRegistration return factory.New(). @@ -218,26 +231,23 @@ func (c *addOnRegistrationController) startRegistration(ctx context.Context, con DNSNames: []string{fmt.Sprintf("%s.addon.open-cluster-management.io", config.addOnName)}, SignerName: config.registration.SignerName, EventFilterFunc: createCSREventFilterFunc(c.clusterName, config.addOnName, config.registration.SignerName), + HaltCSRCreation: c.haltCSRCreationFunc(config.addOnName), } controllerName := fmt.Sprintf("ClientCertController@addon:%s:signer:%s", config.addOnName, config.registration.SignerName) statusUpdater := c.generateStatusUpdate(c.clusterName, config.addOnName) - clientCertController, err := clientcert.NewClientCertificateController( + clientCertController := clientcert.NewClientCertificateController( clientCertOption, csrOption, - c.hubCSRInformer, - c.hubKubeClient, + c.csrControl, kubeInformerFactory.Core().V1().Secrets(), - kubeClient, + kubeClient.CoreV1(), statusUpdater, c.recorder, controllerName, ) - if err != nil { - utilruntime.HandleError(err) - } go kubeInformerFactory.Start(ctx.Done()) go clientCertController.Run(ctx, 1) @@ -245,6 +255,21 @@ func (c *addOnRegistrationController) startRegistration(ctx context.Context, con return stopFunc } +func (c *addOnRegistrationController) haltCSRCreationFunc(addonName string) func() bool { + return func() bool { + items, err := c.csrIndexer.ByIndex(indexByAddon, fmt.Sprintf("%s/%s", c.clusterName, addonName)) + if err != nil { + return false + } + + if len(items) >= addonCSRThreshold { + return true + } + + return false + } +} + func (c *addOnRegistrationController) generateStatusUpdate(clusterName, addonName string) clientcert.StatusUpdateFunc { return func(ctx context.Context, cond metav1.Condition) error { _, _, updatedErr := helpers.UpdateManagedClusterAddOnStatus( @@ -293,6 +318,25 @@ func (c *addOnRegistrationController) cleanup(ctx context.Context, addOnName str return nil } +func indexByAddonFunc(obj interface{}) ([]string, error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + + cluster, ok := accessor.GetLabels()[clientcert.ClusterNameLabel] + if !ok { + return []string{}, nil + } + + addon, ok := accessor.GetLabels()[clientcert.AddonNameLabel] + if !ok { + return []string{}, nil + } + + return []string{fmt.Sprintf("%s/%s", cluster, addon)}, nil +} + func createCSREventFilterFunc(clusterName, addOnName, signerName string) factory.EventFilterFunc { return func(obj interface{}) bool { accessor, err := meta.Accessor(obj) diff --git a/pkg/spoke/managedcluster/registration.go b/pkg/spoke/managedcluster/registration.go index b304e563f..15fe31688 100644 --- a/pkg/spoke/managedcluster/registration.go +++ b/pkg/spoke/managedcluster/registration.go @@ -11,9 +11,10 @@ import ( certificates "k8s.io/api/certificates/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - certificatesinformers "k8s.io/client-go/informers/certificates" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" certutil "k8s.io/client-go/util/cert" clientset "open-cluster-management.io/api/client/cluster/clientset/versioned" @@ -22,6 +23,13 @@ import ( "open-cluster-management.io/registration/pkg/hub/user" ) +const ( + indexByCluster = "indexByCluster" + + // TODO(qiujian16) expose it if necessary in the future. + clusterCSRThreshold = 10 +) + // NewClientCertForHubController returns a controller to // 1). Create a new client certificate and build a hub kubeconfig for the registration agent; // 2). Or rotate the client certificate referenced by the hub kubeconfig before it become expired; @@ -32,13 +40,18 @@ func NewClientCertForHubController( clientCertSecretName string, kubeconfigData []byte, spokeSecretInformer corev1informers.SecretInformer, - hubCSRInformer certificatesinformers.Interface, + csrControl clientcert.CSRControl, spokeKubeClient kubernetes.Interface, - hubKubeClient kubernetes.Interface, statusUpdater clientcert.StatusUpdateFunc, recorder events.Recorder, controllerName string, -) (factory.Controller, error) { +) factory.Controller { + err := csrControl.Informer().AddIndexers(cache.Indexers{ + indexByCluster: indexByClusterFunc, + }) + if err != nil { + utilruntime.HandleError(err) + } clientCertOption := clientcert.ClientCertOption{ SecretNamespace: clientCertSecretNamespace, SecretName: clientCertSecretName, @@ -75,24 +88,44 @@ func NewClientCertForHubController( return false } + // should not contain addon key + _, ok := labels[clientcert.AddonNameLabel] + if ok { + return false + } + // only enqueue csr whose name starts with the cluster name return strings.HasPrefix(accessor.GetName(), fmt.Sprintf("%s-", clusterName)) }, + HaltCSRCreation: haltCSRCreationFunc(csrControl.Informer().GetIndexer(), clusterName), } return clientcert.NewClientCertificateController( clientCertOption, csrOption, - hubCSRInformer, - hubKubeClient, + csrControl, spokeSecretInformer, - spokeKubeClient, + spokeKubeClient.CoreV1(), statusUpdater, recorder, controllerName, ) } +func haltCSRCreationFunc(indexer cache.Indexer, clusterName string) func() bool { + return func() bool { + items, err := indexer.ByIndex(indexByCluster, clusterName) + if err != nil { + return false + } + + if len(items) >= clusterCSRThreshold { + return true + } + return false + } +} + func GenerateBootstrapStatusUpdater() clientcert.StatusUpdateFunc { return func(ctx context.Context, cond metav1.Condition) error { return nil @@ -131,3 +164,23 @@ func GetClusterAgentNamesFromCertificate(certData []byte) (clusterName, agentNam return "", "", nil } + +func indexByClusterFunc(obj interface{}) ([]string, error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + + cluster, ok := accessor.GetLabels()[clientcert.ClusterNameLabel] + if !ok { + return []string{}, nil + } + + // should not contain addon key + _, ok = accessor.GetLabels()[clientcert.AddonNameLabel] + if !ok { + return []string{}, nil + } + + return []string{cluster}, nil +} diff --git a/pkg/spoke/spokeagent.go b/pkg/spoke/spokeagent.go index f5a355a6a..b4fe924c3 100644 --- a/pkg/spoke/spokeagent.go +++ b/pkg/spoke/spokeagent.go @@ -204,22 +204,23 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext return err } + csrControl, err := clientcert.NewCSRControl(bootstrapInformerFactory.Certificates(), bootstrapKubeClient) + if err != nil { + return err + } + controllerName := fmt.Sprintf("BootstrapClientCertController@cluster:%s", o.ClusterName) - clientCertForHubController, err := managedcluster.NewClientCertForHubController( + clientCertForHubController := managedcluster.NewClientCertForHubController( o.ClusterName, o.AgentName, o.ComponentNamespace, o.HubKubeconfigSecret, kubeconfigData, // store the secret in the cluster where the agent pod runs bootstrapNamespacedManagementKubeInformerFactory.Core().V1().Secrets(), - bootstrapInformerFactory.Certificates(), + csrControl, managementKubeClient, - bootstrapKubeClient, managedcluster.GenerateBootstrapStatusUpdater(), controllerContext.EventRecorder, controllerName, ) - if err != nil { - return err - } bootstrapCtx, stopBootstrap := context.WithCancel(ctx) @@ -288,15 +289,19 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext return err } + csrControl, err := clientcert.NewCSRControl(hubKubeInformerFactory.Certificates(), hubKubeClient) + if err != nil { + return err + } + // create another ClientCertForHubController for client certificate rotation controllerName := fmt.Sprintf("ClientCertController@cluster:%s", o.ClusterName) - clientCertForHubController, err := managedcluster.NewClientCertForHubController( + clientCertForHubController := managedcluster.NewClientCertForHubController( o.ClusterName, o.AgentName, o.ComponentNamespace, o.HubKubeconfigSecret, kubeconfigData, namespacedManagementKubeInformerFactory.Core().V1().Secrets(), - hubKubeInformerFactory.Certificates(), + csrControl, managementKubeClient, - hubKubeClient, managedcluster.GenerateStatusUpdater(hubClusterClient, o.ClusterName), controllerContext.EventRecorder, controllerName, @@ -370,9 +375,8 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext addOnClient, managementKubeClient, spokeKubeClient, - hubKubeInformerFactory.Certificates(), + csrControl, addOnInformerFactory.Addon().V1alpha1().ManagedClusterAddOns(), - hubKubeClient, controllerContext.EventRecorder, ) } diff --git a/test/integration/addon_registration_test.go b/test/integration/addon_registration_test.go index 9cb1f7f9f..31b858a2d 100644 --- a/test/integration/addon_registration_test.go +++ b/test/integration/addon_registration_test.go @@ -385,6 +385,53 @@ var _ = ginkgo.Describe("Addon Registration", func() { return !reflect.DeepEqual(secret.Data, newSecret.Data) }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) }) + + ginkgo.It("should stop addon client cert update if too frequent", func() { + assertSuccessClusterBootstrap() + signerName := "kubernetes.io/kube-apiserver-client" + assertSuccessAddOnBootstrap(signerName) + + // update subject for 15 times + for i := 1; i <= 15; i++ { + addOn, err := addOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + addOn.Status = addonv1alpha1.ManagedClusterAddOnStatus{ + Registrations: []addonv1alpha1.RegistrationConfig{ + { + SignerName: addOn.Status.Registrations[0].SignerName, + Subject: addonv1alpha1.Subject{ + User: fmt.Sprintf("test-%d", i), + }, + }, + }, + } + _, err = addOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).UpdateStatus(context.TODO(), addOn, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // sleep 1 second to ensure controller issue a new csr. + time.Sleep(1 * time.Second) + } + + ginkgo.By("CSR should not exceed 10") + csrs, err := kubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", clientcert.ClusterNameLabel, managedClusterName, clientcert.AddonNameLabel, addOnName), + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(csrs.Items) >= 10).ShouldNot(gomega.BeFalse()) + + gomega.Eventually(func() error { + addOn, err := addOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + + if meta.IsStatusConditionFalse(addOn.Status.Conditions, "ClusterCertificateRotated") { + return nil + } + + return fmt.Errorf("addon status is not correct, got %v", addOn.Status.Conditions) + }, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed()) + + }) }) func getSecretName(addOnName, signerName string) string {