From 778fb4bcc28ac5c68cfb623ec05b6f983b0b0e2a Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Fri, 17 Dec 2021 14:09:49 +0100 Subject: [PATCH] fix: starting all controllers only when certificates are generated This is going to solve the issue when upgrading Capsule =v0.1.0: due to a resource reflector many warning were polluting the reconciliation loop and causing unmarshaling errors. Additionally, just the CA secret was checked before starting the Operator, when also the TLS is requested for the webhooks, along with the `/convert` one that is used for the CR version conversion. --- controllers/secret/ca.go | 2 +- controllers/secret/const.go | 2 +- controllers/secret/tls.go | 4 +- main.go | 167 +++++++++++++++++++----------------- 4 files changed, 92 insertions(+), 83 deletions(-) diff --git a/controllers/secret/ca.go b/controllers/secret/ca.go index 5f5688b4..d95bc1e4 100644 --- a/controllers/secret/ca.go +++ b/controllers/secret/ca.go @@ -189,7 +189,7 @@ func (r CAReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl tls := &corev1.Secret{} err = r.Get(ctx, types.NamespacedName{ Namespace: r.Namespace, - Name: tlsSecretName, + Name: TLSSecretName, }, tls) if err != nil { r.Log.Error(err, "Capsule TLS Secret missing") diff --git a/controllers/secret/const.go b/controllers/secret/const.go index 8f55ec3c..514eb92b 100644 --- a/controllers/secret/const.go +++ b/controllers/secret/const.go @@ -8,5 +8,5 @@ const ( privateKeySecretKey = "tls.key" CASecretName = "capsule-ca" - tlsSecretName = "capsule-tls" + TLSSecretName = "capsule-tls" ) diff --git a/controllers/secret/tls.go b/controllers/secret/tls.go index 0664d161..06b85c72 100644 --- a/controllers/secret/tls.go +++ b/controllers/secret/tls.go @@ -33,7 +33,7 @@ type TLSReconciler struct { func (r *TLSReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Secret{}, forOptionPerInstanceName(tlsSecretName)). + For(&corev1.Secret{}, forOptionPerInstanceName(TLSSecretName)). Complete(r) } @@ -112,7 +112,7 @@ func (r TLSReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctr return reconcile.Result{}, err } - if instance.Name == tlsSecretName && res == controllerutil.OperationResultUpdated { + if instance.Name == TLSSecretName && res == controllerutil.OperationResultUpdated { r.Log.Info("Capsule TLS certificates has been updated, Controller pods must be restarted to load new certificate") hostname, _ := os.Hostname() diff --git a/main.go b/main.go index 2dcaaae1..49e9ded5 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilVersion "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -22,8 +23,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/clastix/capsule/pkg/webhook/node" - capsulev1alpha1 "github.com/clastix/capsule/api/v1alpha1" capsulev1beta1 "github.com/clastix/capsule/api/v1beta1" configcontroller "github.com/clastix/capsule/controllers/config" @@ -37,6 +36,7 @@ import ( "github.com/clastix/capsule/pkg/webhook/ingress" namespacewebhook "github.com/clastix/capsule/pkg/webhook/namespace" "github.com/clastix/capsule/pkg/webhook/networkpolicy" + "github.com/clastix/capsule/pkg/webhook/node" "github.com/clastix/capsule/pkg/webhook/ownerreference" "github.com/clastix/capsule/pkg/webhook/pod" "github.com/clastix/capsule/pkg/webhook/pvc" @@ -44,7 +44,6 @@ import ( "github.com/clastix/capsule/pkg/webhook/service" "github.com/clastix/capsule/pkg/webhook/tenant" "github.com/clastix/capsule/pkg/webhook/utils" - // +kubebuilder:scaffold:imports ) var ( @@ -123,12 +122,6 @@ func main() { os.Exit(1) } - kubeVersion, err := utils.GetK8sVersion() - if err != nil { - setupLog.Error(err, "unable to get kubernetes version") - os.Exit(1) - } - _ = manager.AddReadyzCheck("ping", healthz.Ping) _ = manager.AddHealthzCheck("ping", healthz.Ping) @@ -154,75 +147,6 @@ func main() { os.Exit(1) } - // +kubebuilder:scaffold:builder - - cfg := configuration.NewCapsuleConfiguration(manager.GetClient(), configurationName) - - // webhooks: the order matters, don't change it and just append - webhooksList := append( - make([]webhook.Webhook, 0), - route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()), - route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())), - route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), - route.PVC(pvc.Handler()), - route.Service(service.Handler()), - route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())), - route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler()), - route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))), - route.Cordoning(tenant.CordoningHandler(cfg)), - route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), - ) - - nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion) - if !nodeWebhookSupported { - setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735") - } - - if err = webhook.Register(manager, webhooksList...); err != nil { - setupLog.Error(err, "unable to setup webhooks") - os.Exit(1) - } - - rbacManager := &rbaccontroller.Manager{ - Log: ctrl.Log.WithName("controllers").WithName("Rbac"), - Configuration: cfg, - } - if err = manager.Add(rbacManager); err != nil { - setupLog.Error(err, "unable to create cluster roles") - os.Exit(1) - } - if err = rbacManager.SetupWithManager(manager, configurationName); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Rbac") - os.Exit(1) - } - - if err = (&servicelabelscontroller.ServicesLabelsReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"), - }).SetupWithManager(manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels") - os.Exit(1) - } - if err = (&servicelabelscontroller.EndpointsLabelsReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"), - }).SetupWithManager(manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels") - os.Exit(1) - } - if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"), - VersionMinor: kubeVersion.Minor(), - VersionMajor: kubeVersion.Major(), - }).SetupWithManager(manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") - } - - if err = (&configcontroller.Manager{ - Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"), - }).SetupWithManager(manager, configurationName); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") - os.Exit(1) - } - clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) if err != nil { setupLog.Error(err, "unable to create kubernetes clientset") @@ -235,7 +159,13 @@ func main() { os.Exit(1) } - if len(ca.Data) > 0 { + tls, err := clientset.CoreV1().Secrets(namespace).Get(ctx, secretcontroller.TLSSecretName, metav1.GetOptions{}) + if err != nil { + setupLog.Error(err, "unable to get Capsule TLS secret") + os.Exit(1) + } + + if len(ca.Data) > 0 && len(tls.Data) > 0 { if err = (&tenantcontroller.Manager{ Client: manager.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("Tenant"), @@ -245,6 +175,7 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Tenant") os.Exit(1) } + if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil { setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant") os.Exit(1) @@ -254,6 +185,84 @@ func main() { setupLog.Error(err, "unable to setup indexers") os.Exit(1) } + + var kubeVersion *utilVersion.Version + kubeVersion, err = utils.GetK8sVersion() + if err != nil { + setupLog.Error(err, "unable to get kubernetes version") + os.Exit(1) + } + + cfg := configuration.NewCapsuleConfiguration(manager.GetClient(), configurationName) + + // webhooks: the order matters, don't change it and just append + webhooksList := append( + make([]webhook.Webhook, 0), + route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()), + route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())), + route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), + route.PVC(pvc.Handler()), + route.Service(service.Handler()), + route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())), + route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler()), + route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))), + route.Cordoning(tenant.CordoningHandler(cfg)), + route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), + ) + + nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion) + if !nodeWebhookSupported { + setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735") + } + + if err = webhook.Register(manager, webhooksList...); err != nil { + setupLog.Error(err, "unable to setup webhooks") + os.Exit(1) + } + + rbacManager := &rbaccontroller.Manager{ + Log: ctrl.Log.WithName("controllers").WithName("Rbac"), + Configuration: cfg, + } + + if err = manager.Add(rbacManager); err != nil { + setupLog.Error(err, "unable to create cluster roles") + os.Exit(1) + } + + if err = rbacManager.SetupWithManager(manager, configurationName); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Rbac") + os.Exit(1) + } + + if err = (&servicelabelscontroller.ServicesLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"), + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels") + os.Exit(1) + } + + if err = (&servicelabelscontroller.EndpointsLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"), + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels") + os.Exit(1) + } + + if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"), + VersionMinor: kubeVersion.Minor(), + VersionMajor: kubeVersion.Major(), + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") + } + + if err = (&configcontroller.Manager{ + Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"), + }).SetupWithManager(manager, configurationName); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") + os.Exit(1) + } } else { setupLog.Info("skip registering a tenant controller, missing CA secret") }