feat: refactor capsule TLS certificates management

api/v1alpha1/capsuleconfiguration_annotations.go

@@ -9,4 +9,5 @@ const (
 	TLSSecretNameAnnotation                  = "capsule.clastix.io/tls-secret-name"
 	MutatingWebhookConfigurationName         = "capsule.clastix.io/mutating-webhook-configuration-name"
 	ValidatingWebhookConfigurationName       = "capsule.clastix.io/validating-webhook-configuration-name"
+	GenerateCertificatesAnnotationName       = "capsule.clastix.io/generate-certificates"
 )

charts/capsule/templates/ca.yaml

@@ -1,12 +0,0 @@
-{{- if not .Values.certManager.generateCertificates }}
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    {{- include "capsule.labels" . | nindent 4 }}
-  {{- with .Values.customAnnotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  name: {{ include "capsule.secretCaName" . }}
-{{- end }}

controllers/secret/ca.go

@@ -1,258 +0,0 @@
-// Copyright 2020-2021 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package secret
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"time"
-
-	"github.com/go-logr/logr"
-	"golang.org/x/sync/errgroup"
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
-	corev1 "k8s.io/api/core/v1"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/util/retry"
-	"k8s.io/utils/pointer"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/builder"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/controller-runtime/pkg/source"
-
-	"github.com/clastix/capsule/pkg/cert"
-	"github.com/clastix/capsule/pkg/configuration"
-)
-
-type CAReconciler struct {
-	client.Client
-	Log           logr.Logger
-	Scheme        *runtime.Scheme
-	Namespace     string
-	Configuration configuration.Configuration
-}
-
-func (r *CAReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	enqueueFn := handler.EnqueueRequestsFromMapFunc(func(client.Object) []reconcile.Request {
-		return []reconcile.Request{
-			{
-				NamespacedName: types.NamespacedName{
-					Namespace: r.Namespace,
-					Name:      r.Configuration.CASecretName(),
-				},
-			},
-		}
-	})
-
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&corev1.Secret{}).
-		Watches(source.NewKindWithCache(&admissionregistrationv1.ValidatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
-			return object.GetName() == r.Configuration.ValidatingWebhookConfigurationName()
-		}))).
-		Watches(source.NewKindWithCache(&admissionregistrationv1.MutatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
-			return object.GetName() == r.Configuration.MutatingWebhookConfigurationName()
-		}))).
-		Complete(r)
-}
-
-// By default helm doesn't allow to use templates in CRD (https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you).
-// In order to overcome this, we are setting conversion strategy in helm chart to None, and then update it with CA and namespace information.
-func (r *CAReconciler) UpdateCustomResourceDefinition(ctx context.Context, caBundle []byte) error {
-	return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
-		crd := &apiextensionsv1.CustomResourceDefinition{}
-		err = r.Get(ctx, types.NamespacedName{Name: "tenants.capsule.clastix.io"}, crd)
-		if err != nil {
-			r.Log.Error(err, "cannot retrieve CustomResourceDefinition")
-
-			return err
-		}
-
-		_, err = controllerutil.CreateOrUpdate(ctx, r.Client, crd, func() error {
-			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
-				Strategy: "Webhook",
-				Webhook: &apiextensionsv1.WebhookConversion{
-					ClientConfig: &apiextensionsv1.WebhookClientConfig{
-						Service: &apiextensionsv1.ServiceReference{
-							Namespace: r.Namespace,
-							Name:      "capsule-webhook-service",
-							Path:      pointer.StringPtr("/convert"),
-							Port:      pointer.Int32Ptr(443),
-						},
-						CABundle: caBundle,
-					},
-					ConversionReviewVersions: []string{"v1alpha1", "v1beta1"},
-				},
-			}
-
-			return nil
-		})
-
-		return err
-	})
-}
-
-//nolint:dupl
-func (r CAReconciler) UpdateValidatingWebhookConfiguration(ctx context.Context, caBundle []byte) error {
-	return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
-		vw := &admissionregistrationv1.ValidatingWebhookConfiguration{}
-		err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.ValidatingWebhookConfigurationName()}, vw)
-		if err != nil {
-			r.Log.Error(err, "cannot retrieve ValidatingWebhookConfiguration")
-
-			return err
-		}
-		for i, w := range vw.Webhooks {
-			// Updating CABundle only in case of an internal service reference
-			if w.ClientConfig.Service != nil {
-				vw.Webhooks[i].ClientConfig.CABundle = caBundle
-			}
-		}
-
-		return r.Update(ctx, vw, &client.UpdateOptions{})
-	})
-}
-
-//nolint:dupl
-func (r CAReconciler) UpdateMutatingWebhookConfiguration(ctx context.Context, caBundle []byte) error {
-	return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
-		mw := &admissionregistrationv1.MutatingWebhookConfiguration{}
-		err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.MutatingWebhookConfigurationName()}, mw)
-		if err != nil {
-			r.Log.Error(err, "cannot retrieve MutatingWebhookConfiguration")
-
-			return err
-		}
-		for i, w := range mw.Webhooks {
-			// Updating CABundle only in case of an internal service reference
-			if w.ClientConfig.Service != nil {
-				mw.Webhooks[i].ClientConfig.CABundle = caBundle
-			}
-		}
-
-		return r.Update(ctx, mw, &client.UpdateOptions{})
-	})
-}
-
-func (r CAReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
-	var err error
-
-	if request.Name != r.Configuration.CASecretName() {
-		return ctrl.Result{}, nil
-	}
-
-	r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
-	r.Log.Info("Reconciling CA Secret")
-
-	// Fetch the CA instance
-	instance := &corev1.Secret{}
-
-	if err = r.Client.Get(ctx, request.NamespacedName, instance); err != nil {
-		// Error reading the object - requeue the request.
-		return reconcile.Result{}, err
-	}
-
-	var ca cert.CA
-
-	var rq time.Duration
-
-	ca, err = getCertificateAuthority(ctx, r.Client, r.Namespace, r.Configuration.CASecretName())
-	if err != nil && errors.Is(err, MissingCaError{}) {
-		ca, err = cert.GenerateCertificateAuthority()
-		if err != nil {
-			return reconcile.Result{}, err
-		}
-	} else if err != nil {
-		return reconcile.Result{}, err
-	}
-
-	r.Log.Info("Handling CA Secret")
-
-	rq, err = ca.ExpiresIn(time.Now())
-	if err != nil {
-		r.Log.Info("CA is expired, cleaning to obtain a new one")
-
-		instance.Data = map[string][]byte{}
-	} else {
-		r.Log.Info("Updating CA secret with new PEM and RSA")
-
-		var crt *bytes.Buffer
-		var key *bytes.Buffer
-		crt, _ = ca.CACertificatePem()
-		key, _ = ca.CAPrivateKeyPem()
-
-		instance.Data = map[string][]byte{
-			corev1.TLSCertKey:       crt.Bytes(),
-			corev1.TLSPrivateKeyKey: key.Bytes(),
-		}
-
-		group := new(errgroup.Group)
-		group.Go(func() error {
-			return r.UpdateMutatingWebhookConfiguration(ctx, crt.Bytes())
-		})
-		group.Go(func() error {
-			return r.UpdateValidatingWebhookConfiguration(ctx, crt.Bytes())
-		})
-		group.Go(func() error {
-			return r.UpdateCustomResourceDefinition(ctx, crt.Bytes())
-		})
-
-		if err = group.Wait(); err != nil {
-			return reconcile.Result{}, err
-		}
-	}
-
-	var res controllerutil.OperationResult
-
-	t := &corev1.Secret{ObjectMeta: instance.ObjectMeta}
-
-	res, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error {
-		t.Data = instance.Data
-
-		return nil
-	})
-	if err != nil {
-		r.Log.Error(err, "cannot update Capsule TLS")
-
-		return reconcile.Result{}, err
-	}
-
-	if res == controllerutil.OperationResultUpdated {
-		r.Log.Info("Capsule CA has been updated, we need to trigger TLS update too")
-
-		tls := &corev1.Secret{}
-		err = r.Get(ctx, types.NamespacedName{
-			Namespace: r.Namespace,
-			Name:      r.Configuration.TLSSecretName(),
-		}, tls)
-
-		if err != nil {
-			r.Log.Error(err, "Capsule TLS Secret missing")
-		}
-
-		err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
-			_, err = controllerutil.CreateOrUpdate(ctx, r.Client, tls, func() error {
-				tls.Data = map[string][]byte{}
-
-				return nil
-			})
-
-			return err
-		})
-		if err != nil {
-			r.Log.Error(err, "Cannot clean Capsule TLS Secret due to CA update")
-
-			return reconcile.Result{}, err
-		}
-	}
-
-	r.Log.Info("Reconciliation completed, processing back in " + rq.String())
-
-	return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil
-}

controllers/secret/errors.go

@@ -1,10 +0,0 @@
-// Copyright 2020-2021 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package secret
-
-type MissingCaError struct{}
-
-func (MissingCaError) Error() string {
-	return "CA has not been created yet, please generate a new"
-}

controllers/secret/reconciler.go

@@ -1,34 +0,0 @@
-// Copyright 2020-2021 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package secret
-
-import (
-	"context"
-	"fmt"
-
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/clastix/capsule/pkg/cert"
-)
-
-func getCertificateAuthority(ctx context.Context, client client.Client, namespace, name string) (ca cert.CA, err error) {
-	instance := &corev1.Secret{}
-
-	if err = client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, instance); err != nil {
-		return nil, fmt.Errorf("missing secret %s, cannot reconcile", name)
-	}
-
-	if instance.Data == nil {
-		return nil, MissingCaError{}
-	}
-
-	ca, err = cert.NewCertificateAuthorityFromBytes(instance.Data[corev1.TLSCertKey], instance.Data[corev1.TLSPrivateKeyKey])
-	if err != nil {
-		return
-	}
-
-	return
-}

controllers/secret/tls.go

@@ -1,172 +0,0 @@
-// Copyright 2020-2021 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package secret
-
-import (
-	"bytes"
-	"context"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/go-logr/logr"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-
-	"github.com/clastix/capsule/pkg/cert"
-	"github.com/clastix/capsule/pkg/configuration"
-)
-
-type TLSReconciler struct {
-	client.Client
-	Log           logr.Logger
-	Scheme        *runtime.Scheme
-	Namespace     string
-	Configuration configuration.Configuration
-}
-
-func (r *TLSReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&corev1.Secret{}).
-		Complete(r)
-}
-
-func (r TLSReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
-	var err error
-
-	if request.Name != r.Configuration.TLSSecretName() {
-		return ctrl.Result{}, nil
-	}
-
-	r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
-	r.Log.Info("Reconciling TLS Secret")
-
-	// Fetch the Secret instance
-	instance := &corev1.Secret{}
-	if err = r.Get(ctx, request.NamespacedName, instance); err != nil {
-		// Error reading the object - requeue the request.
-		return reconcile.Result{}, err
-	}
-
-	var ca cert.CA
-
-	var rq time.Duration
-
-	ca, err = getCertificateAuthority(ctx, r.Client, r.Namespace, r.Configuration.CASecretName())
-	if err != nil {
-		return reconcile.Result{}, err
-	}
-
-	var shouldCreate bool
-
-	for _, key := range []string{corev1.TLSCertKey, corev1.TLSPrivateKeyKey} {
-		if _, ok := instance.Data[key]; !ok {
-			shouldCreate = true
-
-			break
-		}
-	}
-
-	if shouldCreate {
-		r.Log.Info("Missing Capsule TLS certificate")
-
-		rq = 6 * 30 * 24 * time.Hour
-
-		opts := cert.NewCertOpts(time.Now().Add(rq), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace))
-
-		var crt, key *bytes.Buffer
-
-		if crt, key, err = ca.GenerateCertificate(opts); err != nil {
-			r.Log.Error(err, "Cannot generate new TLS certificate")
-
-			return reconcile.Result{}, err
-		}
-
-		instance.Data = map[string][]byte{
-			corev1.TLSCertKey:       crt.Bytes(),
-			corev1.TLSPrivateKeyKey: key.Bytes(),
-		}
-	} else {
-		var c *x509.Certificate
-		var b *pem.Block
-		b, _ = pem.Decode(instance.Data[corev1.TLSCertKey])
-		c, err = x509.ParseCertificate(b.Bytes)
-		if err != nil {
-			r.Log.Error(err, "cannot parse Capsule TLS")
-
-			return reconcile.Result{}, err
-		}
-
-		rq = time.Until(c.NotAfter)
-
-		err = ca.ValidateCert(c)
-		if err != nil {
-			r.Log.Info("Capsule TLS is expired or invalid, cleaning to obtain a new one")
-			instance.Data = map[string][]byte{}
-		}
-	}
-
-	var res controllerutil.OperationResult
-
-	t := &corev1.Secret{ObjectMeta: instance.ObjectMeta}
-
-	res, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error {
-		t.Data = instance.Data
-
-		return nil
-	})
-	if err != nil {
-		r.Log.Error(err, "cannot update Capsule TLS")
-
-		return reconcile.Result{}, err
-	}
-	// nolint:nestif
-	if instance.Name == r.Configuration.TLSSecretName() && res == controllerutil.OperationResultUpdated {
-		r.Log.Info("Capsule TLS certificates has been updated, Controller pods must be restarted to load new certificate")
-
-		hostname, _ := os.Hostname()
-
-		leaderPod := &corev1.Pod{}
-
-		if err = r.Client.Get(ctx, types.NamespacedName{Namespace: os.Getenv("NAMESPACE"), Name: hostname}, leaderPod); err != nil {
-			r.Log.Error(err, "cannot retrieve the leader Pod, probably running in out of the cluster mode")
-
-			return reconcile.Result{}, nil
-		}
-
-		podList := &corev1.PodList{}
-		if err = r.Client.List(ctx, podList, client.MatchingLabels(leaderPod.ObjectMeta.Labels)); err != nil {
-			r.Log.Error(err, "cannot retrieve list of Capsule pods requiring restart upon TLS update")
-
-			return reconcile.Result{}, nil
-		}
-
-		for _, p := range podList.Items {
-			nonLeaderPod := p
-			// Skipping this Pod, must be deleted at the end
-			if nonLeaderPod.GetName() == leaderPod.GetName() {
-				continue
-			}
-
-			if err = r.Client.Delete(ctx, &nonLeaderPod); err != nil {
-				r.Log.Error(err, "cannot delete the non-leader Pod due to TLS update")
-			}
-		}
-
-		if err = r.Client.Delete(ctx, leaderPod); err != nil {
-			r.Log.Error(err, "cannot delete the leader Pod due to TLS update")
-		}
-	}
-
-	r.Log.Info("Reconciliation completed, processing back in " + rq.String())
-
-	return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil
-}

controllers/tls/manager.go

@@ -0,0 +1,336 @@
+// Copyright 2020-2021 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package tls
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/go-logr/logr"
+	"golang.org/x/sync/errgroup"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/utils/pointer"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/clastix/capsule/controllers/utils"
+	"github.com/clastix/capsule/pkg/cert"
+	"github.com/clastix/capsule/pkg/configuration"
+)
+
+const (
+	certificateExpirationThreshold     = 3 * 24 * time.Hour
+	certificateReconciliationThreshold = 4 * 24 * time.Hour
+	certificateValidity                = 6 * 30 * 24 * time.Hour
+	PodUpdateAnnotationName            = "capsule.clastix.io/updated"
+)
+
+type Reconciler struct {
+	client.Client
+	Log           logr.Logger
+	Scheme        *runtime.Scheme
+	Namespace     string
+	Configuration configuration.Configuration
+}
+
+func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
+	enqueueFn := handler.EnqueueRequestsFromMapFunc(func(client.Object) []reconcile.Request {
+		return []reconcile.Request{
+			{
+				NamespacedName: types.NamespacedName{
+					Namespace: r.Namespace,
+					Name:      r.Configuration.TLSSecretName(),
+				},
+			},
+		}
+	})
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}, utils.NamesMatchingPredicate(r.Configuration.TLSSecretName())).
+		Watches(source.NewKindWithCache(&admissionregistrationv1.ValidatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == r.Configuration.ValidatingWebhookConfigurationName()
+		}))).
+		Watches(source.NewKindWithCache(&admissionregistrationv1.MutatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == r.Configuration.MutatingWebhookConfigurationName()
+		}))).
+		Watches(source.NewKindWithCache(&apiextensionsv1.CustomResourceDefinition{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == r.Configuration.TenantCRDName()
+		}))).
+		Complete(r)
+}
+
+func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
+	r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
+
+	// Fetch the CA instance
+	certSecret := &corev1.Secret{}
+
+	if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil {
+		// Error reading the object - requeue the request.
+		return reconcile.Result{}, err
+	}
+
+	if r.shouldUpdateCertificate(certSecret) {
+		r.Log.Info("Generating new TLS certificate")
+
+		ca, err := cert.GenerateCertificateAuthority()
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+
+		opts := cert.NewCertOpts(time.Now().Add(certificateValidity), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace))
+
+		crt, key, err := ca.GenerateCertificate(opts)
+		if err != nil {
+			r.Log.Error(err, "Cannot generate new TLS certificate")
+
+			return reconcile.Result{}, err
+		}
+
+		caCrt, _ := ca.CACertificatePem()
+
+		certSecret.Data = map[string][]byte{
+			corev1.TLSCertKey:              crt.Bytes(),
+			corev1.TLSPrivateKeyKey:        key.Bytes(),
+			corev1.ServiceAccountRootCAKey: caCrt.Bytes(),
+		}
+
+		t := &corev1.Secret{ObjectMeta: certSecret.ObjectMeta}
+
+		_, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error {
+			t.Data = certSecret.Data
+
+			return nil
+		})
+		if err != nil {
+			r.Log.Error(err, "cannot update Capsule TLS")
+
+			return reconcile.Result{}, err
+		}
+	}
+
+	var caBundle []byte
+
+	var ok bool
+
+	if caBundle, ok = certSecret.Data[corev1.ServiceAccountRootCAKey]; !ok {
+		return reconcile.Result{}, fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
+	}
+
+	operatorPods, err := r.getOperatorPods(ctx)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	r.Log.Info("Updating caBundle in webhooks and crd")
+
+	group := new(errgroup.Group)
+	group.Go(func() error {
+		return r.updateMutatingWebhookConfiguration(ctx, caBundle)
+	})
+	group.Go(func() error {
+		return r.updateValidatingWebhookConfiguration(ctx, caBundle)
+	})
+	group.Go(func() error {
+		return r.updateCustomResourceDefinition(ctx, caBundle)
+	})
+
+	r.Log.Info("Updating capsule operator pods")
+
+	for _, pod := range operatorPods.Items {
+		p := pod
+
+		group.Go(func() error {
+			return r.updateOperatorPod(ctx, p)
+		})
+	}
+
+	if err := group.Wait(); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if r.Configuration.GenerateCertificates() {
+		certificate, err := cert.GetCertificateFromBytes(certSecret.Data[corev1.TLSCertKey])
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+
+		now := time.Now()
+
+		rq := (time.Duration(certificate.NotAfter.Unix()-now.Unix()) * time.Second) - certificateReconciliationThreshold
+
+		r.Log.Info("Reconciliation completed, processing back in " + rq.String())
+
+		return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (r Reconciler) shouldUpdateCertificate(secret *corev1.Secret) bool {
+	if !r.Configuration.GenerateCertificates() {
+		r.Log.Info("Skipping TLS certificate generation as it is disabled in CapsuleConfiguration")
+
+		return false
+	}
+
+	if _, ok := secret.Data[corev1.ServiceAccountRootCAKey]; !ok {
+		return true
+	}
+
+	certificate, key, err := cert.GetCertificateWithPrivateKeyFromBytes(secret.Data[corev1.TLSCertKey], secret.Data[corev1.TLSPrivateKeyKey])
+	if err != nil {
+		return true
+	}
+
+	if err := cert.ValidateCertificate(certificate, key, certificateExpirationThreshold); err != nil {
+		r.Log.Error(err, "failed to validate certificate, generating new one")
+
+		return true
+	}
+
+	r.Log.Info("Skipping TLS certificate generation as it is still valid")
+
+	return false
+}
+
+// By default helm doesn't allow to use templates in CRD (https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you).
+// In order to overcome this, we are setting conversion strategy in helm chart to None, and then update it with CA and namespace information.
+func (r *Reconciler) updateCustomResourceDefinition(ctx context.Context, caBundle []byte) error {
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		crd := &apiextensionsv1.CustomResourceDefinition{}
+		err = r.Get(ctx, types.NamespacedName{Name: "tenants.capsule.clastix.io"}, crd)
+		if err != nil {
+			r.Log.Error(err, "cannot retrieve CustomResourceDefinition")
+
+			return err
+		}
+
+		_, err = controllerutil.CreateOrUpdate(ctx, r.Client, crd, func() error {
+			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
+				Strategy: "Webhook",
+				Webhook: &apiextensionsv1.WebhookConversion{
+					ClientConfig: &apiextensionsv1.WebhookClientConfig{
+						Service: &apiextensionsv1.ServiceReference{
+							Namespace: r.Namespace,
+							Name:      "capsule-webhook-service",
+							Path:      pointer.StringPtr("/convert"),
+							Port:      pointer.Int32Ptr(443),
+						},
+						CABundle: caBundle,
+					},
+					ConversionReviewVersions: []string{"v1alpha1", "v1beta1"},
+				},
+			}
+
+			return nil
+		})
+
+		return err
+	})
+}
+
+//nolint:dupl
+func (r Reconciler) updateValidatingWebhookConfiguration(ctx context.Context, caBundle []byte) error {
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		vw := &admissionregistrationv1.ValidatingWebhookConfiguration{}
+		err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.ValidatingWebhookConfigurationName()}, vw)
+		if err != nil {
+			r.Log.Error(err, "cannot retrieve ValidatingWebhookConfiguration")
+
+			return err
+		}
+		for i, w := range vw.Webhooks {
+			// Updating CABundle only in case of an internal service reference
+			if w.ClientConfig.Service != nil {
+				vw.Webhooks[i].ClientConfig.CABundle = caBundle
+			}
+		}
+
+		return r.Update(ctx, vw, &client.UpdateOptions{})
+	})
+}
+
+//nolint:dupl
+func (r Reconciler) updateMutatingWebhookConfiguration(ctx context.Context, caBundle []byte) error {
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		mw := &admissionregistrationv1.MutatingWebhookConfiguration{}
+		err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.MutatingWebhookConfigurationName()}, mw)
+		if err != nil {
+			r.Log.Error(err, "cannot retrieve MutatingWebhookConfiguration")
+
+			return err
+		}
+		for i, w := range mw.Webhooks {
+			// Updating CABundle only in case of an internal service reference
+			if w.ClientConfig.Service != nil {
+				mw.Webhooks[i].ClientConfig.CABundle = caBundle
+			}
+		}
+
+		return r.Update(ctx, mw, &client.UpdateOptions{})
+	})
+}
+
+func (r Reconciler) updateOperatorPod(ctx context.Context, pod corev1.Pod) error {
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Need to get latest version of pod
+		p := &corev1.Pod{}
+
+		if err := r.Client.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, p); err != nil && !apierrors.IsNotFound(err) {
+			r.Log.Error(err, "cannot get pod", "name", pod.Name, "namespace", pod.Namespace)
+
+			return err
+		}
+
+		if p.Annotations == nil {
+			p.Annotations = map[string]string{}
+		}
+
+		p.Annotations[PodUpdateAnnotationName] = time.Now().Format(time.RFC3339Nano)
+
+		if err := r.Client.Update(ctx, p, &client.UpdateOptions{}); err != nil {
+			r.Log.Error(err, "cannot update pod", "name", pod.Name, "namespace", pod.Namespace)
+
+			return err
+		}
+
+		return nil
+	})
+}
+
+func (r Reconciler) getOperatorPods(ctx context.Context) (*corev1.PodList, error) {
+	hostname, _ := os.Hostname()
+
+	leaderPod := &corev1.Pod{}
+
+	if err := r.Client.Get(ctx, types.NamespacedName{Namespace: os.Getenv("NAMESPACE"), Name: hostname}, leaderPod); err != nil {
+		r.Log.Error(err, "cannot retrieve the leader Pod, probably running in out of the cluster mode")
+
+		return nil, err
+	}
+
+	podList := &corev1.PodList{}
+	if err := r.Client.List(ctx, podList, client.MatchingLabels(leaderPod.ObjectMeta.Labels)); err != nil {
+		r.Log.Error(err, "cannot retrieve list of Capsule pods")
+
+		return nil, err
+	}
+
+	return podList, nil
+}

main.go

@@ -28,9 +28,9 @@ import (
 	capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
 	configcontroller "github.com/clastix/capsule/controllers/config"
 	rbaccontroller "github.com/clastix/capsule/controllers/rbac"
-	secretcontroller "github.com/clastix/capsule/controllers/secret"
 	servicelabelscontroller "github.com/clastix/capsule/controllers/servicelabels"
 	tenantcontroller "github.com/clastix/capsule/controllers/tenant"
+	tlscontroller "github.com/clastix/capsule/controllers/tls"
 	"github.com/clastix/capsule/pkg/configuration"
 	"github.com/clastix/capsule/pkg/indexer"
 	"github.com/clastix/capsule/pkg/webhook"
@@ -70,15 +70,13 @@ func printVersion() {
 
 // nolint:maintidx
 func main() {
-	var enableLeaderElection, enableSecretController, version bool
+	var enableLeaderElection, version bool
 
 	var metricsAddr, namespace, configurationName string
 
 	var goFlagSet goflag.FlagSet
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-	flag.BoolVar(&enableSecretController, "enable-secret-controller", true,
-		"Enable secret controller which reconciles TLS and CA secrets for capsule webhooks.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -133,28 +131,6 @@ func main() {
 
 	cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), configurationName)
 
-	if enableSecretController {
-		if err = (&secretcontroller.CAReconciler{
-			Client:        manager.GetClient(),
-			Log:           ctrl.Log.WithName("controllers").WithName("CA"),
-			Namespace:     namespace,
-			Configuration: cfg,
-		}).SetupWithManager(manager); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "Namespace")
-			os.Exit(1)
-		}
-
-		if err = (&secretcontroller.TLSReconciler{
-			Client:        manager.GetClient(),
-			Log:           ctrl.Log.WithName("controllers").WithName("Tls"),
-			Namespace:     namespace,
-			Configuration: cfg,
-		}).SetupWithManager(manager); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "Namespace")
-			os.Exit(1)
-		}
-	}
-
 	clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie())
 	if err != nil {
 		setupLog.Error(err, "unable to create kubernetes clientset")
@@ -172,9 +148,13 @@ func main() {
 
 	directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, configurationName)
 
-	ca, err := clientset.CoreV1().Secrets(namespace).Get(ctx, directCfg.CASecretName(), metav1.GetOptions{})
-	if err != nil {
-		setupLog.Error(err, "unable to get Capsule CA secret")
+	if err = (&tlscontroller.Reconciler{
+		Client:        manager.GetClient(),
+		Log:           ctrl.Log.WithName("controllers").WithName("TLS"),
+		Namespace:     namespace,
+		Configuration: directCfg,
+	}).SetupWithManager(manager); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Namespace")
 		os.Exit(1)
 	}
 
@@ -184,7 +164,7 @@ func main() {
 		os.Exit(1)
 	}
 	// nolint:nestif
-	if len(ca.Data) > 0 && len(tls.Data) > 0 {
+	if len(tls.Data) > 0 {
 		if err = (&tenantcontroller.Manager{
 			RESTConfig: manager.GetConfig(),
 			Client:     manager.GetClient(),

pkg/cert/ca.go

@@ -12,6 +12,8 @@ import (
 	"encoding/pem"
 	"math/big"
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 type CA interface {
@@ -27,38 +29,6 @@ type CapsuleCA struct {
 	key         *rsa.PrivateKey
 }
 
-func (c CapsuleCA) ValidateCert(certificate *x509.Certificate) (err error) {
-	pool := x509.NewCertPool()
-	pool.AddCert(c.certificate)
-
-	_, err = certificate.Verify(x509.VerifyOptions{
-		Roots:       pool,
-		CurrentTime: time.Time{},
-	})
-
-	return
-}
-
-func (c CapsuleCA) isAlreadyValid(now time.Time) bool {
-	return now.After(c.certificate.NotBefore)
-}
-
-func (c CapsuleCA) isExpired(now time.Time) bool {
-	return now.Before(c.certificate.NotAfter)
-}
-
-func (c CapsuleCA) ExpiresIn(now time.Time) (time.Duration, error) {
-	if !c.isExpired(now) {
-		return time.Nanosecond, CaExpiredError{}
-	}
-
-	if !c.isAlreadyValid(now) {
-		return time.Nanosecond, CaNotYetValidError{}
-	}
-
-	return time.Duration(c.certificate.NotAfter.Unix()-now.Unix()) * time.Second, nil
-}
-
 func (c CapsuleCA) CACertificatePem() (b *bytes.Buffer, err error) {
 	var crtBytes []byte
 	crtBytes, err = x509.CreateCertificate(rand.Reader, c.certificate, c.certificate, &c.key.PublicKey, c.key)
@@ -85,6 +55,24 @@ func (c CapsuleCA) CAPrivateKeyPem() (b *bytes.Buffer, err error) {
 	})
 }
 
+func ValidateCertificate(cert *x509.Certificate, key *rsa.PrivateKey, expirationThreshold time.Duration) error {
+	if !key.PublicKey.Equal(cert.PublicKey) {
+		return errors.New("certificate signed by wrong public key")
+	}
+
+	now := time.Now()
+
+	if now.Before(cert.NotBefore) {
+		return errors.New("certificate is not valid yet")
+	}
+
+	if now.After(cert.NotAfter.Add(-expirationThreshold)) {
+		return errors.New("certificate expired or going to expire soon")
+	}
+
+	return nil
+}
+
 func GenerateCertificateAuthority() (s *CapsuleCA, err error) {
 	s = &CapsuleCA{
 		certificate: &x509.Certificate{
@@ -114,31 +102,46 @@ func GenerateCertificateAuthority() (s *CapsuleCA, err error) {
 	return
 }
 
-func NewCertificateAuthorityFromBytes(certBytes, keyBytes []byte) (s *CapsuleCA, err error) {
+func GetCertificateFromBytes(certBytes []byte) (*x509.Certificate, error) {
 	var b *pem.Block
 
 	b, _ = pem.Decode(certBytes)
 
-	var cert *x509.Certificate
+	return x509.ParseCertificate(b.Bytes)
+}
 
-	if cert, err = x509.ParseCertificate(b.Bytes); err != nil {
-		return
-	}
+func GetPrivateKeyFromBytes(keyBytes []byte) (*rsa.PrivateKey, error) {
+	var b *pem.Block
 
 	b, _ = pem.Decode(keyBytes)
 
-	var key *rsa.PrivateKey
+	return x509.ParsePKCS1PrivateKey(b.Bytes)
+}
 
-	if key, err = x509.ParsePKCS1PrivateKey(b.Bytes); err != nil {
-		return
+func GetCertificateWithPrivateKeyFromBytes(certBytes, keyBytes []byte) (*x509.Certificate, *rsa.PrivateKey, error) {
+	cert, err := GetCertificateFromBytes(certBytes)
+	if err != nil {
+		return nil, nil, err
 	}
 
-	s = &CapsuleCA{
+	key, err := GetPrivateKeyFromBytes(keyBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return cert, key, nil
+}
+
+func NewCertificateAuthorityFromBytes(certBytes, keyBytes []byte) (*CapsuleCA, error) {
+	cert, key, err := GetCertificateWithPrivateKeyFromBytes(certBytes, keyBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &CapsuleCA{
 		certificate: cert,
 		key:         key,
-	}
-
-	return
+	}, nil
 }
 
 // nolint:nakedret

pkg/cert/ca_test.go

@@ -74,40 +74,3 @@ func TestCapsuleCa_GenerateCertificate(t *testing.T) {
 		})
 	}
 }
-
-func TestCapsuleCa_IsValid(t *testing.T) {
-	type testCase struct {
-		notBefore   time.Time
-		notAfter    time.Time
-		returnError bool
-	}
-
-	tc := map[string]testCase{
-		"ok":       {time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 1), false},
-		"expired":  {time.Now().AddDate(1, 0, 0), time.Now(), true},
-		"notValid": {time.Now().AddDate(0, 0, 1), time.Now().AddDate(0, 0, 2), true},
-	}
-
-	for name, c := range tc {
-		t.Run(name, func(t *testing.T) {
-			var ca *CapsuleCA
-			var err error
-
-			ca, err = GenerateCertificateAuthority()
-			assert.Nil(t, err)
-
-			ca.certificate.NotAfter = c.notAfter
-			ca.certificate.NotBefore = c.notBefore
-
-			var w time.Duration
-			w, err = ca.ExpiresIn(time.Now())
-			if c.returnError {
-				assert.Error(t, err)
-
-				return
-			}
-			assert.Nil(t, err)
-			assert.WithinDuration(t, ca.certificate.NotAfter, time.Now().Add(w), time.Minute)
-		})
-	}
-}

pkg/configuration/client.go

@@ -6,6 +6,7 @@ package configuration
 import (
 	"context"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -62,21 +63,6 @@ func (c capsuleConfiguration) ForceTenantPrefix() bool {
 	return c.retrievalFn().Spec.ForceTenantPrefix
 }
 
-func (c capsuleConfiguration) CASecretName() (name string) {
-	name = CASecretName
-
-	if c.retrievalFn().Annotations == nil {
-		return
-	}
-
-	v, ok := c.retrievalFn().Annotations[capsulev1alpha1.CASecretNameAnnotation]
-	if ok {
-		return v
-	}
-
-	return
-}
-
 func (c capsuleConfiguration) TLSSecretName() (name string) {
 	name = TLSSecretName
 
@@ -92,6 +78,21 @@ func (c capsuleConfiguration) TLSSecretName() (name string) {
 	return
 }
 
+func (c capsuleConfiguration) GenerateCertificates() bool {
+	annotationValue, ok := c.retrievalFn().Annotations[capsulev1alpha1.GenerateCertificatesAnnotationName]
+
+	if ok {
+		value, err := strconv.ParseBool(annotationValue)
+		if err != nil {
+			return false
+		}
+
+		return value
+	}
+
+	return false
+}
+
 func (c capsuleConfiguration) MutatingWebhookConfigurationName() (name string) {
 	name = MutatingWebhookConfigurationName
 
@@ -107,6 +108,10 @@ func (c capsuleConfiguration) MutatingWebhookConfigurationName() (name string) {
 	return
 }
 
+func (c capsuleConfiguration) TenantCRDName() string {
+	return TenantCRDName
+}
+
 func (c capsuleConfiguration) ValidatingWebhookConfigurationName() (name string) {
 	name = ValidatingWebhookConfigurationName
 

pkg/configuration/configuration.go

@@ -10,19 +10,20 @@ import (
 )
 
 const (
-	CASecretName                       = "capsule-ca"
 	TLSSecretName                      = "capsule-tls"
 	MutatingWebhookConfigurationName   = "capsule-mutating-webhook-configuration"
 	ValidatingWebhookConfigurationName = "capsule-validating-webhook-configuration"
+	TenantCRDName                      = "tenants.capsule.clastix.io"
 )
 
 type Configuration interface {
 	ProtectedNamespaceRegexp() (*regexp.Regexp, error)
 	ForceTenantPrefix() bool
-	CASecretName() string
+	GenerateCertificates() bool
 	TLSSecretName() string
 	MutatingWebhookConfigurationName() string
 	ValidatingWebhookConfigurationName() string
+	TenantCRDName() string
 	UserGroups() []string
 	ForbiddenUserNodeLabels() *capsulev1beta1.ForbiddenListSpec
 	ForbiddenUserNodeAnnotations() *capsulev1beta1.ForbiddenListSpec