From 82b58d7d53e36c8776642e30bbbe5b42927f627a Mon Sep 17 00:00:00 2001 From: Maksim Fedotov Date: Thu, 2 Jun 2022 14:57:35 +0300 Subject: [PATCH] feat: refactor capsule TLS certificates management --- .../capsuleconfiguration_annotations.go | 1 + charts/capsule/templates/ca.yaml | 12 - controllers/secret/ca.go | 258 -------------- controllers/secret/errors.go | 10 - controllers/secret/reconciler.go | 34 -- controllers/secret/tls.go | 172 --------- controllers/tls/manager.go | 336 ++++++++++++++++++ main.go | 40 +-- pkg/cert/ca.go | 91 ++--- pkg/cert/ca_test.go | 37 -- pkg/configuration/client.go | 35 +- pkg/configuration/configuration.go | 5 +- 12 files changed, 417 insertions(+), 614 deletions(-) delete mode 100644 charts/capsule/templates/ca.yaml delete mode 100644 controllers/secret/ca.go delete mode 100644 controllers/secret/errors.go delete mode 100644 controllers/secret/reconciler.go delete mode 100644 controllers/secret/tls.go create mode 100644 controllers/tls/manager.go diff --git a/api/v1alpha1/capsuleconfiguration_annotations.go b/api/v1alpha1/capsuleconfiguration_annotations.go index 67902f68..f647cdd4 100644 --- a/api/v1alpha1/capsuleconfiguration_annotations.go +++ b/api/v1alpha1/capsuleconfiguration_annotations.go @@ -9,4 +9,5 @@ const ( TLSSecretNameAnnotation = "capsule.clastix.io/tls-secret-name" MutatingWebhookConfigurationName = "capsule.clastix.io/mutating-webhook-configuration-name" ValidatingWebhookConfigurationName = "capsule.clastix.io/validating-webhook-configuration-name" + GenerateCertificatesAnnotationName = "capsule.clastix.io/generate-certificates" ) diff --git a/charts/capsule/templates/ca.yaml b/charts/capsule/templates/ca.yaml deleted file mode 100644 index 49a495ef..00000000 --- a/charts/capsule/templates/ca.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if not .Values.certManager.generateCertificates }} -apiVersion: v1 -kind: Secret -metadata: - labels: - {{- include "capsule.labels" . | nindent 4 }} - {{- with .Values.customAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "capsule.secretCaName" . }} -{{- end }} diff --git a/controllers/secret/ca.go b/controllers/secret/ca.go deleted file mode 100644 index a07a760d..00000000 --- a/controllers/secret/ca.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2020-2021 Clastix Labs -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "bytes" - "context" - "errors" - "time" - - "github.com/go-logr/logr" - "golang.org/x/sync/errgroup" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" - "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/clastix/capsule/pkg/cert" - "github.com/clastix/capsule/pkg/configuration" -) - -type CAReconciler struct { - client.Client - Log logr.Logger - Scheme *runtime.Scheme - Namespace string - Configuration configuration.Configuration -} - -func (r *CAReconciler) SetupWithManager(mgr ctrl.Manager) error { - enqueueFn := handler.EnqueueRequestsFromMapFunc(func(client.Object) []reconcile.Request { - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: r.Namespace, - Name: r.Configuration.CASecretName(), - }, - }, - } - }) - - return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Secret{}). - Watches(source.NewKindWithCache(&admissionregistrationv1.ValidatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetName() == r.Configuration.ValidatingWebhookConfigurationName() - }))). - Watches(source.NewKindWithCache(&admissionregistrationv1.MutatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetName() == r.Configuration.MutatingWebhookConfigurationName() - }))). - Complete(r) -} - -// By default helm doesn't allow to use templates in CRD (https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you). -// In order to overcome this, we are setting conversion strategy in helm chart to None, and then update it with CA and namespace information. -func (r *CAReconciler) UpdateCustomResourceDefinition(ctx context.Context, caBundle []byte) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { - crd := &apiextensionsv1.CustomResourceDefinition{} - err = r.Get(ctx, types.NamespacedName{Name: "tenants.capsule.clastix.io"}, crd) - if err != nil { - r.Log.Error(err, "cannot retrieve CustomResourceDefinition") - - return err - } - - _, err = controllerutil.CreateOrUpdate(ctx, r.Client, crd, func() error { - crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ - Strategy: "Webhook", - Webhook: &apiextensionsv1.WebhookConversion{ - ClientConfig: &apiextensionsv1.WebhookClientConfig{ - Service: &apiextensionsv1.ServiceReference{ - Namespace: r.Namespace, - Name: "capsule-webhook-service", - Path: pointer.StringPtr("/convert"), - Port: pointer.Int32Ptr(443), - }, - CABundle: caBundle, - }, - ConversionReviewVersions: []string{"v1alpha1", "v1beta1"}, - }, - } - - return nil - }) - - return err - }) -} - -//nolint:dupl -func (r CAReconciler) UpdateValidatingWebhookConfiguration(ctx context.Context, caBundle []byte) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { - vw := &admissionregistrationv1.ValidatingWebhookConfiguration{} - err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.ValidatingWebhookConfigurationName()}, vw) - if err != nil { - r.Log.Error(err, "cannot retrieve ValidatingWebhookConfiguration") - - return err - } - for i, w := range vw.Webhooks { - // Updating CABundle only in case of an internal service reference - if w.ClientConfig.Service != nil { - vw.Webhooks[i].ClientConfig.CABundle = caBundle - } - } - - return r.Update(ctx, vw, &client.UpdateOptions{}) - }) -} - -//nolint:dupl -func (r CAReconciler) UpdateMutatingWebhookConfiguration(ctx context.Context, caBundle []byte) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { - mw := &admissionregistrationv1.MutatingWebhookConfiguration{} - err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.MutatingWebhookConfigurationName()}, mw) - if err != nil { - r.Log.Error(err, "cannot retrieve MutatingWebhookConfiguration") - - return err - } - for i, w := range mw.Webhooks { - // Updating CABundle only in case of an internal service reference - if w.ClientConfig.Service != nil { - mw.Webhooks[i].ClientConfig.CABundle = caBundle - } - } - - return r.Update(ctx, mw, &client.UpdateOptions{}) - }) -} - -func (r CAReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - var err error - - if request.Name != r.Configuration.CASecretName() { - return ctrl.Result{}, nil - } - - r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - r.Log.Info("Reconciling CA Secret") - - // Fetch the CA instance - instance := &corev1.Secret{} - - if err = r.Client.Get(ctx, request.NamespacedName, instance); err != nil { - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - var ca cert.CA - - var rq time.Duration - - ca, err = getCertificateAuthority(ctx, r.Client, r.Namespace, r.Configuration.CASecretName()) - if err != nil && errors.Is(err, MissingCaError{}) { - ca, err = cert.GenerateCertificateAuthority() - if err != nil { - return reconcile.Result{}, err - } - } else if err != nil { - return reconcile.Result{}, err - } - - r.Log.Info("Handling CA Secret") - - rq, err = ca.ExpiresIn(time.Now()) - if err != nil { - r.Log.Info("CA is expired, cleaning to obtain a new one") - - instance.Data = map[string][]byte{} - } else { - r.Log.Info("Updating CA secret with new PEM and RSA") - - var crt *bytes.Buffer - var key *bytes.Buffer - crt, _ = ca.CACertificatePem() - key, _ = ca.CAPrivateKeyPem() - - instance.Data = map[string][]byte{ - corev1.TLSCertKey: crt.Bytes(), - corev1.TLSPrivateKeyKey: key.Bytes(), - } - - group := new(errgroup.Group) - group.Go(func() error { - return r.UpdateMutatingWebhookConfiguration(ctx, crt.Bytes()) - }) - group.Go(func() error { - return r.UpdateValidatingWebhookConfiguration(ctx, crt.Bytes()) - }) - group.Go(func() error { - return r.UpdateCustomResourceDefinition(ctx, crt.Bytes()) - }) - - if err = group.Wait(); err != nil { - return reconcile.Result{}, err - } - } - - var res controllerutil.OperationResult - - t := &corev1.Secret{ObjectMeta: instance.ObjectMeta} - - res, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error { - t.Data = instance.Data - - return nil - }) - if err != nil { - r.Log.Error(err, "cannot update Capsule TLS") - - return reconcile.Result{}, err - } - - if res == controllerutil.OperationResultUpdated { - r.Log.Info("Capsule CA has been updated, we need to trigger TLS update too") - - tls := &corev1.Secret{} - err = r.Get(ctx, types.NamespacedName{ - Namespace: r.Namespace, - Name: r.Configuration.TLSSecretName(), - }, tls) - - if err != nil { - r.Log.Error(err, "Capsule TLS Secret missing") - } - - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - _, err = controllerutil.CreateOrUpdate(ctx, r.Client, tls, func() error { - tls.Data = map[string][]byte{} - - return nil - }) - - return err - }) - if err != nil { - r.Log.Error(err, "Cannot clean Capsule TLS Secret due to CA update") - - return reconcile.Result{}, err - } - } - - r.Log.Info("Reconciliation completed, processing back in " + rq.String()) - - return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil -} diff --git a/controllers/secret/errors.go b/controllers/secret/errors.go deleted file mode 100644 index 4e8d6117..00000000 --- a/controllers/secret/errors.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2020-2021 Clastix Labs -// SPDX-License-Identifier: Apache-2.0 - -package secret - -type MissingCaError struct{} - -func (MissingCaError) Error() string { - return "CA has not been created yet, please generate a new" -} diff --git a/controllers/secret/reconciler.go b/controllers/secret/reconciler.go deleted file mode 100644 index 6d6ebbea..00000000 --- a/controllers/secret/reconciler.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020-2021 Clastix Labs -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/clastix/capsule/pkg/cert" -) - -func getCertificateAuthority(ctx context.Context, client client.Client, namespace, name string) (ca cert.CA, err error) { - instance := &corev1.Secret{} - - if err = client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, instance); err != nil { - return nil, fmt.Errorf("missing secret %s, cannot reconcile", name) - } - - if instance.Data == nil { - return nil, MissingCaError{} - } - - ca, err = cert.NewCertificateAuthorityFromBytes(instance.Data[corev1.TLSCertKey], instance.Data[corev1.TLSPrivateKeyKey]) - if err != nil { - return - } - - return -} diff --git a/controllers/secret/tls.go b/controllers/secret/tls.go deleted file mode 100644 index 0a7e01c7..00000000 --- a/controllers/secret/tls.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2020-2021 Clastix Labs -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "bytes" - "context" - "crypto/x509" - "encoding/pem" - "fmt" - "os" - "time" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/clastix/capsule/pkg/cert" - "github.com/clastix/capsule/pkg/configuration" -) - -type TLSReconciler struct { - client.Client - Log logr.Logger - Scheme *runtime.Scheme - Namespace string - Configuration configuration.Configuration -} - -func (r *TLSReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Secret{}). - Complete(r) -} - -func (r TLSReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - var err error - - if request.Name != r.Configuration.TLSSecretName() { - return ctrl.Result{}, nil - } - - r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - r.Log.Info("Reconciling TLS Secret") - - // Fetch the Secret instance - instance := &corev1.Secret{} - if err = r.Get(ctx, request.NamespacedName, instance); err != nil { - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - var ca cert.CA - - var rq time.Duration - - ca, err = getCertificateAuthority(ctx, r.Client, r.Namespace, r.Configuration.CASecretName()) - if err != nil { - return reconcile.Result{}, err - } - - var shouldCreate bool - - for _, key := range []string{corev1.TLSCertKey, corev1.TLSPrivateKeyKey} { - if _, ok := instance.Data[key]; !ok { - shouldCreate = true - - break - } - } - - if shouldCreate { - r.Log.Info("Missing Capsule TLS certificate") - - rq = 6 * 30 * 24 * time.Hour - - opts := cert.NewCertOpts(time.Now().Add(rq), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace)) - - var crt, key *bytes.Buffer - - if crt, key, err = ca.GenerateCertificate(opts); err != nil { - r.Log.Error(err, "Cannot generate new TLS certificate") - - return reconcile.Result{}, err - } - - instance.Data = map[string][]byte{ - corev1.TLSCertKey: crt.Bytes(), - corev1.TLSPrivateKeyKey: key.Bytes(), - } - } else { - var c *x509.Certificate - var b *pem.Block - b, _ = pem.Decode(instance.Data[corev1.TLSCertKey]) - c, err = x509.ParseCertificate(b.Bytes) - if err != nil { - r.Log.Error(err, "cannot parse Capsule TLS") - - return reconcile.Result{}, err - } - - rq = time.Until(c.NotAfter) - - err = ca.ValidateCert(c) - if err != nil { - r.Log.Info("Capsule TLS is expired or invalid, cleaning to obtain a new one") - instance.Data = map[string][]byte{} - } - } - - var res controllerutil.OperationResult - - t := &corev1.Secret{ObjectMeta: instance.ObjectMeta} - - res, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error { - t.Data = instance.Data - - return nil - }) - if err != nil { - r.Log.Error(err, "cannot update Capsule TLS") - - return reconcile.Result{}, err - } - // nolint:nestif - if instance.Name == r.Configuration.TLSSecretName() && res == controllerutil.OperationResultUpdated { - r.Log.Info("Capsule TLS certificates has been updated, Controller pods must be restarted to load new certificate") - - hostname, _ := os.Hostname() - - leaderPod := &corev1.Pod{} - - if err = r.Client.Get(ctx, types.NamespacedName{Namespace: os.Getenv("NAMESPACE"), Name: hostname}, leaderPod); err != nil { - r.Log.Error(err, "cannot retrieve the leader Pod, probably running in out of the cluster mode") - - return reconcile.Result{}, nil - } - - podList := &corev1.PodList{} - if err = r.Client.List(ctx, podList, client.MatchingLabels(leaderPod.ObjectMeta.Labels)); err != nil { - r.Log.Error(err, "cannot retrieve list of Capsule pods requiring restart upon TLS update") - - return reconcile.Result{}, nil - } - - for _, p := range podList.Items { - nonLeaderPod := p - // Skipping this Pod, must be deleted at the end - if nonLeaderPod.GetName() == leaderPod.GetName() { - continue - } - - if err = r.Client.Delete(ctx, &nonLeaderPod); err != nil { - r.Log.Error(err, "cannot delete the non-leader Pod due to TLS update") - } - } - - if err = r.Client.Delete(ctx, leaderPod); err != nil { - r.Log.Error(err, "cannot delete the leader Pod due to TLS update") - } - } - - r.Log.Info("Reconciliation completed, processing back in " + rq.String()) - - return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil -} diff --git a/controllers/tls/manager.go b/controllers/tls/manager.go new file mode 100644 index 00000000..7b8dfb76 --- /dev/null +++ b/controllers/tls/manager.go @@ -0,0 +1,336 @@ +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package tls + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/go-logr/logr" + "golang.org/x/sync/errgroup" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/clastix/capsule/controllers/utils" + "github.com/clastix/capsule/pkg/cert" + "github.com/clastix/capsule/pkg/configuration" +) + +const ( + certificateExpirationThreshold = 3 * 24 * time.Hour + certificateReconciliationThreshold = 4 * 24 * time.Hour + certificateValidity = 6 * 30 * 24 * time.Hour + PodUpdateAnnotationName = "capsule.clastix.io/updated" +) + +type Reconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + Namespace string + Configuration configuration.Configuration +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + enqueueFn := handler.EnqueueRequestsFromMapFunc(func(client.Object) []reconcile.Request { + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: r.Namespace, + Name: r.Configuration.TLSSecretName(), + }, + }, + } + }) + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Secret{}, utils.NamesMatchingPredicate(r.Configuration.TLSSecretName())). + Watches(source.NewKindWithCache(&admissionregistrationv1.ValidatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetName() == r.Configuration.ValidatingWebhookConfigurationName() + }))). + Watches(source.NewKindWithCache(&admissionregistrationv1.MutatingWebhookConfiguration{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetName() == r.Configuration.MutatingWebhookConfigurationName() + }))). + Watches(source.NewKindWithCache(&apiextensionsv1.CustomResourceDefinition{}, mgr.GetCache()), enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetName() == r.Configuration.TenantCRDName() + }))). + Complete(r) +} + +func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + + // Fetch the CA instance + certSecret := &corev1.Secret{} + + if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil { + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + if r.shouldUpdateCertificate(certSecret) { + r.Log.Info("Generating new TLS certificate") + + ca, err := cert.GenerateCertificateAuthority() + if err != nil { + return reconcile.Result{}, err + } + + opts := cert.NewCertOpts(time.Now().Add(certificateValidity), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace)) + + crt, key, err := ca.GenerateCertificate(opts) + if err != nil { + r.Log.Error(err, "Cannot generate new TLS certificate") + + return reconcile.Result{}, err + } + + caCrt, _ := ca.CACertificatePem() + + certSecret.Data = map[string][]byte{ + corev1.TLSCertKey: crt.Bytes(), + corev1.TLSPrivateKeyKey: key.Bytes(), + corev1.ServiceAccountRootCAKey: caCrt.Bytes(), + } + + t := &corev1.Secret{ObjectMeta: certSecret.ObjectMeta} + + _, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error { + t.Data = certSecret.Data + + return nil + }) + if err != nil { + r.Log.Error(err, "cannot update Capsule TLS") + + return reconcile.Result{}, err + } + } + + var caBundle []byte + + var ok bool + + if caBundle, ok = certSecret.Data[corev1.ServiceAccountRootCAKey]; !ok { + return reconcile.Result{}, fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName()) + } + + operatorPods, err := r.getOperatorPods(ctx) + if err != nil { + return reconcile.Result{}, err + } + + r.Log.Info("Updating caBundle in webhooks and crd") + + group := new(errgroup.Group) + group.Go(func() error { + return r.updateMutatingWebhookConfiguration(ctx, caBundle) + }) + group.Go(func() error { + return r.updateValidatingWebhookConfiguration(ctx, caBundle) + }) + group.Go(func() error { + return r.updateCustomResourceDefinition(ctx, caBundle) + }) + + r.Log.Info("Updating capsule operator pods") + + for _, pod := range operatorPods.Items { + p := pod + + group.Go(func() error { + return r.updateOperatorPod(ctx, p) + }) + } + + if err := group.Wait(); err != nil { + return reconcile.Result{}, err + } + + if r.Configuration.GenerateCertificates() { + certificate, err := cert.GetCertificateFromBytes(certSecret.Data[corev1.TLSCertKey]) + if err != nil { + return reconcile.Result{}, err + } + + now := time.Now() + + rq := (time.Duration(certificate.NotAfter.Unix()-now.Unix()) * time.Second) - certificateReconciliationThreshold + + r.Log.Info("Reconciliation completed, processing back in " + rq.String()) + + return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil + } + + return reconcile.Result{}, nil +} + +func (r Reconciler) shouldUpdateCertificate(secret *corev1.Secret) bool { + if !r.Configuration.GenerateCertificates() { + r.Log.Info("Skipping TLS certificate generation as it is disabled in CapsuleConfiguration") + + return false + } + + if _, ok := secret.Data[corev1.ServiceAccountRootCAKey]; !ok { + return true + } + + certificate, key, err := cert.GetCertificateWithPrivateKeyFromBytes(secret.Data[corev1.TLSCertKey], secret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + return true + } + + if err := cert.ValidateCertificate(certificate, key, certificateExpirationThreshold); err != nil { + r.Log.Error(err, "failed to validate certificate, generating new one") + + return true + } + + r.Log.Info("Skipping TLS certificate generation as it is still valid") + + return false +} + +// By default helm doesn't allow to use templates in CRD (https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you). +// In order to overcome this, we are setting conversion strategy in helm chart to None, and then update it with CA and namespace information. +func (r *Reconciler) updateCustomResourceDefinition(ctx context.Context, caBundle []byte) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + crd := &apiextensionsv1.CustomResourceDefinition{} + err = r.Get(ctx, types.NamespacedName{Name: "tenants.capsule.clastix.io"}, crd) + if err != nil { + r.Log.Error(err, "cannot retrieve CustomResourceDefinition") + + return err + } + + _, err = controllerutil.CreateOrUpdate(ctx, r.Client, crd, func() error { + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: "Webhook", + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + Service: &apiextensionsv1.ServiceReference{ + Namespace: r.Namespace, + Name: "capsule-webhook-service", + Path: pointer.StringPtr("/convert"), + Port: pointer.Int32Ptr(443), + }, + CABundle: caBundle, + }, + ConversionReviewVersions: []string{"v1alpha1", "v1beta1"}, + }, + } + + return nil + }) + + return err + }) +} + +//nolint:dupl +func (r Reconciler) updateValidatingWebhookConfiguration(ctx context.Context, caBundle []byte) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + vw := &admissionregistrationv1.ValidatingWebhookConfiguration{} + err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.ValidatingWebhookConfigurationName()}, vw) + if err != nil { + r.Log.Error(err, "cannot retrieve ValidatingWebhookConfiguration") + + return err + } + for i, w := range vw.Webhooks { + // Updating CABundle only in case of an internal service reference + if w.ClientConfig.Service != nil { + vw.Webhooks[i].ClientConfig.CABundle = caBundle + } + } + + return r.Update(ctx, vw, &client.UpdateOptions{}) + }) +} + +//nolint:dupl +func (r Reconciler) updateMutatingWebhookConfiguration(ctx context.Context, caBundle []byte) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + mw := &admissionregistrationv1.MutatingWebhookConfiguration{} + err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.MutatingWebhookConfigurationName()}, mw) + if err != nil { + r.Log.Error(err, "cannot retrieve MutatingWebhookConfiguration") + + return err + } + for i, w := range mw.Webhooks { + // Updating CABundle only in case of an internal service reference + if w.ClientConfig.Service != nil { + mw.Webhooks[i].ClientConfig.CABundle = caBundle + } + } + + return r.Update(ctx, mw, &client.UpdateOptions{}) + }) +} + +func (r Reconciler) updateOperatorPod(ctx context.Context, pod corev1.Pod) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + // Need to get latest version of pod + p := &corev1.Pod{} + + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, p); err != nil && !apierrors.IsNotFound(err) { + r.Log.Error(err, "cannot get pod", "name", pod.Name, "namespace", pod.Namespace) + + return err + } + + if p.Annotations == nil { + p.Annotations = map[string]string{} + } + + p.Annotations[PodUpdateAnnotationName] = time.Now().Format(time.RFC3339Nano) + + if err := r.Client.Update(ctx, p, &client.UpdateOptions{}); err != nil { + r.Log.Error(err, "cannot update pod", "name", pod.Name, "namespace", pod.Namespace) + + return err + } + + return nil + }) +} + +func (r Reconciler) getOperatorPods(ctx context.Context) (*corev1.PodList, error) { + hostname, _ := os.Hostname() + + leaderPod := &corev1.Pod{} + + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: os.Getenv("NAMESPACE"), Name: hostname}, leaderPod); err != nil { + r.Log.Error(err, "cannot retrieve the leader Pod, probably running in out of the cluster mode") + + return nil, err + } + + podList := &corev1.PodList{} + if err := r.Client.List(ctx, podList, client.MatchingLabels(leaderPod.ObjectMeta.Labels)); err != nil { + r.Log.Error(err, "cannot retrieve list of Capsule pods") + + return nil, err + } + + return podList, nil +} diff --git a/main.go b/main.go index ee9535fd..328fb6d2 100644 --- a/main.go +++ b/main.go @@ -28,9 +28,9 @@ import ( capsulev1beta1 "github.com/clastix/capsule/api/v1beta1" configcontroller "github.com/clastix/capsule/controllers/config" rbaccontroller "github.com/clastix/capsule/controllers/rbac" - secretcontroller "github.com/clastix/capsule/controllers/secret" servicelabelscontroller "github.com/clastix/capsule/controllers/servicelabels" tenantcontroller "github.com/clastix/capsule/controllers/tenant" + tlscontroller "github.com/clastix/capsule/controllers/tls" "github.com/clastix/capsule/pkg/configuration" "github.com/clastix/capsule/pkg/indexer" "github.com/clastix/capsule/pkg/webhook" @@ -70,15 +70,13 @@ func printVersion() { // nolint:maintidx func main() { - var enableLeaderElection, enableSecretController, version bool + var enableLeaderElection, version bool var metricsAddr, namespace, configurationName string var goFlagSet goflag.FlagSet flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") - flag.BoolVar(&enableSecretController, "enable-secret-controller", true, - "Enable secret controller which reconciles TLS and CA secrets for capsule webhooks.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -133,28 +131,6 @@ func main() { cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), configurationName) - if enableSecretController { - if err = (&secretcontroller.CAReconciler{ - Client: manager.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("CA"), - Namespace: namespace, - Configuration: cfg, - }).SetupWithManager(manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Namespace") - os.Exit(1) - } - - if err = (&secretcontroller.TLSReconciler{ - Client: manager.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Tls"), - Namespace: namespace, - Configuration: cfg, - }).SetupWithManager(manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Namespace") - os.Exit(1) - } - } - clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) if err != nil { setupLog.Error(err, "unable to create kubernetes clientset") @@ -172,9 +148,13 @@ func main() { directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, configurationName) - ca, err := clientset.CoreV1().Secrets(namespace).Get(ctx, directCfg.CASecretName(), metav1.GetOptions{}) - if err != nil { - setupLog.Error(err, "unable to get Capsule CA secret") + if err = (&tlscontroller.Reconciler{ + Client: manager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TLS"), + Namespace: namespace, + Configuration: directCfg, + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Namespace") os.Exit(1) } @@ -184,7 +164,7 @@ func main() { os.Exit(1) } // nolint:nestif - if len(ca.Data) > 0 && len(tls.Data) > 0 { + if len(tls.Data) > 0 { if err = (&tenantcontroller.Manager{ RESTConfig: manager.GetConfig(), Client: manager.GetClient(), diff --git a/pkg/cert/ca.go b/pkg/cert/ca.go index fdd67a36..cd676e83 100644 --- a/pkg/cert/ca.go +++ b/pkg/cert/ca.go @@ -12,6 +12,8 @@ import ( "encoding/pem" "math/big" "time" + + "github.com/pkg/errors" ) type CA interface { @@ -27,38 +29,6 @@ type CapsuleCA struct { key *rsa.PrivateKey } -func (c CapsuleCA) ValidateCert(certificate *x509.Certificate) (err error) { - pool := x509.NewCertPool() - pool.AddCert(c.certificate) - - _, err = certificate.Verify(x509.VerifyOptions{ - Roots: pool, - CurrentTime: time.Time{}, - }) - - return -} - -func (c CapsuleCA) isAlreadyValid(now time.Time) bool { - return now.After(c.certificate.NotBefore) -} - -func (c CapsuleCA) isExpired(now time.Time) bool { - return now.Before(c.certificate.NotAfter) -} - -func (c CapsuleCA) ExpiresIn(now time.Time) (time.Duration, error) { - if !c.isExpired(now) { - return time.Nanosecond, CaExpiredError{} - } - - if !c.isAlreadyValid(now) { - return time.Nanosecond, CaNotYetValidError{} - } - - return time.Duration(c.certificate.NotAfter.Unix()-now.Unix()) * time.Second, nil -} - func (c CapsuleCA) CACertificatePem() (b *bytes.Buffer, err error) { var crtBytes []byte crtBytes, err = x509.CreateCertificate(rand.Reader, c.certificate, c.certificate, &c.key.PublicKey, c.key) @@ -85,6 +55,24 @@ func (c CapsuleCA) CAPrivateKeyPem() (b *bytes.Buffer, err error) { }) } +func ValidateCertificate(cert *x509.Certificate, key *rsa.PrivateKey, expirationThreshold time.Duration) error { + if !key.PublicKey.Equal(cert.PublicKey) { + return errors.New("certificate signed by wrong public key") + } + + now := time.Now() + + if now.Before(cert.NotBefore) { + return errors.New("certificate is not valid yet") + } + + if now.After(cert.NotAfter.Add(-expirationThreshold)) { + return errors.New("certificate expired or going to expire soon") + } + + return nil +} + func GenerateCertificateAuthority() (s *CapsuleCA, err error) { s = &CapsuleCA{ certificate: &x509.Certificate{ @@ -114,31 +102,46 @@ func GenerateCertificateAuthority() (s *CapsuleCA, err error) { return } -func NewCertificateAuthorityFromBytes(certBytes, keyBytes []byte) (s *CapsuleCA, err error) { +func GetCertificateFromBytes(certBytes []byte) (*x509.Certificate, error) { var b *pem.Block b, _ = pem.Decode(certBytes) - var cert *x509.Certificate + return x509.ParseCertificate(b.Bytes) +} - if cert, err = x509.ParseCertificate(b.Bytes); err != nil { - return - } +func GetPrivateKeyFromBytes(keyBytes []byte) (*rsa.PrivateKey, error) { + var b *pem.Block b, _ = pem.Decode(keyBytes) - var key *rsa.PrivateKey + return x509.ParsePKCS1PrivateKey(b.Bytes) +} - if key, err = x509.ParsePKCS1PrivateKey(b.Bytes); err != nil { - return +func GetCertificateWithPrivateKeyFromBytes(certBytes, keyBytes []byte) (*x509.Certificate, *rsa.PrivateKey, error) { + cert, err := GetCertificateFromBytes(certBytes) + if err != nil { + return nil, nil, err } - s = &CapsuleCA{ + key, err := GetPrivateKeyFromBytes(keyBytes) + if err != nil { + return nil, nil, err + } + + return cert, key, nil +} + +func NewCertificateAuthorityFromBytes(certBytes, keyBytes []byte) (*CapsuleCA, error) { + cert, key, err := GetCertificateWithPrivateKeyFromBytes(certBytes, keyBytes) + if err != nil { + return nil, err + } + + return &CapsuleCA{ certificate: cert, key: key, - } - - return + }, nil } // nolint:nakedret diff --git a/pkg/cert/ca_test.go b/pkg/cert/ca_test.go index 0c3b9989..7f38e10d 100644 --- a/pkg/cert/ca_test.go +++ b/pkg/cert/ca_test.go @@ -74,40 +74,3 @@ func TestCapsuleCa_GenerateCertificate(t *testing.T) { }) } } - -func TestCapsuleCa_IsValid(t *testing.T) { - type testCase struct { - notBefore time.Time - notAfter time.Time - returnError bool - } - - tc := map[string]testCase{ - "ok": {time.Now().AddDate(0, 0, -1), time.Now().AddDate(0, 0, 1), false}, - "expired": {time.Now().AddDate(1, 0, 0), time.Now(), true}, - "notValid": {time.Now().AddDate(0, 0, 1), time.Now().AddDate(0, 0, 2), true}, - } - - for name, c := range tc { - t.Run(name, func(t *testing.T) { - var ca *CapsuleCA - var err error - - ca, err = GenerateCertificateAuthority() - assert.Nil(t, err) - - ca.certificate.NotAfter = c.notAfter - ca.certificate.NotBefore = c.notBefore - - var w time.Duration - w, err = ca.ExpiresIn(time.Now()) - if c.returnError { - assert.Error(t, err) - - return - } - assert.Nil(t, err) - assert.WithinDuration(t, ca.certificate.NotAfter, time.Now().Add(w), time.Minute) - }) - } -} diff --git a/pkg/configuration/client.go b/pkg/configuration/client.go index 65a05e22..57fe7281 100644 --- a/pkg/configuration/client.go +++ b/pkg/configuration/client.go @@ -6,6 +6,7 @@ package configuration import ( "context" "regexp" + "strconv" "strings" "github.com/pkg/errors" @@ -62,21 +63,6 @@ func (c capsuleConfiguration) ForceTenantPrefix() bool { return c.retrievalFn().Spec.ForceTenantPrefix } -func (c capsuleConfiguration) CASecretName() (name string) { - name = CASecretName - - if c.retrievalFn().Annotations == nil { - return - } - - v, ok := c.retrievalFn().Annotations[capsulev1alpha1.CASecretNameAnnotation] - if ok { - return v - } - - return -} - func (c capsuleConfiguration) TLSSecretName() (name string) { name = TLSSecretName @@ -92,6 +78,21 @@ func (c capsuleConfiguration) TLSSecretName() (name string) { return } +func (c capsuleConfiguration) GenerateCertificates() bool { + annotationValue, ok := c.retrievalFn().Annotations[capsulev1alpha1.GenerateCertificatesAnnotationName] + + if ok { + value, err := strconv.ParseBool(annotationValue) + if err != nil { + return false + } + + return value + } + + return false +} + func (c capsuleConfiguration) MutatingWebhookConfigurationName() (name string) { name = MutatingWebhookConfigurationName @@ -107,6 +108,10 @@ func (c capsuleConfiguration) MutatingWebhookConfigurationName() (name string) { return } +func (c capsuleConfiguration) TenantCRDName() string { + return TenantCRDName +} + func (c capsuleConfiguration) ValidatingWebhookConfigurationName() (name string) { name = ValidatingWebhookConfigurationName diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go index c45459c2..35f80c5f 100644 --- a/pkg/configuration/configuration.go +++ b/pkg/configuration/configuration.go @@ -10,19 +10,20 @@ import ( ) const ( - CASecretName = "capsule-ca" TLSSecretName = "capsule-tls" MutatingWebhookConfigurationName = "capsule-mutating-webhook-configuration" ValidatingWebhookConfigurationName = "capsule-validating-webhook-configuration" + TenantCRDName = "tenants.capsule.clastix.io" ) type Configuration interface { ProtectedNamespaceRegexp() (*regexp.Regexp, error) ForceTenantPrefix() bool - CASecretName() string + GenerateCertificates() bool TLSSecretName() string MutatingWebhookConfigurationName() string ValidatingWebhookConfigurationName() string + TenantCRDName() string UserGroups() []string ForbiddenUserNodeLabels() *capsulev1beta1.ForbiddenListSpec ForbiddenUserNodeAnnotations() *capsulev1beta1.ForbiddenListSpec