mirror of
https://github.com/projectcapsule/capsule.git
synced 2026-08-25 16:07:24 +00:00
feat: generate TLS certificates before starting controllers
This commit is contained in:
Maksim Fedotov
committed by
Dario Tranchitella
Dario Tranchitella
parent
37381184d2
commit
f1dc028649
+28
-22
@@ -34,10 +34,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
certificateExpirationThreshold = 3 * 24 * time.Hour
|
||||
certificateReconciliationThreshold = 4 * 24 * time.Hour
|
||||
certificateValidity = 6 * 30 * 24 * time.Hour
|
||||
PodUpdateAnnotationName = "capsule.clastix.io/updated"
|
||||
certificateExpirationThreshold = 3 * 24 * time.Hour
|
||||
certificateValidity = 6 * 30 * 24 * time.Hour
|
||||
PodUpdateAnnotationName = "capsule.clastix.io/updated"
|
||||
)
|
||||
|
||||
type Reconciler struct {
|
||||
@@ -74,23 +73,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
|
||||
r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
|
||||
|
||||
// Fetch the CA instance
|
||||
certSecret := &corev1.Secret{}
|
||||
|
||||
if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil {
|
||||
// Error reading the object - requeue the request.
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
func (r Reconciler) ReconcileCertificates(ctx context.Context, certSecret *corev1.Secret) error {
|
||||
if r.shouldUpdateCertificate(certSecret) {
|
||||
r.Log.Info("Generating new TLS certificate")
|
||||
|
||||
ca, err := cert.GenerateCertificateAuthority()
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
return err
|
||||
}
|
||||
|
||||
opts := cert.NewCertOpts(time.Now().Add(certificateValidity), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace))
|
||||
@@ -99,7 +88,7 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
|
||||
if err != nil {
|
||||
r.Log.Error(err, "Cannot generate new TLS certificate")
|
||||
|
||||
return reconcile.Result{}, err
|
||||
return err
|
||||
}
|
||||
|
||||
caCrt, _ := ca.CACertificatePem()
|
||||
@@ -120,7 +109,7 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
|
||||
if err != nil {
|
||||
r.Log.Error(err, "cannot update Capsule TLS")
|
||||
|
||||
return reconcile.Result{}, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,12 +118,12 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
|
||||
var ok bool
|
||||
|
||||
if caBundle, ok = certSecret.Data[corev1.ServiceAccountRootCAKey]; !ok {
|
||||
return reconcile.Result{}, fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
|
||||
return fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
|
||||
}
|
||||
|
||||
operatorPods, err := r.getOperatorPods(ctx)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
return err
|
||||
}
|
||||
|
||||
r.Log.Info("Updating caBundle in webhooks and crd")
|
||||
@@ -161,6 +150,23 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
|
||||
}
|
||||
|
||||
if err := group.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
|
||||
r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
|
||||
|
||||
certSecret := &corev1.Secret{}
|
||||
|
||||
if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil {
|
||||
// Error reading the object - requeue the request.
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
if err := r.ReconcileCertificates(ctx, certSecret); err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
@@ -171,8 +177,8 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
rq := (time.Duration(certificate.NotAfter.Unix()-now.Unix()) * time.Second) - certificateReconciliationThreshold
|
||||
requeueTime := certificate.NotAfter.Add(-(certificateExpirationThreshold - 1*time.Second))
|
||||
rq := requeueTime.Sub(now)
|
||||
|
||||
r.Log.Info("Reconciliation completed, processing back in " + rq.String())
|
||||
|
||||
|
||||
@@ -11,12 +11,12 @@ import (
|
||||
|
||||
flag "github.com/spf13/pflag"
|
||||
"go.uber.org/zap/zapcore"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
utilVersion "k8s.io/apimachinery/pkg/util/version"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
@@ -131,12 +131,6 @@ func main() {
|
||||
|
||||
cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), configurationName)
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie())
|
||||
if err != nil {
|
||||
setupLog.Error(err, "unable to create kubernetes clientset")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
directClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{
|
||||
Scheme: manager.GetScheme(),
|
||||
Mapper: manager.GetRESTMapper(),
|
||||
@@ -148,120 +142,125 @@ func main() {
|
||||
|
||||
directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, configurationName)
|
||||
|
||||
if err = (&tlscontroller.Reconciler{
|
||||
Client: manager.GetClient(),
|
||||
tlsReconciler := &tlscontroller.Reconciler{
|
||||
Client: directClient,
|
||||
Log: ctrl.Log.WithName("controllers").WithName("TLS"),
|
||||
Namespace: namespace,
|
||||
Configuration: directCfg,
|
||||
}).SetupWithManager(manager); err != nil {
|
||||
}
|
||||
|
||||
if err = tlsReconciler.SetupWithManager(manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Namespace")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
tls, err := clientset.CoreV1().Secrets(namespace).Get(ctx, directCfg.TLSSecretName(), metav1.GetOptions{})
|
||||
if err != nil {
|
||||
tlsCert := &corev1.Secret{}
|
||||
|
||||
if err = directClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: directCfg.TLSSecretName()}, tlsCert); err != nil {
|
||||
setupLog.Error(err, "unable to get Capsule TLS secret")
|
||||
os.Exit(1)
|
||||
}
|
||||
// nolint:nestif
|
||||
if len(tls.Data) > 0 {
|
||||
if err = (&tenantcontroller.Manager{
|
||||
RESTConfig: manager.GetConfig(),
|
||||
Client: manager.GetClient(),
|
||||
Log: ctrl.Log.WithName("controllers").WithName("Tenant"),
|
||||
Recorder: manager.GetEventRecorderFor("tenant-controller"),
|
||||
}).SetupWithManager(manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Tenant")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil {
|
||||
setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant")
|
||||
os.Exit(1)
|
||||
}
|
||||
// Reconcile TLS certificates before starting controllers and webhooks
|
||||
if err = tlsReconciler.ReconcileCertificates(ctx, tlsCert); err != nil {
|
||||
setupLog.Error(err, "unable to reconcile Capsule TLS secret")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = indexer.AddToManager(ctx, setupLog, manager); err != nil {
|
||||
setupLog.Error(err, "unable to setup indexers")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err = (&tenantcontroller.Manager{
|
||||
RESTConfig: manager.GetConfig(),
|
||||
Client: manager.GetClient(),
|
||||
Log: ctrl.Log.WithName("controllers").WithName("Tenant"),
|
||||
Recorder: manager.GetEventRecorderFor("tenant-controller"),
|
||||
}).SetupWithManager(manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Tenant")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var kubeVersion *utilVersion.Version
|
||||
if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil {
|
||||
setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if kubeVersion, err = utils.GetK8sVersion(); err != nil {
|
||||
setupLog.Error(err, "unable to get kubernetes version")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err = indexer.AddToManager(ctx, setupLog, manager); err != nil {
|
||||
setupLog.Error(err, "unable to setup indexers")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// webhooks: the order matters, don't change it and just append
|
||||
webhooksList := append(
|
||||
make([]webhook.Webhook, 0),
|
||||
route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()),
|
||||
route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())),
|
||||
route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
|
||||
route.PVC(pvc.Handler()),
|
||||
route.Service(service.Handler()),
|
||||
route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())),
|
||||
route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()),
|
||||
route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))),
|
||||
route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()),
|
||||
route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))),
|
||||
)
|
||||
var kubeVersion *utilVersion.Version
|
||||
|
||||
nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion)
|
||||
if !nodeWebhookSupported {
|
||||
setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735")
|
||||
}
|
||||
if kubeVersion, err = utils.GetK8sVersion(); err != nil {
|
||||
setupLog.Error(err, "unable to get kubernetes version")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = webhook.Register(manager, webhooksList...); err != nil {
|
||||
setupLog.Error(err, "unable to setup webhooks")
|
||||
os.Exit(1)
|
||||
}
|
||||
// webhooks: the order matters, don't change it and just append
|
||||
webhooksList := append(
|
||||
make([]webhook.Webhook, 0),
|
||||
route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()),
|
||||
route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())),
|
||||
route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
|
||||
route.PVC(pvc.Handler()),
|
||||
route.Service(service.Handler()),
|
||||
route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())),
|
||||
route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()),
|
||||
route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))),
|
||||
route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()),
|
||||
route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))),
|
||||
)
|
||||
|
||||
rbacManager := &rbaccontroller.Manager{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("Rbac"),
|
||||
Configuration: cfg,
|
||||
}
|
||||
nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion)
|
||||
if !nodeWebhookSupported {
|
||||
setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735")
|
||||
}
|
||||
|
||||
if err = manager.Add(rbacManager); err != nil {
|
||||
setupLog.Error(err, "unable to create cluster roles")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err = webhook.Register(manager, webhooksList...); err != nil {
|
||||
setupLog.Error(err, "unable to setup webhooks")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = rbacManager.SetupWithManager(ctx, manager, configurationName); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Rbac")
|
||||
os.Exit(1)
|
||||
}
|
||||
rbacManager := &rbaccontroller.Manager{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("Rbac"),
|
||||
Configuration: cfg,
|
||||
}
|
||||
|
||||
if err = (&servicelabelscontroller.ServicesLabelsReconciler{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"),
|
||||
}).SetupWithManager(ctx, manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err = manager.Add(rbacManager); err != nil {
|
||||
setupLog.Error(err, "unable to create cluster roles")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&servicelabelscontroller.EndpointsLabelsReconciler{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"),
|
||||
}).SetupWithManager(ctx, manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err = rbacManager.SetupWithManager(ctx, manager, configurationName); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Rbac")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"),
|
||||
VersionMinor: kubeVersion.Minor(),
|
||||
VersionMajor: kubeVersion.Major(),
|
||||
}).SetupWithManager(ctx, manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels")
|
||||
}
|
||||
if err = (&servicelabelscontroller.ServicesLabelsReconciler{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"),
|
||||
}).SetupWithManager(ctx, manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&configcontroller.Manager{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"),
|
||||
}).SetupWithManager(manager, configurationName); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration")
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
setupLog.Info("skip registering a tenant controller, missing CA secret")
|
||||
if err = (&servicelabelscontroller.EndpointsLabelsReconciler{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"),
|
||||
}).SetupWithManager(ctx, manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"),
|
||||
VersionMinor: kubeVersion.Minor(),
|
||||
VersionMajor: kubeVersion.Major(),
|
||||
}).SetupWithManager(ctx, manager); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels")
|
||||
}
|
||||
|
||||
if err = (&configcontroller.Manager{
|
||||
Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"),
|
||||
}).SetupWithManager(manager, configurationName); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
setupLog.Info("starting manager")
|
||||
|
||||
@@ -5,7 +5,6 @@ package webhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
@@ -16,12 +15,6 @@ import (
|
||||
)
|
||||
|
||||
func Register(manager controllerruntime.Manager, webhookList ...Webhook) error {
|
||||
// skipping webhook setup if certificate is missing
|
||||
certData, _ := ioutil.ReadFile("/tmp/k8s-webhook-server/serving-certs/tls.crt")
|
||||
if len(certData) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
recorder := manager.GetEventRecorderFor("tenant-webhook")
|
||||
|
||||
server := manager.GetWebhookServer()
|
||||
|
||||
Reference in New Issue
Block a user