diff --git a/controllers/soot/manager.go b/controllers/soot/manager.go index 11b9d55..fc2cfcb 100644 --- a/controllers/soot/manager.go +++ b/controllers/soot/manager.go @@ -5,6 +5,7 @@ package soot import ( "context" + "errors" "fmt" "time" @@ -29,16 +30,17 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/finalizers" "github.com/clastix/kamaji/controllers/soot/controllers" - "github.com/clastix/kamaji/controllers/soot/controllers/errors" + kamajierrors "github.com/clastix/kamaji/controllers/soot/controllers/errors" "github.com/clastix/kamaji/controllers/utils" "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) type sootItem struct { - triggers []chan event.GenericEvent - cancelFn context.CancelFunc - completedCh chan struct{} + certificateSha string + triggers []chan event.GenericEvent + cancelFn context.CancelFunc + completedCh chan struct{} } type sootMap map[string]sootItem @@ -71,7 +73,7 @@ func (m *Manager) retrieveTenantControlPlane(ctx context.Context, request reconc } if utils.IsPaused(tcp) { - return nil, errors.ErrPausedReconciliation + return nil, kamajierrors.ErrPausedReconciliation } return tcp, nil @@ -147,8 +149,9 @@ func (m *Manager) retryTenantControlPlaneAnnotations(ctx context.Context, reques }) } -//nolint:maintidx +//nolint:maintidx,gocyclo func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, err error) { + logger := log.FromContext(ctx) // Retrieving the TenantControlPlane: // in case of deletion, we must be sure to properly remove from the memory the soot manager. tcp := &kamajiv1alpha1.TenantControlPlane{} @@ -189,6 +192,10 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res // we don't want to pollute with messages due to broken connection. // Once the TCP will be ready again, the event will be intercepted and the manager started back. return reconcile.Result{}, m.cleanup(ctx, request, tcp) + case tcp.Status.KubeConfig.Admin.Checksum != v.certificateSha: + // The stored kubeconfig to access the Tenant Control Plane has changed: + // we need to clean-up and requeue to fetch the updated value. + return reconcile.Result{RequeueAfter: time.Second}, m.cleanup(ctx, request, tcp) default: for _, trigger := range v.triggers { var shrunkTCP kamajiv1alpha1.TenantControlPlane @@ -205,10 +212,22 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res // No need to start a soot manager if the TenantControlPlane is not ready: // enqueuing back is not required since we're going to get that event once ready. if tcpStatus == kamajiv1alpha1.VersionNotReady || tcpStatus == kamajiv1alpha1.VersionCARotating || tcpStatus == kamajiv1alpha1.VersionSleeping { - log.FromContext(ctx).Info("skipping start of the soot manager for a not ready instance") + logger.Info("skipping start of the soot manager for a not ready instance") return reconcile.Result{}, nil } + // Generating the manager and starting it: + // in case of any error, reconciling the request to start it back from the beginning. + tcpRest, err := utilities.GetRESTClientConfig(ctx, m.AdminClient, tcp) + if err != nil { + if errors.Is(err, utilities.ErrMissingKubeconfigKey) { + logger.Info("soot manager waiting for kubeconfig, enqueuing back") + + return reconcile.Result{RequeueAfter: time.Second}, nil + } + + return reconcile.Result{}, err + } // Setting the finalizer for the soot manager: // upon deletion the soot manager will be shut down prior the Deployment, avoiding logs pollution. if !controllerutil.ContainsFinalizer(tcp, finalizers.SootFinalizer) { @@ -220,12 +239,6 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res return reconcile.Result{RequeueAfter: time.Second}, finalizerErr } - // Generating the manager and starting it: - // in case of any error, reconciling the request to start it back from the beginning. - tcpRest, err := utilities.GetRESTClientConfig(ctx, m.AdminClient, tcp) - if err != nil { - return reconcile.Result{}, err - } tcpCtx, tcpCancelFn := context.WithCancel(ctx) defer func() { @@ -371,14 +384,14 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res // Starting the manager go func() { if err = mgr.Start(tcpCtx); err != nil { - log.FromContext(ctx).Error(err, "unable to start soot manager") + logger.Error(err, "unable to start soot manager") // The sootManagerAnnotation is used to propagate the error between reconciliations with its state: // this is required to avoid mutex and prevent concurrent read/write on the soot map annotationErr := m.retryTenantControlPlaneAnnotations(ctx, request, func(annotations map[string]string) { annotations[sootManagerAnnotation] = sootManagerFailedAnnotation }) if annotationErr != nil { - log.FromContext(ctx).Error(err, "unable to update TenantControlPlane for soot failed annotation") + logger.Error(err, "unable to update TenantControlPlane for soot failed annotation") } // When the manager cannot start we're enqueuing back the request to take advantage of the backoff factor // of the queue: this is a goroutine and cannot return an error since the manager is running on its own, @@ -394,6 +407,7 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res }() m.sootMap[request.NamespacedName.String()] = sootItem{ + certificateSha: tcp.Status.KubeConfig.Admin.Checksum, triggers: []chan event.GenericEvent{ writePermissions.TriggerChannel, migrate.TriggerChannel, diff --git a/internal/resources/kubeconfig.go b/internal/resources/kubeconfig.go index 0a670c7..c0db65f 100644 --- a/internal/resources/kubeconfig.go +++ b/internal/resources/kubeconfig.go @@ -128,11 +128,12 @@ func (r *KubeconfigResource) CreateOrUpdate(ctx context.Context, tenantControlPl return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) } -func (r *KubeconfigResource) checksum(caCertificatesSecret *corev1.Secret, kubeadmChecksum string) string { +func (r *KubeconfigResource) checksum(caCertificatesSecret *corev1.Secret, kubeadmChecksum string, checksum string) string { return utilities.CalculateMapChecksum(map[string][]byte{ "ca-cert-checksum": caCertificatesSecret.Data[kubeadmconstants.CACertName], "ca-key-checksum": caCertificatesSecret.Data[kubeadmconstants.CAKeyName], "kubeadmconfig": []byte(kubeadmChecksum), + "kubeconfig": []byte(checksum), }) } @@ -162,7 +163,7 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam return err } - checksum := r.checksum(caCertificatesSecret, config.Checksum()) + checksum := r.checksum(caCertificatesSecret, config.Checksum(), utilities.CalculateMapChecksum(r.resource.Data)) status, err := r.getKubeconfigStatus(tenantControlPlane) if err != nil { @@ -241,6 +242,9 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam r.resource.Data[key] = kubeconfig } + + checksum = r.checksum(caCertificatesSecret, config.Checksum(), utilities.CalculateMapChecksum(r.resource.Data)) + r.resource.SetAnnotations(utilities.MergeMaps(r.resource.GetAnnotations(), map[string]string{constants.Checksum: checksum})) } return nil diff --git a/internal/utilities/kubeconfig.go b/internal/utilities/kubeconfig.go index 422a082..b66ee34 100644 --- a/internal/utilities/kubeconfig.go +++ b/internal/utilities/kubeconfig.go @@ -4,16 +4,19 @@ package utilities import ( + "errors" "fmt" corev1 "k8s.io/api/core/v1" clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" ) +var ErrMissingKubeconfigKey = errors.New("the given key is not available in the kubeconfig Secret") + func DecodeKubeconfig(secret corev1.Secret, key string) (*clientcmdapiv1.Config, error) { bytes, ok := secret.Data[key] if !ok { - return nil, fmt.Errorf("%s is not into kubeconfig secret", key) + return nil, fmt.Errorf("%w: %s", ErrMissingKubeconfigKey, key) } return DecodeKubeconfigYAML(bytes)