diff --git a/controllers/resources.go b/controllers/resources.go index 1dafb4d..0e08c66 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -182,6 +182,12 @@ func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPl KubeConfigFileName: resources.AdminKubeConfigFileName, TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), }, + &resources.KubeconfigResource{ + Name: "admin-kubeconfig", + Client: c, + KubeConfigFileName: resources.SuperAdminKubeConfigFileName, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, &resources.KubeconfigResource{ Name: "controller-manager-kubeconfig", Client: c, diff --git a/controllers/soot/manager.go b/controllers/soot/manager.go index 7317896..6d2a684 100644 --- a/controllers/soot/manager.go +++ b/controllers/soot/manager.go @@ -258,6 +258,17 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res if err = bootstrapToken.SetupWithManager(mgr); err != nil { return reconcile.Result{}, err } + + kubeadmRbac := &controllers.KubeadmPhase{ + GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request), + Phase: &resources.KubeadmPhase{ + Client: m.AdminClient, + Phase: resources.PhaseClusterAdminRBAC, + }, + } + if err = kubeadmRbac.SetupWithManager(mgr); err != nil { + return reconcile.Result{}, err + } // Starting the manager go func() { if err = mgr.Start(tcpCtx); err != nil { diff --git a/internal/resources/kubeadm_phases.go b/internal/resources/kubeadm_phases.go index b705401..15ef570 100644 --- a/internal/resources/kubeadm_phases.go +++ b/internal/resources/kubeadm_phases.go @@ -6,12 +6,17 @@ package resources import ( "context" "fmt" + "os" + "strings" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" @@ -27,10 +32,11 @@ const ( PhaseUploadConfigKubeadm kubeadmPhase = iota PhaseUploadConfigKubelet PhaseBootstrapToken + PhaseClusterAdminRBAC ) func (d kubeadmPhase) String() string { - return [...]string{"PhaseUploadConfigKubeadm", "PhaseUploadConfigKubelet", "PhaseBootstrapToken"}[d] + return [...]string{"PhaseUploadConfigKubeadm", "PhaseUploadConfigKubelet", "PhaseBootstrapToken", "PhaseClusterAdminRBAC"}[d] } type KubeadmPhase struct { @@ -47,6 +53,8 @@ func (r *KubeadmPhase) GetWatchedObject() client.Object { return &corev1.ConfigMap{} case PhaseBootstrapToken: return &corev1.Secret{} + case PhaseClusterAdminRBAC: + return &rbacv1.ClusterRoleBinding{} default: panic("shouldn't happen") } @@ -56,11 +64,11 @@ func (r *KubeadmPhase) GetPredicateFunc() func(obj client.Object) bool { switch r.Phase { case PhaseUploadConfigKubeadm: return func(obj client.Object) bool { - return obj.GetName() == constants.KubeadmConfigConfigMap && obj.GetNamespace() == metav1.NamespaceSystem + return obj.GetName() == kubeadmconstants.KubeadmConfigConfigMap && obj.GetNamespace() == metav1.NamespaceSystem } case PhaseUploadConfigKubelet: return func(obj client.Object) bool { - return obj.GetName() == constants.KubeletBaseConfigurationConfigMap && obj.GetNamespace() == metav1.NamespaceSystem + return obj.GetName() == kubeadmconstants.KubeletBaseConfigurationConfigMap && obj.GetNamespace() == metav1.NamespaceSystem } case PhaseBootstrapToken: return func(obj client.Object) bool { @@ -68,6 +76,12 @@ func (r *KubeadmPhase) GetPredicateFunc() func(obj client.Object) bool { return secret.Type == "bootstrap.kubernetes.io/token" && secret.GetNamespace() == metav1.NamespaceSystem } + case PhaseClusterAdminRBAC: + return func(obj client.Object) bool { + cr := obj.(*rbacv1.ClusterRoleBinding) //nolint:forcetypeassert + + return strings.HasPrefix(cr.Name, "kubeadm:") + } default: panic("shouldn't happen") } @@ -107,7 +121,7 @@ func (r *KubeadmPhase) Define(context.Context, *kamajiv1alpha1.TenantControlPlan return nil } -func (r *KubeadmPhase) GetKubeadmFunction() (func(clientset.Interface, *kubeadm.Configuration) ([]byte, error), error) { +func (r *KubeadmPhase) GetKubeadmFunction(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (func(clientset.Interface, *kubeadm.Configuration) ([]byte, error), error) { switch r.Phase { case PhaseUploadConfigKubeadm: return kubeadm.UploadKubeadmConfig, nil @@ -119,6 +133,43 @@ func (r *KubeadmPhase) GetKubeadmFunction() (func(clientset.Interface, *kubeadm. return nil, kubeadm.BootstrapToken(client, config) }, nil + case PhaseClusterAdminRBAC: + return func(c clientset.Interface, configuration *kubeadm.Configuration) ([]byte, error) { + tmp, err := os.MkdirTemp("", string(tcp.UID)) + if err != nil { + return nil, err + } + + defer func() { _ = os.Remove(tmp) }() + + var caSecret corev1.Secret + + if err = r.Client.Get(ctx, types.NamespacedName{Name: tcp.Status.Certificates.CA.SecretName, Namespace: tcp.Namespace}, &caSecret); err != nil { + return nil, err + } + + crtKeyPair := kubeadm.CertificatePrivateKeyPair{ + Certificate: caSecret.Data[kubeadmconstants.CACertName], + PrivateKey: caSecret.Data[kubeadmconstants.CAKeyName], + } + + for _, i := range []string{AdminKubeConfigFileName, SuperAdminKubeConfigFileName} { + configuration.InitConfiguration.CertificatesDir, _ = os.MkdirTemp(tmp, "") + + kubeconfigValue, err := kubeadm.CreateKubeconfig(SuperAdminKubeConfigFileName, crtKeyPair, configuration) + if err != nil { + return nil, err + } + + _ = os.WriteFile(fmt.Sprintf("%s/%s", tmp, i), kubeconfigValue, os.ModePerm) + } + + if _, err = kubeconfig.EnsureAdminClusterRoleBinding(tmp, nil); err != nil { + return nil, err + } + + return nil, nil + }, nil default: return nil, fmt.Errorf("no available functionality for phase %s", r.Phase) } @@ -175,7 +226,7 @@ func (r *KubeadmPhase) UpdateTenantControlPlaneStatus(ctx context.Context, tenan func (r *KubeadmPhase) GetStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (kamajiv1alpha1.KubeadmConfigChecksumDependant, error) { switch r.Phase { - case PhaseUploadConfigKubeadm, PhaseUploadConfigKubelet: + case PhaseUploadConfigKubeadm, PhaseUploadConfigKubelet, PhaseClusterAdminRBAC: return nil, nil case PhaseBootstrapToken: return &tenantControlPlane.Status.KubeadmPhase.BootstrapToken, nil diff --git a/internal/resources/kubeadm_utils.go b/internal/resources/kubeadm_utils.go index 04571b7..28b4584 100644 --- a/internal/resources/kubeadm_utils.go +++ b/internal/resources/kubeadm_utils.go @@ -142,7 +142,7 @@ func KubeadmPhaseCreate(ctx context.Context, r KubeadmPhaseResource, logger logr return controllerutil.OperationResultNone, err } - fun, err := r.GetKubeadmFunction() + fun, err := r.GetKubeadmFunction(ctx, tenantControlPlane) if err != nil { logger.Error(err, "cannot retrieve kubeadm function") diff --git a/internal/resources/kubeconfig.go b/internal/resources/kubeconfig.go index 2cc97cc..d589bfa 100644 --- a/internal/resources/kubeconfig.go +++ b/internal/resources/kubeconfig.go @@ -24,6 +24,7 @@ import ( const ( AdminKubeConfigFileName = kubeadmconstants.AdminKubeConfigFileName + SuperAdminKubeConfigFileName = kubeadmconstants.SuperAdminKubeConfigFileName ControllerManagerKubeConfigFileName = kubeadmconstants.ControllerManagerKubeConfigFileName SchedulerKubeConfigFileName = kubeadmconstants.SchedulerKubeConfigFileName localhost = "127.0.0.1" @@ -102,7 +103,7 @@ func (r *KubeconfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, func (r *KubeconfigResource) getKubeconfigStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.KubeconfigStatus, error) { switch r.KubeConfigFileName { - case kubeadmconstants.AdminKubeConfigFileName: + case kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.SuperAdminKubeConfigFileName: return &tenantControlPlane.Status.KubeConfig.Admin, nil case kubeadmconstants.ControllerManagerKubeConfigFileName: return &tenantControlPlane.Status.KubeConfig.ControllerManager, nil @@ -181,6 +182,11 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam shouldCreate = shouldCreate || !kubeadm.IsKubeconfigValid(r.resource.Data[r.KubeConfigFileName]) // invalid kubeconfig, or expired client certificate shouldCreate = shouldCreate || status.Checksum != checksum || len(r.resource.UID) == 0 // Wrong checksum + if !shouldCreate { + v, ok := r.resource.Data[r.KubeConfigFileName] + shouldCreate = len(v) == 0 || !ok + } + if shouldCreate { crtKeyPair := kubeadm.CertificatePrivateKeyPair{ Certificate: caCertificatesSecret.Data[kubeadmconstants.CACertName], @@ -194,9 +200,11 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam return kcErr } - r.resource.Data = map[string][]byte{ - r.KubeConfigFileName: kubeconfig, + if r.resource.Data == nil { + r.resource.Data = map[string][]byte{} } + + r.resource.Data[r.KubeConfigFileName] = kubeconfig } return nil diff --git a/internal/resources/resource.go b/internal/resources/resource.go index a63d25f..a8c0a00 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -45,7 +45,7 @@ type KubeadmPhaseResource interface { Resource KubeadmResource GetClient() client.Client - GetKubeadmFunction() (func(clientset.Interface, *kubeadm.Configuration) ([]byte, error), error) + GetKubeadmFunction(context.Context, *kamajiv1alpha1.TenantControlPlane) (func(clientset.Interface, *kubeadm.Configuration) ([]byte, error), error) GetStatus(*kamajiv1alpha1.TenantControlPlane) (kamajiv1alpha1.KubeadmConfigChecksumDependant, error) SetKubeadmConfigChecksum(string) GetWatchedObject() client.Object diff --git a/internal/utilities/tenant_client.go b/internal/utilities/tenant_client.go index 20603c2..1531e98 100644 --- a/internal/utilities/tenant_client.go +++ b/internal/utilities/tenant_client.go @@ -44,7 +44,7 @@ func GetTenantKubeconfig(ctx context.Context, client client.Client, tenantContro return nil, err } - return DecodeKubeconfig(*secretKubeconfig, kubeadmconstants.AdminKubeConfigFileName) + return DecodeKubeconfig(*secretKubeconfig, kubeadmconstants.SuperAdminKubeConfigFileName) } func GetRESTClientConfig(ctx context.Context, client client.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) {