diff --git a/api/interfaces.go b/api/interfaces.go new file mode 100644 index 0000000..0ca2135 --- /dev/null +++ b/api/interfaces.go @@ -0,0 +1,6 @@ +package api + +type KubeadmConfigResourceVersionDependant interface { + GetKubeadmConfigResourceVersion() string + SetKubeadmConfigResourceVersion(string) +} diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index b1826d7..9213954 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -90,6 +90,20 @@ type ServiceSpec struct { ServiceType ServiceType `json:"serviceType"` } +// AddonSpec defines the spec for every addon. +type AddonSpec struct { + // +kubebuilder:default=true + Enabled *bool `json:"enabled,omitempty"` +} + +// AddonsSpec defines the enabled addons and their features. +type AddonsSpec struct { + // +kubebuilder:default={enabled: true} + CoreDNS AddonSpec `json:"coreDNS,omitempty"` + // +kubebuilder:default={enabled: true} + KubeProxy AddonSpec `json:"kubeProxy,omitempty"` +} + // TenantControlPlaneSpec defines the desired state of TenantControlPlane. type TenantControlPlaneSpec struct { ControlPlane ControlPlane `json:"controlPlane"` @@ -99,6 +113,10 @@ type TenantControlPlaneSpec struct { // NetworkProfile specifies how the network is NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"` + + // Addons contain which addons are enabled + // +kubebuilder:default={coreDNS: {enabled: true}, kubeProxy: {enabled: true}} + Addons AddonsSpec `json:"addons,omitempty"` } // ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. @@ -179,15 +197,42 @@ type KubeadmPhaseStatus struct { LastUpdate metav1.Time `json:"lastUpdate,omitempty"` } +func (d KubeadmPhaseStatus) GetKubeadmConfigResourceVersion() string { + return d.KubeadmConfigResourceVersion +} + +func (d *KubeadmPhaseStatus) SetKubeadmConfigResourceVersion(rv string) { + d.KubeadmConfigResourceVersion = rv +} + // KubeadmPhasesStatus contains the status of the different kubeadm phases action. type KubeadmPhasesStatus struct { UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"` UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"` - AddonCoreDNS KubeadmPhaseStatus `json:"addonCoreDNS"` - AddonKubeProxy KubeadmPhaseStatus `json:"addonKubeProxy"` BootstrapToken KubeadmPhaseStatus `json:"bootstrapToken"` } +// AddonStatus defines the observed state of an Addon. +type AddonStatus struct { + Enabled bool `json:"enabled"` + KubeadmConfigResourceVersion string `json:"kubeadmConfigResourceVersion,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +func (d AddonStatus) GetKubeadmConfigResourceVersion() string { + return d.KubeadmConfigResourceVersion +} + +func (d *AddonStatus) SetKubeadmConfigResourceVersion(rv string) { + d.KubeadmConfigResourceVersion = rv +} + +// AddonsStatus defines the observed state of the different Addons. +type AddonsStatus struct { + CoreDNS AddonStatus `json:"coreDNS,omitempty"` + KubeProxy AddonStatus `json:"kubeProxy,omitempty"` +} + // TenantControlPlaneStatus defines the observed state of TenantControlPlane. type TenantControlPlaneStatus struct { // Storage Status contains information about Kubernetes storage system @@ -205,6 +250,8 @@ type TenantControlPlaneStatus struct { KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"` // ControlPlaneEndpoint contains the status of the kubernetes control plane ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"` + // Addons contains the status of the different Addons + Addons AddonsStatus `json:"addons,omitempty"` } // KubernetesStatus defines the status of the resources deployed in the management cluster, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 469a9ed..94a6595 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -57,6 +57,76 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonSpec) DeepCopyInto(out *AddonSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec. +func (in *AddonSpec) DeepCopy() *AddonSpec { + if in == nil { + return nil + } + out := new(AddonSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonStatus) DeepCopyInto(out *AddonStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonStatus. +func (in *AddonStatus) DeepCopy() *AddonStatus { + if in == nil { + return nil + } + out := new(AddonStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) { + *out = *in + in.CoreDNS.DeepCopyInto(&out.CoreDNS) + in.KubeProxy.DeepCopyInto(&out.KubeProxy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec. +func (in *AddonsSpec) DeepCopy() *AddonsSpec { + if in == nil { + return nil + } + out := new(AddonsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) { + *out = *in + in.CoreDNS.DeepCopyInto(&out.CoreDNS) + in.KubeProxy.DeepCopyInto(&out.KubeProxy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus. +func (in *AddonsStatus) DeepCopy() *AddonsStatus { + if in == nil { + return nil + } + out := new(AddonsStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in AdmissionControllers) DeepCopyInto(out *AdmissionControllers) { { @@ -255,8 +325,6 @@ func (in *KubeadmPhasesStatus) DeepCopyInto(out *KubeadmPhasesStatus) { *out = *in in.UploadConfigKubeadm.DeepCopyInto(&out.UploadConfigKubeadm) in.UploadConfigKubelet.DeepCopyInto(&out.UploadConfigKubelet) - in.AddonCoreDNS.DeepCopyInto(&out.AddonCoreDNS) - in.AddonKubeProxy.DeepCopyInto(&out.AddonKubeProxy) in.BootstrapToken.DeepCopyInto(&out.BootstrapToken) } @@ -564,6 +632,7 @@ func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) { in.ControlPlane.DeepCopyInto(&out.ControlPlane) in.Kubernetes.DeepCopyInto(&out.Kubernetes) in.NetworkProfile.DeepCopyInto(&out.NetworkProfile) + in.Addons.DeepCopyInto(&out.Addons) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneSpec. @@ -585,6 +654,7 @@ func (in *TenantControlPlaneStatus) DeepCopyInto(out *TenantControlPlaneStatus) in.Kubernetes.DeepCopyInto(&out.Kubernetes) in.KubeadmConfig.DeepCopyInto(&out.KubeadmConfig) in.KubeadmPhase.DeepCopyInto(&out.KubeadmPhase) + in.Addons.DeepCopyInto(&out.Addons) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatus. diff --git a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml index a82633b..17abd58 100644 --- a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -60,6 +60,33 @@ spec: spec: description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. properties: + addons: + default: + coreDNS: + enabled: true + kubeProxy: + enabled: true + description: Addons contain which addons are enabled + properties: + coreDNS: + default: + enabled: true + description: AddonSpec defines the spec for every addon. + properties: + enabled: + default: true + type: boolean + type: object + kubeProxy: + default: + enabled: true + description: AddonSpec defines the spec for every addon. + properties: + enabled: + default: true + type: boolean + type: object + type: object controlPlane: description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number @@ -278,6 +305,36 @@ spec: status: description: TenantControlPlaneStatus defines the observed state of TenantControlPlane. properties: + addons: + description: Addons contains the status of the different Addons + properties: + coreDNS: + description: AddonStatus defines the observed state of an Addon. + properties: + enabled: + type: boolean + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + required: + - enabled + type: object + kubeProxy: + description: AddonStatus defines the observed state of an Addon. + properties: + enabled: + type: boolean + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + required: + - enabled + type: object + type: object certificates: description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane @@ -370,26 +427,6 @@ spec: description: KubeadmPhase contains the status of the kubeadm phases action properties: - addonCoreDNS: - description: KubeadmPhasesStatus contains the status of of a kubeadm - phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object - addonKubeProxy: - description: KubeadmPhasesStatus contains the status of of a kubeadm - phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object bootstrapToken: description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. @@ -421,8 +458,6 @@ spec: type: string type: object required: - - addonCoreDNS - - addonKubeProxy - bootstrapToken - uploadConfigKubeadm - uploadConfigKubelet diff --git a/config/install.yaml b/config/install.yaml index 6c67187..2fa3c38 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -60,6 +60,33 @@ spec: spec: description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. properties: + addons: + default: + coreDNS: + enabled: true + kubeProxy: + enabled: true + description: Addons contain which addons are enabled + properties: + coreDNS: + default: + enabled: true + description: AddonSpec defines the spec for every addon. + properties: + enabled: + default: true + type: boolean + type: object + kubeProxy: + default: + enabled: true + description: AddonSpec defines the spec for every addon. + properties: + enabled: + default: true + type: boolean + type: object + type: object controlPlane: description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress. properties: @@ -257,6 +284,36 @@ spec: status: description: TenantControlPlaneStatus defines the observed state of TenantControlPlane. properties: + addons: + description: Addons contains the status of the different Addons + properties: + coreDNS: + description: AddonStatus defines the observed state of an Addon. + properties: + enabled: + type: boolean + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + required: + - enabled + type: object + kubeProxy: + description: AddonStatus defines the observed state of an Addon. + properties: + enabled: + type: boolean + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + required: + - enabled + type: object + type: object certificates: description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane properties: @@ -343,24 +400,6 @@ spec: kubeadmPhase: description: KubeadmPhase contains the status of the kubeadm phases action properties: - addonCoreDNS: - description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object - addonKubeProxy: - description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object bootstrapToken: description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. properties: @@ -389,8 +428,6 @@ spec: type: string type: object required: - - addonCoreDNS - - addonKubeProxy - bootstrapToken - uploadConfigKubeadm - uploadConfigKubelet diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml index b3b3923..374dc87 100644 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml @@ -46,3 +46,8 @@ spec: podCidr: "10.244.0.0/16" dnsServiceIPs: - "10.96.0.10" + addons: + coreDNS: + enabled: true + kubeProxy: + enabled: true diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 0fd90b0..069d6db 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -224,35 +224,35 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R &resources.KubernetesIngressResource{ Client: r.Client, }, - &resources.KubeadmPhaseResource{ - Name: "upload-config-kubeadm", - Client: r.Client, - Log: log, - KubeadmPhase: resources.PhaseUploadConfigKubeadm, + &resources.KubeadmPhase{ + Name: "upload-config-kubeadm", + Client: r.Client, + Log: log, + Phase: resources.PhaseUploadConfigKubeadm, }, - &resources.KubeadmPhaseResource{ - Name: "upload-config-kubelet", - Client: r.Client, - Log: log, - KubeadmPhase: resources.PhaseUploadConfigKubelet, + &resources.KubeadmPhase{ + Name: "upload-config-kubelet", + Client: r.Client, + Log: log, + Phase: resources.PhaseUploadConfigKubelet, }, - &resources.KubeadmPhaseResource{ - Name: "addon-coredns", - Client: r.Client, - Log: log, - KubeadmPhase: resources.PhaseAddonCoreDNS, + &resources.KubeadmPhase{ + Name: "bootstrap-token", + Client: r.Client, + Log: log, + Phase: resources.PhaseBootstrapToken, }, - &resources.KubeadmPhaseResource{ - Name: "addon-kubeproxy", + &resources.KubeadmAddonResource{ + Name: "coredns", Client: r.Client, Log: log, - KubeadmPhase: resources.PhaseAddonKubeProxy, + KubeadmAddon: resources.AddonCoreDNS, }, - &resources.KubeadmPhaseResource{ - Name: "bootstrap-token", + &resources.KubeadmAddonResource{ + Name: "kubeproxy", Client: r.Client, Log: log, - KubeadmPhase: resources.PhaseBootstrapToken, + KubeadmAddon: resources.AddonKubeProxy, }, } diff --git a/internal/kubeadm/addon.go b/internal/kubeadm/addon.go index 16a6b81..aa821da 100644 --- a/internal/kubeadm/addon.go +++ b/internal/kubeadm/addon.go @@ -4,12 +4,14 @@ package kubeadm import ( + "context" "fmt" "time" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" clientcmdapi "k8s.io/client-go/tools/clientcmd/api/v1" @@ -22,11 +24,84 @@ import ( "k8s.io/utils/pointer" ) -func CoreDNSAddon(client kubernetes.Interface, config *Configuration) error { +const ( + kubeSystemNamespace = "kube-system" + kubeProxyName = "kube-proxy" + coreDNSName = "coredns" + kubeDNSName = "kube-dns" +) + +func AddCoreDNS(client kubernetes.Interface, config *Configuration) error { return dns.EnsureDNSAddon(&config.InitConfiguration.ClusterConfiguration, client) } -func KubeProxyAddon(client kubernetes.Interface, config *Configuration) error { +func RemoveCoreDNSAddon(ctx context.Context, client kubernetes.Interface) error { + var result error + + if err := removeCoreDNSService(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + if err := removeCoreDNSDeployment(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + if err := removeCoreDNSConfigMap(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + return result +} + +func removeCoreDNSService(ctx context.Context, client kubernetes.Interface) error { + name, _ := getCoreDNSServiceName(ctx) + opts := metav1.DeleteOptions{} + + return client.CoreV1().Services(kubeSystemNamespace).Delete(ctx, name, opts) +} + +func removeCoreDNSDeployment(ctx context.Context, client kubernetes.Interface) error { + name, _ := getCoreDNSDeploymentName(ctx) + opts := metav1.DeleteOptions{} + + return client.AppsV1().Deployments(kubeSystemNamespace).Delete(ctx, name, opts) +} + +func removeCoreDNSConfigMap(ctx context.Context, client kubernetes.Interface) error { + name, _ := getCoreDNSConfigMapName(ctx) + opts := metav1.DeleteOptions{} + + return client.CoreV1().ConfigMaps(kubeSystemNamespace).Delete(ctx, name, opts) +} + +func getCoreDNSServiceName(ctx context.Context) (string, error) { + // TODO: Currently, DNS is installed using kubeadm phases, therefore we know the name. + // Implement a method for future approaches + return kubeDNSName, nil +} + +func getCoreDNSDeploymentName(ctx context.Context) (string, error) { + // TODO: Currently, DNS is installed using kubeadm phases, therefore we know the name. + // Implement a method for future approaches + return coreDNSName, nil +} + +func getCoreDNSConfigMapName(ctx context.Context) (string, error) { + // TODO: Currently, DNS is installed using kubeadm phases, therefore we know the name. + // Implement a method for future approaches + return coreDNSName, nil +} + +func AddKubeProxy(client kubernetes.Interface, config *Configuration) error { if err := proxy.CreateServiceAccount(client); err != nil { return errors.Wrap(err, "error when creating kube-proxy service account") } @@ -46,6 +121,95 @@ func KubeProxyAddon(client kubernetes.Interface, config *Configuration) error { return nil } +func RemoveKubeProxy(ctx context.Context, client kubernetes.Interface) error { + var result error + + if err := removeKubeProxyDaemonSet(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + if err := removeKubeProxyConfigMap(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + if err := removeKubeProxyRBAC(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + return result +} + +func removeKubeProxyDaemonSet(ctx context.Context, client kubernetes.Interface) error { + name, _ := getKubeProxyDaemonSetName(ctx) + opts := metav1.DeleteOptions{} + + return client.AppsV1().DaemonSets(kubeSystemNamespace).Delete(ctx, name, opts) +} + +func removeKubeProxyConfigMap(ctx context.Context, client kubernetes.Interface) error { + name, _ := getKubeProxyConfigMapName(ctx) + opts := metav1.DeleteOptions{} + + return client.CoreV1().ConfigMaps(kubeSystemNamespace).Delete(ctx, name, opts) +} + +func removeKubeProxyRBAC(ctx context.Context, client kubernetes.Interface) error { + // TODO: Currently, kube-proxy is installed using kubeadm phases, therefore, name is the same. + name, _ := getKubeProxyRBACName(ctx) + opts := metav1.DeleteOptions{} + var result error + + if err := client.RbacV1().RoleBindings(kubeSystemNamespace).Delete(ctx, name, opts); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + if err := client.RbacV1().Roles(kubeSystemNamespace).Delete(ctx, name, opts); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + if err := client.CoreV1().ServiceAccounts(kubeSystemNamespace).Delete(ctx, name, opts); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + result = err + } + + return result +} + +func getKubeProxyRBACName(ctx context.Context) (string, error) { + // TODO: Currently, kube-proxy is installed using kubeadm phases, therefore we know the name. + // Implement a method for future approaches + return kubeProxyName, nil +} + +func getKubeProxyDaemonSetName(ctx context.Context) (string, error) { + // TODO: Currently, kube-proxy is installed using kubeadm phases, therefore we know the name. + // Implement a method for future approaches + return kubeProxyName, nil +} + +func getKubeProxyConfigMapName(ctx context.Context) (string, error) { + // TODO: Currently, kube-proxy is installed using kubeadm phases, therefore we know the name. + // Implement a method for future approaches + return kubeProxyName, nil +} + func createKubeProxyConfigMap(client kubernetes.Interface, config *Configuration) error { configConf, err := getKubeproxyConfigmapContent(config) if err != nil { diff --git a/internal/resources/api_server_certificate.go b/internal/resources/api_server_certificate.go index afd46f6..8e126dd 100644 --- a/internal/resources/api_server_certificate.go +++ b/internal/resources/api_server_certificate.go @@ -37,7 +37,7 @@ func (r *APIServerCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControl return false } -func (r *APIServerCertificate) CleanUp(ctx context.Context) (bool, error) { +func (r *APIServerCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/api_server_kubelet_client_certificate.go b/internal/resources/api_server_kubelet_client_certificate.go index bc8fc32..cbc0d76 100644 --- a/internal/resources/api_server_kubelet_client_certificate.go +++ b/internal/resources/api_server_kubelet_client_certificate.go @@ -37,7 +37,7 @@ func (r *APIServerKubeletClientCertificate) ShouldCleanup(plane *kamajiv1alpha1. return false } -func (r *APIServerKubeletClientCertificate) CleanUp(ctx context.Context) (bool, error) { +func (r *APIServerKubeletClientCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/ca_certificate.go b/internal/resources/ca_certificate.go index 0b8bf53..b5aa08a 100644 --- a/internal/resources/ca_certificate.go +++ b/internal/resources/ca_certificate.go @@ -36,7 +36,7 @@ func (r *CACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) return false } -func (r *CACertificate) CleanUp(ctx context.Context) (bool, error) { +func (r *CACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/constants.go b/internal/resources/constants.go index 7b6c6f5..c8d6b79 100644 --- a/internal/resources/constants.go +++ b/internal/resources/constants.go @@ -4,5 +4,6 @@ package resources const ( - defaultIngressPort = 443 + kubeconfigAdminKeyName = "admin.conf" + defaultIngressPort = 443 ) diff --git a/internal/resources/etcd_ca_certificates.go b/internal/resources/etcd_ca_certificates.go index f206db4..8fc4dae 100644 --- a/internal/resources/etcd_ca_certificates.go +++ b/internal/resources/etcd_ca_certificates.go @@ -43,7 +43,7 @@ func (r *ETCDCACertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantC return false } -func (r *ETCDCACertificatesResource) CleanUp(ctx context.Context) (bool, error) { +func (r *ETCDCACertificatesResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/etcd_certificates.go b/internal/resources/etcd_certificates.go index ac20ff8..32efe44 100644 --- a/internal/resources/etcd_certificates.go +++ b/internal/resources/etcd_certificates.go @@ -40,7 +40,7 @@ func (r *ETCDCertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantCon return false } -func (r *ETCDCertificatesResource) CleanUp(ctx context.Context) (bool, error) { +func (r *ETCDCertificatesResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/etcd_setup.go b/internal/resources/etcd_setup.go index 358851b..bac92e1 100644 --- a/internal/resources/etcd_setup.go +++ b/internal/resources/etcd_setup.go @@ -52,7 +52,7 @@ func (r *ETCDSetupResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPla return false } -func (r *ETCDSetupResource) CleanUp(ctx context.Context) (bool, error) { +func (r *ETCDSetupResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/front-proxy-client-certificate.go b/internal/resources/front-proxy-client-certificate.go index 0c5b462..a3b4102 100644 --- a/internal/resources/front-proxy-client-certificate.go +++ b/internal/resources/front-proxy-client-certificate.go @@ -37,7 +37,7 @@ func (r *FrontProxyClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.Tenant return false } -func (r *FrontProxyClientCertificate) CleanUp(ctx context.Context) (bool, error) { +func (r *FrontProxyClientCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/front_proxy_ca_certificate.go b/internal/resources/front_proxy_ca_certificate.go index 2fc0d7f..a981527 100644 --- a/internal/resources/front_proxy_ca_certificate.go +++ b/internal/resources/front_proxy_ca_certificate.go @@ -36,7 +36,7 @@ func (r *FrontProxyCACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantCont return false } -func (r *FrontProxyCACertificate) CleanUp(ctx context.Context) (bool, error) { +func (r *FrontProxyCACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index 8dcafe6..f58be65 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -48,7 +48,7 @@ func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.Tenan return false } -func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context) (bool, error) { +func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/k8s_ingress_resource.go b/internal/resources/k8s_ingress_resource.go index ca7c61d..4660f9d 100644 --- a/internal/resources/k8s_ingress_resource.go +++ b/internal/resources/k8s_ingress_resource.go @@ -33,7 +33,7 @@ func (r *KubernetesIngressResource) ShouldCleanup(tenantControlPlane *kamajiv1al return !tenantControlPlane.Spec.ControlPlane.Ingress.Enabled } -func (r *KubernetesIngressResource) CleanUp(ctx context.Context) (bool, error) { +func (r *KubernetesIngressResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { if err := r.Client.Delete(ctx, r.resource); err != nil { if !k8serrors.IsNotFound(err) { return false, err diff --git a/internal/resources/k8s_service_resource.go b/internal/resources/k8s_service_resource.go index 7c40dc4..3d418f3 100644 --- a/internal/resources/k8s_service_resource.go +++ b/internal/resources/k8s_service_resource.go @@ -34,7 +34,7 @@ func (r *KubernetesServiceResource) ShouldCleanup(plane *kamajiv1alpha1.TenantCo return false } -func (r *KubernetesServiceResource) CleanUp(ctx context.Context) (bool, error) { +func (r *KubernetesServiceResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/kubeadm_addons.go b/internal/resources/kubeadm_addons.go new file mode 100644 index 0000000..fa44547 --- /dev/null +++ b/internal/resources/kubeadm_addons.go @@ -0,0 +1,185 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiapi "github.com/clastix/kamaji/api" + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" +) + +type KubeadmAddon int + +const ( + AddonCoreDNS KubeadmAddon = iota + AddonKubeProxy +) + +func (d KubeadmAddon) String() string { + return [...]string{"PhaseAddonCoreDNS", "PhaseAddonKubeProxy"}[d] +} + +type KubeadmAddonResource struct { + Client client.Client + Log logr.Logger + Name string + KubeadmAddon KubeadmAddon + kubeadmConfigResourceVersion string +} + +func (r *KubeadmAddonResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + addonSpec, err := r.getSpec(tenantControlPlane) + if err != nil { + return false + } + + i, err := r.GetStatus(tenantControlPlane) + if err != nil { + return false + } + + addonStatus, ok := i.(*kamajiv1alpha1.AddonStatus) + if !ok { + return false + } + + return *addonSpec.Enabled == addonStatus.Enabled +} + +func (r *KubeadmAddonResource) SetKubeadmConfigResourceVersion(rv string) { + r.kubeadmConfigResourceVersion = rv +} + +func (r *KubeadmAddonResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return !r.isStatusEqual(tenantControlPlane) +} + +func (r *KubeadmAddonResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + spec, err := r.getSpec(tenantControlPlane) + if err != nil { + return false + } + + return !*spec.Enabled +} + +func (r *KubeadmAddonResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + client, err := GetRESTClient(ctx, r, tenantControlPlane) + if err != nil { + return false, err + } + + fun, err := r.getRemoveAddonFunction() + if err != nil { + return false, err + } + + if err := fun(ctx, client); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *KubeadmAddonResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + return nil +} + +func (r *KubeadmAddonResource) GetKubeadmFunction() (func(clientset.Interface, *kubeadm.Configuration) error, error) { + switch r.KubeadmAddon { + case AddonCoreDNS: + return kubeadm.AddCoreDNS, nil + case AddonKubeProxy: + return kubeadm.AddKubeProxy, nil + + default: + return nil, fmt.Errorf("no available functionality for phase %s", r.KubeadmAddon) + } +} + +func (r *KubeadmAddonResource) getRemoveAddonFunction() (func(context.Context, clientset.Interface) error, error) { + switch r.KubeadmAddon { + case AddonCoreDNS: + return kubeadm.RemoveCoreDNSAddon, nil + case AddonKubeProxy: + return kubeadm.RemoveKubeProxy, nil + default: + return nil, fmt.Errorf("no available functionality for removing addon %s", r.KubeadmAddon) + } +} + +func (r *KubeadmAddonResource) GetClient() client.Client { + return r.Client +} + +func (r *KubeadmAddonResource) GetTmpDirectory() string { + return "" +} + +func (r *KubeadmAddonResource) GetName() string { + return r.Name +} + +func (r *KubeadmAddonResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + i, err := r.GetStatus(tenantControlPlane) + if err != nil { + return err + } + + addonSpec, err := r.getSpec(tenantControlPlane) + if err != nil { + return err + } + + status, ok := i.(*kamajiv1alpha1.AddonStatus) + if !ok { + return fmt.Errorf("error addon status") + } + + status.Enabled = *addonSpec.Enabled + status.LastUpdate = metav1.Now() + status.KubeadmConfigResourceVersion = r.kubeadmConfigResourceVersion + + return nil +} + +func (r *KubeadmAddonResource) GetStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (kamajiapi.KubeadmConfigResourceVersionDependant, error) { + switch r.KubeadmAddon { + case AddonCoreDNS: + return &tenantControlPlane.Status.Addons.CoreDNS, nil + case AddonKubeProxy: + return &tenantControlPlane.Status.Addons.KubeProxy, nil + default: + return nil, fmt.Errorf("%s has no addon status", r.KubeadmAddon) + } +} + +func (r *KubeadmAddonResource) getSpec(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.AddonSpec, error) { + switch r.KubeadmAddon { + case AddonCoreDNS: + return &tenantControlPlane.Spec.Addons.CoreDNS, nil + case AddonKubeProxy: + return &tenantControlPlane.Spec.Addons.KubeProxy, nil + default: + return nil, fmt.Errorf("%s has no spec", r.KubeadmAddon) + } +} + +func (r *KubeadmAddonResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return KubeadmPhaseCreate(ctx, r, tenantControlPlane) +} diff --git a/internal/resources/kubeadm_config.go b/internal/resources/kubeadm_config.go index 6f993eb..3088971 100644 --- a/internal/resources/kubeadm_config.go +++ b/internal/resources/kubeadm_config.go @@ -51,7 +51,7 @@ func (r *KubeadmConfigResource) ShouldCleanup(plane *kamajiv1alpha1.TenantContro return false } -func (r *KubeadmConfigResource) CleanUp(ctx context.Context) (bool, error) { +func (r *KubeadmConfigResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/kubeadm_phases.go b/internal/resources/kubeadm_phases.go index f50a4be..fa8ef40 100644 --- a/internal/resources/kubeadm_phases.go +++ b/internal/resources/kubeadm_phases.go @@ -6,86 +6,81 @@ package resources import ( "context" "fmt" - "time" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + kamajiapi "github.com/clastix/kamaji/api" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/kubeadm" - kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig" ) +type kubeadmPhase int + const kubeadmPhaseTimeout = 10 // seconds -type KubeadmPhase int - const ( - PhaseUploadConfigKubeadm KubeadmPhase = iota + PhaseUploadConfigKubeadm kubeadmPhase = iota PhaseUploadConfigKubelet - PhaseAddonCoreDNS - PhaseAddonKubeProxy PhaseBootstrapToken ) -func (d KubeadmPhase) String() string { +func (d kubeadmPhase) String() string { return [...]string{"PhaseUploadConfigKubeadm", "PhaseUploadConfigKubelet", "PhaseAddonCoreDNS", "PhaseAddonKubeProxy", "PhaseBootstrapToken"}[d] } -const ( - kubeconfigAdminKeyName = "admin.conf" -) - -type KubeadmPhaseResource struct { +type KubeadmPhase struct { Client client.Client Log logr.Logger Name string - KubeadmPhase KubeadmPhase + Phase kubeadmPhase kubeadmConfigResourceVersion string } -func (r *KubeadmPhaseResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - status, err := r.getStatus(tenantControlPlane) +func (r *KubeadmPhase) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + i, err := r.GetStatus(tenantControlPlane) if err != nil { return true } + status, ok := i.(*kamajiv1alpha1.KubeadmPhaseStatus) + if !ok { + return false + } + return status.KubeadmConfigResourceVersion == r.kubeadmConfigResourceVersion } -func (r *KubeadmPhaseResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { +func (r *KubeadmPhase) SetKubeadmConfigResourceVersion(rv string) { + r.kubeadmConfigResourceVersion = rv +} + +func (r *KubeadmPhase) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return !r.isStatusEqual(tenantControlPlane) } -func (r *KubeadmPhaseResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { +func (r *KubeadmPhase) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return false } -func (r *KubeadmPhaseResource) CleanUp(ctx context.Context) (bool, error) { +func (r *KubeadmPhase) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } -func (r *KubeadmPhaseResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { +func (r *KubeadmPhase) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { return nil } -func (r *KubeadmPhaseResource) getKubeadmPhaseFunction() (func(clientset.Interface, *kubeadm.Configuration) error, error) { - switch r.KubeadmPhase { +func (r *KubeadmPhase) GetKubeadmFunction() (func(clientset.Interface, *kubeadm.Configuration) error, error) { + switch r.Phase { case PhaseUploadConfigKubeadm: return kubeadm.UploadKubeadmConfig, nil case PhaseUploadConfigKubelet: return kubeadm.UploadKubeletConfig, nil - case PhaseAddonCoreDNS: - return kubeadm.CoreDNSAddon, nil - case PhaseAddonKubeProxy: - return kubeadm.KubeProxyAddon, nil case PhaseBootstrapToken: return func(client clientset.Interface, config *kubeadm.Configuration) error { bootstrapTokensEnrichment(config.InitConfiguration.BootstrapTokens) @@ -93,7 +88,7 @@ func (r *KubeadmPhaseResource) getKubeadmPhaseFunction() (func(clientset.Interfa return kubeadm.BootstrapToken(client, config) }, nil default: - return nil, fmt.Errorf("no available functionality for phase %s", r.KubeadmPhase) + return nil, fmt.Errorf("no available functionality for phase %s", r.Phase) } } @@ -117,155 +112,48 @@ func enrichBootstrapToken(bootstrapToken *bootstraptokenv1.BootstrapToken) { } } -func (r *KubeadmPhaseResource) GetClient() client.Client { +func (r *KubeadmPhase) GetClient() client.Client { return r.Client } -func (r *KubeadmPhaseResource) GetTmpDirectory() string { +func (r *KubeadmPhase) GetTmpDirectory() string { return "" } -func (r *KubeadmPhaseResource) GetName() string { +func (r *KubeadmPhase) GetName() string { return r.Name } -func (r *KubeadmPhaseResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { - status, err := r.getStatus(tenantControlPlane) +func (r *KubeadmPhase) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + i, err := r.GetStatus(tenantControlPlane) if err != nil { return err } - status.LastUpdate = metav1.Now() - status.KubeadmConfigResourceVersion = r.kubeadmConfigResourceVersion + kubeadmStatus, ok := i.(*kamajiv1alpha1.KubeadmPhaseStatus) + if !ok { + return fmt.Errorf("error status kubeadm phase") + } + + kubeadmStatus.LastUpdate = metav1.Now() + kubeadmStatus.KubeadmConfigResourceVersion = r.kubeadmConfigResourceVersion return nil } -func (r *KubeadmPhaseResource) getStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.KubeadmPhaseStatus, error) { - switch r.KubeadmPhase { +func (r *KubeadmPhase) GetStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (kamajiapi.KubeadmConfigResourceVersionDependant, error) { + switch r.Phase { case PhaseUploadConfigKubeadm: return &tenantControlPlane.Status.KubeadmPhase.UploadConfigKubeadm, nil case PhaseUploadConfigKubelet: return &tenantControlPlane.Status.KubeadmPhase.UploadConfigKubelet, nil - case PhaseAddonCoreDNS: - return &tenantControlPlane.Status.KubeadmPhase.AddonCoreDNS, nil - case PhaseAddonKubeProxy: - return &tenantControlPlane.Status.KubeadmPhase.AddonKubeProxy, nil case PhaseBootstrapToken: return &tenantControlPlane.Status.KubeadmPhase.BootstrapToken, nil default: - return nil, fmt.Errorf("%s is not a right kubeadm phase", r.KubeadmPhase) + return nil, fmt.Errorf("%s is not a right kubeadm phase", r.Phase) } } -func (r *KubeadmPhaseResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { - return r.reconcile(ctx, tenantControlPlane) -} - -func (r *KubeadmPhaseResource) reconcile(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { - config, resourceVersion, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) - if err != nil { - return controllerutil.OperationResultNone, err - } - - kubeconfig, err := r.getKubeconfig(ctx, tenantControlPlane) - if err != nil { - return controllerutil.OperationResultNone, err - } - - config.Kubeconfig = *kubeconfig - config.Parameters = kubeadm.Parameters{ - TenantControlPlaneName: tenantControlPlane.GetName(), - TenantDNSServiceIPs: tenantControlPlane.Spec.NetworkProfile.DNSServiceIPs, - TenantControlPlaneVersion: tenantControlPlane.Spec.Kubernetes.Version, - TenantControlPlanePodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, - TenantControlPlaneAddress: tenantControlPlane.Spec.NetworkProfile.Address, - TenantControlPlanePort: tenantControlPlane.Spec.NetworkProfile.Port, - TenantControlPlaneCGroupDriver: tenantControlPlane.Spec.Kubernetes.Kubelet.CGroupFS.String(), - } - - status, err := r.getStatus(tenantControlPlane) - if err != nil { - return controllerutil.OperationResultNone, err - } - - if resourceVersion == status.KubeadmConfigResourceVersion { - r.kubeadmConfigResourceVersion = resourceVersion - - return controllerutil.OperationResultNone, nil - } - - client, err := r.getRESTClient(ctx, tenantControlPlane) - if err != nil { - return controllerutil.OperationResultNone, err - } - - fun, err := r.getKubeadmPhaseFunction() - if err != nil { - return controllerutil.OperationResultNone, err - } - if err = fun(client, config); err != nil { - return controllerutil.OperationResultNone, err - } - - r.kubeadmConfigResourceVersion = resourceVersion - - if status.LastUpdate.IsZero() { - return controllerutil.OperationResultCreated, nil - } - - return controllerutil.OperationResultUpdated, nil -} - -func (r *KubeadmPhaseResource) getKubeconfigSecret(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*corev1.Secret, error) { - kubeconfigSecretName := tenantControlPlane.Status.KubeConfig.Admin.SecretName - namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: kubeconfigSecretName} - secret := &corev1.Secret{} - if err := r.Client.Get(ctx, namespacedName, secret); err != nil { - return nil, err - } - - return secret, nil -} - -func (r *KubeadmPhaseResource) getKubeconfig(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kubeconfigutil.Kubeconfig, error) { - secretKubeconfig, err := r.getKubeconfigSecret(ctx, tenantControlPlane) - if err != nil { - return nil, err - } - - bytes, ok := secretKubeconfig.Data[kubeconfigAdminKeyName] - if !ok { - return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeconfigAdminKeyName) - } - - return kubeconfigutil.GetKubeconfigFromBytes(bytes) -} - -func (r *KubeadmPhaseResource) getRESTClient(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*clientset.Clientset, error) { - config, err := r.getRESTClientConfig(ctx, tenantControlPlane) - if err != nil { - return nil, err - } - - return clientset.NewForConfig(config) -} - -func (r *KubeadmPhaseResource) getRESTClientConfig(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) { - kubeconfig, err := r.getKubeconfig(ctx, tenantControlPlane) - if err != nil { - return nil, err - } - - config := &restclient.Config{ - Host: fmt.Sprintf("https://%s:%d", getTenantControllerInternalFQDN(*tenantControlPlane), tenantControlPlane.Spec.NetworkProfile.Port), - TLSClientConfig: restclient.TLSClientConfig{ - CAData: kubeconfig.Clusters[0].Cluster.CertificateAuthorityData, - CertData: kubeconfig.AuthInfos[0].AuthInfo.ClientCertificateData, - KeyData: kubeconfig.AuthInfos[0].AuthInfo.ClientKeyData, - }, - Timeout: time.Second * kubeadmPhaseTimeout, - } - - return config, nil +func (r *KubeadmPhase) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return KubeadmPhaseCreate(ctx, r, tenantControlPlane) } diff --git a/internal/resources/kubeadm_upgrade.go b/internal/resources/kubeadm_upgrade.go index f5b70eb..65b4b1a 100644 --- a/internal/resources/kubeadm_upgrade.go +++ b/internal/resources/kubeadm_upgrade.go @@ -48,7 +48,7 @@ func (k *KubernetesUpgrade) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bo return false } -func (k *KubernetesUpgrade) CleanUp(context.Context) (bool, error) { +func (k *KubernetesUpgrade) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/kubeadm_utils.go b/internal/resources/kubeadm_utils.go new file mode 100644 index 0000000..6fcd11c --- /dev/null +++ b/internal/resources/kubeadm_utils.go @@ -0,0 +1,126 @@ +package resources + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig" +) + +func KubeadmPhaseCreate(ctx context.Context, r KubeadmPhaseResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + config, resourceVersion, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + kubeconfig, err := getKubeconfig(ctx, r, tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + config.Kubeconfig = *kubeconfig + config.Parameters = kubeadm.Parameters{ + TenantControlPlaneName: tenantControlPlane.GetName(), + TenantDNSServiceIPs: tenantControlPlane.Spec.NetworkProfile.DNSServiceIPs, + TenantControlPlaneVersion: tenantControlPlane.Spec.Kubernetes.Version, + TenantControlPlanePodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, + TenantControlPlaneAddress: tenantControlPlane.Spec.NetworkProfile.Address, + TenantControlPlanePort: tenantControlPlane.Spec.NetworkProfile.Port, + TenantControlPlaneCGroupDriver: tenantControlPlane.Spec.Kubernetes.Kubelet.CGroupFS.String(), + } + + status, err := r.GetStatus(tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + storedResourceVersion := status.GetKubeadmConfigResourceVersion() + if resourceVersion == storedResourceVersion { + r.SetKubeadmConfigResourceVersion(resourceVersion) + + return controllerutil.OperationResultNone, nil + } + + client, err := GetRESTClient(ctx, r, tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + fun, err := r.GetKubeadmFunction() + if err != nil { + return controllerutil.OperationResultNone, err + } + if err = fun(client, config); err != nil { + return controllerutil.OperationResultNone, err + } + + r.SetKubeadmConfigResourceVersion(resourceVersion) + + if storedResourceVersion == "" { + return controllerutil.OperationResultCreated, nil + } + + return controllerutil.OperationResultUpdated, nil +} + +func getKubeconfigSecret(ctx context.Context, r KubeadmPhaseResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*corev1.Secret, error) { + kubeconfigSecretName := tenantControlPlane.Status.KubeConfig.Admin.SecretName + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: kubeconfigSecretName} + secret := &corev1.Secret{} + if err := r.GetClient().Get(ctx, namespacedName, secret); err != nil { + return nil, err + } + + return secret, nil +} + +func getKubeconfig(ctx context.Context, r KubeadmPhaseResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kubeconfigutil.Kubeconfig, error) { + secretKubeconfig, err := getKubeconfigSecret(ctx, r, tenantControlPlane) + if err != nil { + return nil, err + } + + bytes, ok := secretKubeconfig.Data[kubeconfigAdminKeyName] + if !ok { + return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeconfigAdminKeyName) + } + + return kubeconfigutil.GetKubeconfigFromBytes(bytes) +} + +func GetRESTClient(ctx context.Context, r KubeadmPhaseResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*clientset.Clientset, error) { + config, err := getRESTClientConfig(ctx, r, tenantControlPlane) + if err != nil { + return nil, err + } + + return clientset.NewForConfig(config) +} + +func getRESTClientConfig(ctx context.Context, r KubeadmPhaseResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) { + kubeconfig, err := getKubeconfig(ctx, r, tenantControlPlane) + if err != nil { + return nil, err + } + + config := &restclient.Config{ + Host: fmt.Sprintf("https://%s:%d", getTenantControllerInternalFQDN(*tenantControlPlane), tenantControlPlane.Spec.NetworkProfile.Port), + TLSClientConfig: restclient.TLSClientConfig{ + CAData: kubeconfig.Clusters[0].Cluster.CertificateAuthorityData, + CertData: kubeconfig.AuthInfos[0].AuthInfo.ClientCertificateData, + KeyData: kubeconfig.AuthInfos[0].AuthInfo.ClientKeyData, + }, + Timeout: time.Second * kubeadmPhaseTimeout, + } + + return config, nil +} diff --git a/internal/resources/kubeconfig.go b/internal/resources/kubeconfig.go index 446e45f..5fb050a 100644 --- a/internal/resources/kubeconfig.go +++ b/internal/resources/kubeconfig.go @@ -47,7 +47,7 @@ func (r *KubeconfigResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPl return false } -func (r *KubeconfigResource) CleanUp(ctx context.Context) (bool, error) { +func (r *KubeconfigResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } diff --git a/internal/resources/resource.go b/internal/resources/resource.go index 359c5c9..c753023 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -8,9 +8,11 @@ import ( corev1 "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + kamajiapi "github.com/clastix/kamaji/api" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/kubeadm" ) @@ -18,7 +20,7 @@ import ( type Resource interface { Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool - CleanUp(context.Context) (bool, error) + CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) CreateOrUpdate(context.Context, *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) GetName() string ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool @@ -35,6 +37,14 @@ type KubeadmResource interface { GetTmpDirectory() string } +type KubeadmPhaseResource interface { + KubeadmResource + GetClient() client.Client + GetKubeadmFunction() (func(clientset.Interface, *kubeadm.Configuration) error, error) + GetStatus(*kamajiv1alpha1.TenantControlPlane) (kamajiapi.KubeadmConfigResourceVersionDependant, error) + SetKubeadmConfigResourceVersion(string) +} + type HandlerConfig struct { Resource Resource TenantControlPlane *kamajiv1alpha1.TenantControlPlane @@ -50,7 +60,7 @@ func Handle(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1 return createOrUpdate(ctx, resource, tenantControlPlane) } - cleanUp, err := resource.CleanUp(ctx) + cleanUp, err := resource.CleanUp(ctx, tenantControlPlane) if err != nil { return controllerutil.OperationResultNone, err } diff --git a/internal/resources/sa_certificate.go b/internal/resources/sa_certificate.go index c38f540..3ee6d8f 100644 --- a/internal/resources/sa_certificate.go +++ b/internal/resources/sa_certificate.go @@ -36,7 +36,7 @@ func (r *SACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) return false } -func (r *SACertificate) CleanUp(ctx context.Context) (bool, error) { +func (r *SACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil }