diff --git a/Makefile b/Makefile index bbfcd7c..15ddbaf 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,7 @@ kustomize: ## Download kustomize locally if necessary. manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml helm/kamaji/crds/tenantcontrolplane.yaml generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." diff --git a/api/v1alpha1/tenantcontrolplane_funcs.go b/api/v1alpha1/tenantcontrolplane_funcs.go index fc32779..3d80473 100644 --- a/api/v1alpha1/tenantcontrolplane_funcs.go +++ b/api/v1alpha1/tenantcontrolplane_funcs.go @@ -14,7 +14,7 @@ import ( kamajierrors "github.com/clastix/kamaji/internal/errors" ) -func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Client) (string, error) { +func (in *TenantControlPlane) GetControlPlaneAddress(ctx context.Context, client client.Client) (string, error) { var loadBalancerStatus corev1.LoadBalancerStatus svc := &corev1.Service{} diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index 4805556..6baca68 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -93,17 +93,32 @@ type ServiceSpec struct { } // AddonSpec defines the spec for every addon. -type AddonSpec struct { - // +kubebuilder:default=true - Enabled *bool `json:"enabled,omitempty"` +type AddonSpec struct{} + +// KonnectivitySpec defines the spec for Konnectivity. +type KonnectivitySpec struct { + // Port of Konnectivity proxy server. + // +kubebuilder:default=8132 + ProxyPort int32 `json:"proxyPort"` + // Host of Konnectivity proxy server. + ProxyHost string `json:"proxyHost,omitempty"` + AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"` + // Version for Konnectivity server and agent. + // +kubebuilder:default=v0.0.16 + Version string `json:"version,omitempty"` + // ServerImage defines the container image for Konnectivity's server. + // +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server + ServerImage string `json:"serverImage,omitempty"` + // AgentImage defines the container image for Konnectivity's agent. + // +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent + AgentImage string `json:"agentImage,omitempty"` } // AddonsSpec defines the enabled addons and their features. type AddonsSpec struct { - // +kubebuilder:default={enabled: true} - CoreDNS AddonSpec `json:"coreDNS,omitempty"` - // +kubebuilder:default={enabled: true} - KubeProxy AddonSpec `json:"kubeProxy,omitempty"` + CoreDNS *AddonSpec `json:"coreDNS,omitempty"` + Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"` + KubeProxy *AddonSpec `json:"kubeProxy,omitempty"` } // TenantControlPlaneSpec defines the desired state of TenantControlPlane. @@ -117,7 +132,6 @@ type TenantControlPlaneSpec struct { NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"` // Addons contain which addons are enabled - // +kubebuilder:default={coreDNS: {enabled: true}, kubeProxy: {enabled: true}} Addons AddonsSpec `json:"addons,omitempty"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 94a6595..e0b1363 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -60,11 +60,6 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddonSpec) DeepCopyInto(out *AddonSpec) { *out = *in - if in.Enabled != nil { - in, out := &in.Enabled, &out.Enabled - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec. @@ -96,8 +91,21 @@ func (in *AddonStatus) DeepCopy() *AddonStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) { *out = *in - in.CoreDNS.DeepCopyInto(&out.CoreDNS) - in.KubeProxy.DeepCopyInto(&out.KubeProxy) + if in.CoreDNS != nil { + in, out := &in.CoreDNS, &out.CoreDNS + *out = new(AddonSpec) + **out = **in + } + if in.Konnectivity != nil { + in, out := &in.Konnectivity, &out.Konnectivity + *out = new(KonnectivitySpec) + **out = **in + } + if in.KubeProxy != nil { + in, out := &in.KubeProxy, &out.KubeProxy + *out = new(AddonSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec. @@ -288,6 +296,42 @@ func (in *IngressSpec) DeepCopy() *IngressSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivitySpec. +func (in *KonnectivitySpec) DeepCopy() *KonnectivitySpec { + if in == nil { + return nil + } + out := new(KonnectivitySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) { + *out = *in + in.Certificate.DeepCopyInto(&out.Certificate) + in.Kubeconfig.DeepCopyInto(&out.Kubeconfig) + in.ServiceAccount.DeepCopyInto(&out.ServiceAccount) + in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding) + in.Agent.DeepCopyInto(&out.Agent) + in.Service.DeepCopyInto(&out.Service) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus. +func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus { + if in == nil { + return nil + } + out := new(KonnectivityStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) { *out = *in diff --git a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml index 774a3b8..ade7f8a 100644 --- a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -61,30 +61,43 @@ spec: description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. properties: addons: - default: - coreDNS: - enabled: true - kubeProxy: - enabled: true description: Addons contain which addons are enabled properties: coreDNS: - default: - enabled: true description: AddonSpec defines the spec for every addon. + type: object + konnectivity: + description: KonnectivitySpec defines the spec for Konnectivity. properties: - enabled: - default: true + agentImage: + default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent + description: AgentImage defines the container image for Konnectivity's + agent. + type: string + allowAddressAsExternalIP: type: boolean + proxyHost: + description: Host of Konnectivity proxy server. + type: string + proxyPort: + default: 8132 + description: Port of Konnectivity proxy server. + format: int32 + type: integer + serverImage: + default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server + description: ServerImage defines the container image for Konnectivity's + server. + type: string + version: + default: v0.0.16 + description: Version for Konnectivity server and agent. + type: string + required: + - proxyPort type: object kubeProxy: - default: - enabled: true description: AddonSpec defines the spec for every addon. - properties: - enabled: - default: true - type: boolean type: object type: object controlPlane: diff --git a/config/install.yaml b/config/install.yaml index 048822a..20c1707 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -77,6 +77,15 @@ spec: default: true type: boolean type: object + konnectivity: + default: + enabled: true + description: AddonSpec defines the spec for every addon. + properties: + enabled: + default: true + type: boolean + type: object kubeProxy: default: enabled: true @@ -300,6 +309,12 @@ spec: required: - enabled type: object + konnectivity: + description: KonnectivityStatus defines the status of Konnectivity as Addon + properties: + egressSelectorConfigurationStatus: + type: boolean + type: object kubeProxy: description: AddonStatus defines the observed state of an Addon. properties: diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml index a415d8f..877a0d9 100644 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml @@ -47,7 +47,5 @@ spec: dnsServiceIPs: - "10.96.0.10" addons: - coreDNS: - enabled: true - kubeProxy: - enabled: true + coreDNS: {} + kubeProxy: {} diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml new file mode 100644 index 0000000..b4736d6 --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml @@ -0,0 +1,53 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tenantkind +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: service + serviceType: ClusterIP + ingress: + enabled: true + hostname: kamaji.local + ingressClassName: nginx + additionalMetadata: + annotations: + kubernetes.io/ingress.allow-http: "false" + nginx.ingress.kubernetes.io/secure-backends: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + kubernetes: + version: "v1.23.4" + kubelet: + cgroupfs: systemd + admissionControllers: + - LimitRanger + - ResourceQuota + networkProfile: + address: "172.18.0.2" + port: 6443 + domain: "clastix.labs" + serviceCidr: "10.96.0.0/16" + podCidr: "10.244.0.0/16" + dnsServiceIPs: + - "10.96.0.10" + # addons: + # coreDNS: + # # enabled: true + # kubeProxy: + # # enabled: false \ No newline at end of file diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml new file mode 100644 index 0000000..50b6d30 --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml @@ -0,0 +1,48 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tenant1 +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: service + serviceType: NodePort + ingress: + enabled: false + kubernetes: + version: "v1.23.4" + kubelet: + cgroupfs: cgroupfs + admissionControllers: + - LimitRanger + - ResourceQuota + networkProfile: + address: "172.18.0.2" + port: 31443 + domain: "clastix.labs" + serviceCidr: "10.96.0.0/16" + podCidr: "10.244.0.0/16" + dnsServiceIPs: + - "10.96.0.10" + addons: + konnectivity: + proxyPort: 31132 + proxyHost: "172.18.0.2" + version: v0.0.31 + coreDNS: {} + kubeProxy: diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml new file mode 100644 index 0000000..33ebde2 --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml @@ -0,0 +1,53 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tenantkind +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: service + serviceType: ClusterIP + ingress: + enabled: true + hostname: kamaji.local + ingressClassName: nginx + additionalMetadata: + annotations: + kubernetes.io/ingress.allow-http: "false" + nginx.ingress.kubernetes.io/secure-backends: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + kubernetes: + version: "v1.23.4" + kubelet: + cgroupfs: systemd + # admissionControllers: + # - LimitRanger + # - ResourceQuota + networkProfile: + address: "172.18.0.2" + port: 6443 + domain: "clastix.labs" + serviceCidr: "10.96.0.0/16" + podCidr: "10.244.0.0/16" + dnsServiceIPs: + - "10.96.0.10" + addons: + coreDNS: + enabled: false + # kubeProxy: + # enabled: false \ No newline at end of file diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml new file mode 100644 index 0000000..9355b2b --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml @@ -0,0 +1,40 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tenant-00 + namespace: tenants +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: tenant-00 + labels: + tenant.clastix.io: tenant-00 + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: tenant-00 + labels: + tenant.clastix.io: tenant-00 + kind.clastix.io: service + serviceType: LoadBalancer + ingress: + enabled: false + kubernetes: + version: v1.23.1 + kubelet: + cgroupfs: systemd + admissionControllers: + - ResourceQuota + - LimitRanger + networkProfile: + address: 192.168.32.150 + port: 6443 + domain: clastix.labs + serviceCidr: 10.96.0.0/16 + podCidr: 10.36.0.0/16 + dnsServiceIPs: + - 10.96.0.10 diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml new file mode 100644 index 0000000..658d7ec --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml @@ -0,0 +1,49 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: test + namespace: default +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: service + serviceType: ClusterIP + ingress: + enabled: false + hostname: kamaji.local + ingressClassName: nginx + additionalMetadata: + annotations: + kubernetes.io/ingress.allow-http: "false" + nginx.ingress.kubernetes.io/secure-backends: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + kubernetes: + version: "v1.23.1" + kubelet: + cgroupfs: systemd + admissionControllers: + - ResourceQuota + - LimitRanger + networkProfile: + address: "192.168.1.47" + port: 6443 + domain: "clastix.labs" + serviceCidr: "10.152.0.0/16" + podCidr: "10.1.0.0/16" + dnsServiceIPs: + - "10.152.183.10" diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml new file mode 100644 index 0000000..a6685ee --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml @@ -0,0 +1,41 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tenant1 +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: service + serviceType: NodePort + ingress: + enabled: false + kubernetes: + version: "v1.23.4" + kubelet: + cgroupfs: cgroupfs + admissionControllers: + - LimitRanger + - ResourceQuota + networkProfile: + address: "192.168.1.47" + port: 31443 + domain: "clastix.labs" + serviceCidr: "10.96.0.0/16" + podCidr: "10.244.0.0/16" + dnsServiceIPs: + - "10.96.0.10" diff --git a/controllers/resources.go b/controllers/resources.go new file mode 100644 index 0000000..779cdab --- /dev/null +++ b/controllers/resources.go @@ -0,0 +1,283 @@ +package controllers + +import ( + "fmt" + "strings" + + "github.com/go-logr/logr" + "github.com/google/uuid" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" + "github.com/clastix/kamaji/internal/resources/konnectivity" +) + +const ( + separator = "," +) + +type GroupResourceBuilderConfiguration struct { + client client.Client + log logr.Logger + tcpReconcilerConfig TenantControlPlaneReconcilerConfig + tenantControlPlane kamajiv1alpha1.TenantControlPlane +} + +type GroupDeleteableResourceBuilderConfiguration struct { + client client.Client + log logr.Logger + tcpReconcilerConfig TenantControlPlaneReconcilerConfig + tenantControlPlane kamajiv1alpha1.TenantControlPlane +} + +// GetResources returns a list of resources that will be used to provide tenant control planes +// Currently there is only a default approach +// TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration. +func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource { + return getDefaultResources(config) +} + +// GetDeleteableResources returns a list of resources that have to be deleted when tenant control planes are deleted +// Currently there is only a default approach +// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration +func GetDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource { + return getDefaultDeleteableResources(config) +} + +func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource { + resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...) + resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKonnectivityResources(config.client, config.tenantControlPlane)...) + resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...) + resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...) + resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...) + + return resources +} + +func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource { + return []resources.DeleteableResource{ + &resources.ETCDSetupResource{ + Name: "etcd-setup", + Client: config.client, + Log: config.log, + ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName), + ETCDCACertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName), + Endpoints: getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints), + }, + } +} + +func getUpgradeResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubernetesUpgrade{ + Name: "upgrade", + Client: c, + }, + } +} + +func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubernetesServiceResource{ + Client: c, + }, + } +} + +func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubeadmConfigResource{ + Name: "kubeadmconfig", + Port: tenantControlPlane.Spec.NetworkProfile.Port, + KubernetesVersion: tenantControlPlane.Spec.Kubernetes.Version, + PodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, + ServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR, + Domain: tenantControlPlane.Spec.NetworkProfile.Domain, + ETCDs: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints), + ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval, + Client: c, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + } +} + +func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.CACertificate{ + Name: "ca", + Client: c, + Log: log, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.FrontProxyCACertificate{ + Name: "front-proxy-ca-certificate", + Client: c, + Log: log, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.SACertificate{ + Name: "sa-certificate", + Client: c, + Log: log, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.APIServerCertificate{ + Name: "api-server-certificate", + Client: c, + Log: log, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.APIServerKubeletClientCertificate{ + Name: "api-server-kubelet-client-certificate", + Client: c, + Log: log, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.FrontProxyClientCertificate{ + Name: "front-proxy-client-certificate", + Client: c, + Log: log, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + } +} + +func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubeconfigResource{ + Name: "admin-kubeconfig", + Client: c, + Log: log, + KubeConfigFileName: resources.AdminKubeConfigFileName, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.KubeconfigResource{ + Name: "controller-manager-kubeconfig", + Client: c, + Log: log, + KubeConfigFileName: resources.ControllerManagerKubeConfigFileName, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + &resources.KubeconfigResource{ + Name: "scheduler-kubeconfig", + Client: c, + Log: log, + KubeConfigFileName: resources.SchedulerKubeConfigFileName, + TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane), + }, + } +} + +func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.ETCDCACertificatesResource{ + Name: "etcd-ca-certificates", + Client: c, + Log: log, + ETCDCASecretName: tcpReconcilerConfig.ETCDCASecretName, + ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace, + }, + &resources.ETCDCertificatesResource{ + Name: "etcd-certificates", + Client: c, + Log: log, + }, + &resources.ETCDSetupResource{ + Name: "etcd-setup", + Client: c, + Log: log, + ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName), + ETCDCACertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName), + Endpoints: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints), + }, + } +} + +func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubernetesDeploymentResource{ + Client: c, + ETCDEndpoints: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints), + ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval, + }, + } +} + +func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubernetesIngressResource{ + Client: c, + }, + } +} + +func getKubeadmPhaseResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubeadmPhase{ + Name: "upload-config-kubeadm", + Client: c, + Log: log, + Phase: resources.PhaseUploadConfigKubeadm, + }, + &resources.KubeadmPhase{ + Name: "upload-config-kubelet", + Client: c, + Log: log, + Phase: resources.PhaseUploadConfigKubelet, + }, + &resources.KubeadmPhase{ + Name: "bootstrap-token", + Client: c, + Log: log, + Phase: resources.PhaseBootstrapToken, + }, + } +} + +func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &resources.KubeadmAddonResource{ + Name: "coredns", + Client: c, + Log: log, + KubeadmAddon: resources.AddonCoreDNS, + }, + &resources.KubeadmAddonResource{ + Name: "kubeproxy", + Client: c, + Log: log, + KubeadmAddon: resources.AddonKubeProxy, + }, + } +} + +func getKonnectivityResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &konnectivity.EgressSelectorConfiguration{ + Client: c, + Name: "konnectivity-egress-selector-configuration", + }, + } +} + +func getArrayFromString(s string) []string { + var a []string + a = append(a, strings.Split(s, separator)...) + + return a +} + +func getNamespacedName(namespace string, name string) k8stypes.NamespacedName { + return k8stypes.NamespacedName{Namespace: namespace, Name: name} +} + +func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New()) +} diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index c19d443..60859ea 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -6,9 +6,7 @@ package controllers import ( "context" "fmt" - "strings" - "github.com/google/uuid" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -26,7 +24,6 @@ import ( ) const ( - separator = "," finalizer = "finalizer.kamaji.clastix.io" ) @@ -77,17 +74,13 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R } if markedToBeDeleted { - registeredDeleteableResources := []resources.DeleteableResource{ - &resources.ETCDSetupResource{ - Name: "etcd-setup", - Client: r.Client, - Scheme: r.Scheme, - Log: log, - ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName), - ETCDCACertsSecret: getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName), - Endpoints: getArrayFromString(r.Config.ETCDEndpoints), - }, + groupDeleteableResourceBuilderConfiguration := GroupDeleteableResourceBuilderConfiguration{ + client: r.Client, + log: log, + tcpReconcilerConfig: r.Config, + tenantControlPlane: *tenantControlPlane, } + registeredDeleteableResources := GetDeleteableResources(groupDeleteableResourceBuilderConfiguration) for _, resource := range registeredDeleteableResources { if err := resource.Delete(ctx, tenantControlPlane); err != nil { @@ -114,148 +107,13 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } - registeredResources := []resources.Resource{ - &resources.KubernetesUpgrade{ - Name: "upgrade", - Client: r.Client, - }, - &resources.KubernetesServiceResource{ - Client: r.Client, - }, - &resources.KubeadmConfigResource{ - Name: "kubeadmconfig", - Port: tenantControlPlane.Spec.NetworkProfile.Port, - KubernetesVersion: tenantControlPlane.Spec.Kubernetes.Version, - PodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, - ServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR, - Domain: tenantControlPlane.Spec.NetworkProfile.Domain, - ETCDs: getArrayFromString(r.Config.ETCDEndpoints), - ETCDCompactionInterval: r.Config.ETCDCompactionInterval, - Client: r.Client, - Scheme: r.Scheme, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.CACertificate{ - Name: "ca", - Client: r.Client, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.FrontProxyCACertificate{ - Name: "front-proxy-ca-certificate", - Client: r.Client, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.SACertificate{ - Name: "sa-certificate", - Client: r.Client, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.APIServerCertificate{ - Name: "api-server-certificate", - Client: r.Client, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.APIServerKubeletClientCertificate{ - Name: "api-server-kubelet-client-certificate", - Client: r.Client, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.FrontProxyClientCertificate{ - Name: "front-proxy-client-certificate", - Client: r.Client, - Log: log, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.KubeconfigResource{ - Name: "admin-kubeconfig", - Client: r.Client, - Scheme: r.Scheme, - Log: log, - KubeConfigFileName: resources.AdminKubeConfigFileName, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.KubeconfigResource{ - Name: "controller-manager-kubeconfig", - Client: r.Client, - Scheme: r.Scheme, - Log: log, - KubeConfigFileName: resources.ControllerManagerKubeConfigFileName, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.KubeconfigResource{ - Name: "scheduler-kubeconfig", - Client: r.Client, - Scheme: r.Scheme, - Log: log, - KubeConfigFileName: resources.SchedulerKubeConfigFileName, - TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), - }, - &resources.ETCDCACertificatesResource{ - Name: "etcd-ca-certificates", - Client: r.Client, - Log: log, - ETCDCASecretName: r.Config.ETCDCASecretName, - ETCDCASecretNamespace: r.Config.ETCDCASecretNamespace, - }, - &resources.ETCDCertificatesResource{ - Name: "etcd-certificates", - Client: r.Client, - Log: log, - }, - &resources.ETCDSetupResource{ - Name: "etcd-setup", - Client: r.Client, - Scheme: r.Scheme, - Log: log, - ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName), - ETCDCACertsSecret: getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName), - Endpoints: getArrayFromString(r.Config.ETCDEndpoints), - }, - &resources.KubernetesDeploymentResource{ - Client: r.Client, - ETCDEndpoints: getArrayFromString(r.Config.ETCDEndpoints), - ETCDCompactionInterval: r.Config.ETCDCompactionInterval, - }, - &resources.KubernetesIngressResource{ - Client: r.Client, - }, - &resources.KubeadmPhase{ - Name: "upload-config-kubeadm", - Client: r.Client, - Log: log, - Phase: resources.PhaseUploadConfigKubeadm, - }, - &resources.KubeadmPhase{ - Name: "upload-config-kubelet", - Client: r.Client, - Log: log, - Phase: resources.PhaseUploadConfigKubelet, - }, - &resources.KubeadmPhase{ - Name: "bootstrap-token", - Client: r.Client, - Log: log, - Phase: resources.PhaseBootstrapToken, - }, - &resources.KubeadmAddonResource{ - Name: "coredns", - Client: r.Client, - Log: log, - KubeadmAddon: resources.AddonCoreDNS, - }, - &resources.KubeadmAddonResource{ - Name: "kubeproxy", - Client: r.Client, - Log: log, - KubeadmAddon: resources.AddonKubeProxy, - }, + groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{ + client: r.Client, + log: log, + tcpReconcilerConfig: r.Config, + tenantControlPlane: *tenantControlPlane, } + registeredResources := GetResources(groupResourceBuilderConfiguration) for _, resource := range registeredResources { result, err := resources.Handle(ctx, resource, tenantControlPlane) @@ -282,6 +140,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } + log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName())) + return ctrl.Result{}, nil } @@ -331,21 +191,6 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa return nil } -func getArrayFromString(s string) []string { - var a []string - a = append(a, strings.Split(s, separator)...) - - return a -} - -func getNamespacedName(namespace string, name string) k8stypes.NamespacedName { - return k8stypes.NamespacedName{Namespace: namespace, Name: name} -} - -func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { - return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New()) -} - func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool { for _, f := range tenantControlPlane.GetFinalizers() { if f == finalizer { diff --git a/docs/getting-started-with-kamaji.md b/docs/getting-started-with-kamaji.md index 8aa669d..16c0338 100644 --- a/docs/getting-started-with-kamaji.md +++ b/docs/getting-started-with-kamaji.md @@ -97,10 +97,8 @@ spec: dnsServiceIPs: - "10.96.0.10" addons: - coreDNS: - enabled: true - kubeProxy: - enabled: true + coreDNS: {} + kubeProxy: {} EOF ``` diff --git a/docs/kamaji-azure-deployment-guide.md b/docs/kamaji-azure-deployment-guide.md index ff8fc22..9052d9e 100644 --- a/docs/kamaji-azure-deployment-guide.md +++ b/docs/kamaji-azure-deployment-guide.md @@ -135,10 +135,8 @@ spec: dnsServiceIPs: - ${TENANT_DNS_SERVICE} addons: - coreDNS: - enabled: true - kubeProxy: - enabled: true + coreDNS: {} + kubeProxy: {} --- apiVersion: v1 kind: Service diff --git a/docs/kamaji-tenant-deployment-guide.md b/docs/kamaji-tenant-deployment-guide.md index ebea7e6..c73bb16 100644 --- a/docs/kamaji-tenant-deployment-guide.md +++ b/docs/kamaji-tenant-deployment-guide.md @@ -68,10 +68,8 @@ spec: dnsServiceIPs: - ${TENANT_DNS_SERVICE} addons: - coreDNS: - enabled: true - kubeProxy: - enabled: true + coreDNS: {} + kubeProxy: {} EOF ``` diff --git a/docs/reference.md b/docs/reference.md index bd22e73..59f2c79 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -89,20 +89,16 @@ It will generate a yaml installation file at `config/install.yaml`. It should be Kamaji provides optional installations into the deployed tenant control plane through add-ons. Is it possible to enable/disable them through the `tcp` definition. -By default, add-ons are installed if nothing is specified in the `tcp` definition. - ### Core DNS ```yaml addons: - coreDNS: - enabled: true + coreDNS: {} ``` ### Kube-Proxy ```yaml addons: - kubeProxy: - enabled: true + kubeProxy: {} ``` diff --git a/e2e/tenant_control_plane_ready_test.go b/e2e/tenant_control_plane_ready_test.go index a1d9eb1..122db92 100644 --- a/e2e/tenant_control_plane_ready_test.go +++ b/e2e/tenant_control_plane_ready_test.go @@ -50,6 +50,7 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() { "ResourceQuota", }, }, + Addons: kamajiv1alpha1.AddonsSpec{}, }, } diff --git a/e2e/worker_kubeadm_join_test.go b/e2e/worker_kubeadm_join_test.go index 212d1aa..87a6155 100644 --- a/e2e/worker_kubeadm_join_test.go +++ b/e2e/worker_kubeadm_join_test.go @@ -72,6 +72,7 @@ var _ = Describe("starting a kind worker with kubeadm", func() { "ResourceQuota", }, }, + Addons: kamajiv1alpha1.AddonsSpec{}, }, } Expect(k8sClient.Create(ctx, &tcp)).NotTo(HaveOccurred()) diff --git a/go.mod b/go.mod index 17a9320..5a0c7b0 100644 --- a/go.mod +++ b/go.mod @@ -142,7 +142,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiextensions-apiserver v0.23.5 // indirect - k8s.io/apiserver v0.23.5 // indirect k8s.io/cli-runtime v0.23.5 // indirect k8s.io/klog/v2 v2.60.1 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect diff --git a/helm/kamaji/README.md b/helm/kamaji/README.md index fc5451b..26665e9 100644 --- a/helm/kamaji/README.md +++ b/helm/kamaji/README.md @@ -55,8 +55,8 @@ Kubernetes: `>=1.18` | Key | Type | Default | Description | |-----|------|---------|-------------| -| addons.coreDNS.enabled | boolean | `true` | Enabling CoreDNS installation. If the value is not specified, the installation is enabled | -| addons.kubeProxy.enabled | boolean | `true` | Enabling KubeProxy installation. If the value is not specified, the installation is enabled | +| addons.coreDNS | object | | Enabling CoreDNS installation. | +| addons.kubeProxy | object | | Enabling KubeProxy installation | | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods | | configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") | | etcd.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") | diff --git a/helm/kamaji/values.yaml b/helm/kamaji/values.yaml index fd6e16e..b92d4fc 100644 --- a/helm/kamaji/values.yaml +++ b/helm/kamaji/values.yaml @@ -134,10 +134,3 @@ temporaryDirectoryPath: "/tmp/kamaji" loggingDevel: # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) enable: false - -# -- Kubernetes Addons -addons: - coreDNS: - enabled: true - kubeProxy: - enabled: true diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index d9f349c..dff53cf 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -13,6 +13,24 @@ import ( "time" ) +const ( + certBitSize = 2048 +) + +func GetCertificateAndKeyPair(template *x509.Certificate, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) { + caCertBytes, err := GetCertificate(caCert) + if err != nil { + return nil, nil, err + } + + caPrivKeyBytes, err := GetPrivateKey(caPrivKey) + if err != nil { + return nil, nil, err + } + + return GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes) +} + func GetCertificate(cert []byte) (*x509.Certificate, error) { pemContent, _ := pem.Decode(cert) if pemContent == nil { diff --git a/internal/etcd/certs.go b/internal/etcd/certs.go index 5d91fc6..42b4511 100644 --- a/internal/etcd/certs.go +++ b/internal/etcd/certs.go @@ -17,17 +17,7 @@ import ( func GetETCDCACertificateAndKeyPair(tenant string, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) { template := getCertTemplate(tenant) - caCertBytes, err := crypto.GetCertificate(caCert) - if err != nil { - return nil, nil, err - } - - caPrivKeyBytes, err := crypto.GetPrivateKey(caPrivKey) - if err != nil { - return nil, nil, err - } - - return crypto.GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes) + return crypto.GetCertificateAndKeyPair(template, caCert, caPrivKey) } func IsETCDCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) { diff --git a/internal/etcd/constant.go b/internal/etcd/constant.go index 86a672d..bc15b05 100644 --- a/internal/etcd/constant.go +++ b/internal/etcd/constant.go @@ -6,5 +6,4 @@ package etcd const ( certExpirationDelayYears = 10 certOrganization = "system:masters" - certBitSize = 2048 ) diff --git a/internal/kubeadm/addon.go b/internal/kubeadm/addon.go index aa821da..b0e67e0 100644 --- a/internal/kubeadm/addon.go +++ b/internal/kubeadm/addon.go @@ -22,6 +22,8 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/utils/pointer" + + "github.com/clastix/kamaji/internal/utilities" ) const ( @@ -412,7 +414,7 @@ func getKubeproxyConfigmapContent(config *Configuration) ([]byte, error) { }, } - return EncondeToYaml(&kubeProxyConfiguration) + return utilities.EncondeToYaml(&kubeProxyConfiguration) } func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) { @@ -447,5 +449,5 @@ func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) { }, } - return EncondeToYaml(&kubeconfig) + return utilities.EncondeToYaml(&kubeconfig) } diff --git a/internal/kubeadm/uploadconfig.go b/internal/kubeadm/uploadconfig.go index 8291598..21e053a 100644 --- a/internal/kubeadm/uploadconfig.go +++ b/internal/kubeadm/uploadconfig.go @@ -18,6 +18,8 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/apis/rbac" "k8s.io/utils/pointer" + + "github.com/clastix/kamaji/internal/utilities" ) func UploadKubeadmConfig(client kubernetes.Interface, config *Configuration) error { @@ -115,7 +117,7 @@ func getKubeletConfigmapContent(kubeletConfiguration KubeletConfiguration) ([]by VolumeStatsAggPeriod: zeroDuration, } - return EncondeToYaml(&kc) + return utilities.EncondeToYaml(&kc) } func createConfigMapRBACRules(client kubernetes.Interface, kubernetesVersion string) error { diff --git a/internal/kubeadm/utils.go b/internal/kubeadm/utils.go deleted file mode 100644 index 63fa14d..0000000 --- a/internal/kubeadm/utils.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2022 Clastix Labs -// SPDX-License-Identifier: Apache-2.0 - -package kubeadm - -import ( - "bytes" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer/json" -) - -func EncondeToYaml(o runtime.Object) ([]byte, error) { - scheme := runtime.NewScheme() - encoder := json.NewSerializerWithOptions(json.SimpleMetaFactory{}, scheme, scheme, json.SerializerOptions{}) - buf := bytes.NewBuffer([]byte{}) - err := encoder.Encode(o, buf) - - return buf.Bytes(), err -} diff --git a/internal/resources/etcd_setup.go b/internal/resources/etcd_setup.go index bac92e1..f7496b4 100644 --- a/internal/resources/etcd_setup.go +++ b/internal/resources/etcd_setup.go @@ -9,7 +9,6 @@ import ( "github.com/go-logr/logr" etcdclient "go.etcd.io/etcd/client/v3" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,7 +30,6 @@ type resource struct { type ETCDSetupResource struct { resource *resource Client client.Client - Scheme *runtime.Scheme Log logr.Logger Name string Endpoints []string diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index f58be65..9f834de 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -49,7 +49,7 @@ func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.Tenan } func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { - return false, nil + return tenantControlPlane.Spec.Addons.Konnectivity != nil, nil } func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { @@ -102,7 +102,7 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan etcdEndpoints[i] = fmt.Sprintf("https://%s", v) } - address, err := tenantControlPlane.GetAddress(ctx, r.Client) + address, err := tenantControlPlane.GetControlPlaneAddress(ctx, r.Client) if err != nil { return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create TenantControlPlane Deployment") } @@ -603,3 +603,163 @@ func (r *KubernetesDeploymentResource) isProvisioning(tenantControlPlane *kamaji func (r *KubernetesDeploymentResource) isNotReady() bool { return r.resource.Status.ReadyReplicas == 0 } + +func (r *KubernetesDeploymentResource) reconcileKonnectivity(podSpec *corev1.PodSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + return nil + } + + return r.addKonnectivity(podSpec, tenantControlPlane) +} + +func (r *KubernetesDeploymentResource) addKonnectivity(podSpec *corev1.PodSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) error { + flags := r.buildKonnectivityFlags() + podSpec.Containers[0].Command = append(podSpec.Containers[0].Command, flags...) + + volumes := r.buildKonnectivityVolumes(tenantControlPlane) + podSpec.Volumes = append(podSpec.Volumes, volumes...) + + volumeMounts := r.buildKonnectivityVolumeMounts() + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...) + + container := r.buildKonnectivityServerContainer(tenantControlPlane) + podSpec.Containers = append(podSpec.Containers, container) + + return nil +} + +func (r *KubernetesDeploymentResource) buildKonnectivityFlags() []string { + return []string{ + fmt.Sprintf("--egress-selector-config-file=%s", konnectivityEgressSelectorConfigurationPath), + } +} + +func (r *KubernetesDeploymentResource) buildKonnectivityVolumes(tenantControlPlane kamajiv1alpha1.TenantControlPlane) []corev1.Volume { + return []corev1.Volume{ + { + Name: konnectivityUDSName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: "Memory", + }, + }, + }, + { + Name: "egress-selector-configuration", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration, + }, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "konnectivity-server-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + } +} + +func (r *KubernetesDeploymentResource) buildKonnectivityVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: konnectivityUDSName, + ReadOnly: false, + MountPath: konnectivityServerPath, + }, + { + Name: "egress-selector-configuration", + ReadOnly: true, + MountPath: "/etc/kubernetes/konnectivity/configurations", + }, + } +} + +func (r *KubernetesDeploymentResource) buildKonnectivityServerContainer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) corev1.Container { + return corev1.Container{ + Name: konnectivityServerName, + Image: fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.ServerImage, tenantControlPlane.Spec.Addons.Konnectivity.Version), + Command: []string{"/proxy-server"}, + Args: []string{ + "-v=8", + "--logtostderr=true", + fmt.Sprintf("--uds-name=%s/konnectivity-server.socket", konnectivityServerPath), + "--cluster-cert=/etc/kubernetes/pki/apiserver.crt", + "--cluster-key=/etc/kubernetes/pki/apiserver.key", + "--mode=grpc", + "--server-port=0", + fmt.Sprintf("--agent-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort), + "--admin-port=8133", + "--health-port=8134", + "--agent-namespace=kube-system", + fmt.Sprintf("--agent-service-account=%s", konnectivity.AgentName), + "--kubeconfig=/etc/kubernetes/konnectivity-server.conf", + fmt.Sprintf("--authentication-audience=%s", konnectivity.CertCommonName), + fmt.Sprintf("--server-count=%d", tenantControlPlane.Spec.ControlPlane.Deployment.Replicas), + }, + LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 30, + TimeoutSeconds: 60, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8134), + Scheme: corev1.URISchemeHTTP, + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: quantity.MustParse("100m"), + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "agentport", + ContainerPort: tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "adminport", + ContainerPort: 8133, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "healthport", + ContainerPort: 8134, + Protocol: corev1.ProtocolTCP, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "etc-kubernetes-pki", + MountPath: "/etc/kubernetes/pki", + ReadOnly: true, + }, + { + Name: "konnectivity-server-kubeconfig", + MountPath: "/etc/kubernetes/konnectivity-server.conf", + SubPath: "konnectivity-server.conf", + ReadOnly: true, + }, + { + Name: "konnectivity-uds", + MountPath: konnectivityServerPath, + ReadOnly: false, + }, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: corev1.PullIfNotPresent, + } +} diff --git a/internal/resources/k8s_service_resource.go b/internal/resources/k8s_service_resource.go index 3d418f3..a15c58c 100644 --- a/internal/resources/k8s_service_resource.go +++ b/internal/resources/k8s_service_resource.go @@ -64,7 +64,7 @@ func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPla func (r *KubernetesServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { // We don't need to check error here: in case of dynamic external IP, the Service must be created in advance. // After that, the specific cloud controller-manager will provide an IP that will be then used. - address, _ := tenantControlPlane.GetAddress(ctx, r.Client) + address, _ := tenantControlPlane.GetControlPlaneAddress(ctx, r.Client) return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error { var servicePort corev1.ServicePort diff --git a/internal/resources/konnectivity/agent.go b/internal/resources/konnectivity/agent.go new file mode 100644 index 0000000..96e6234 --- /dev/null +++ b/internal/resources/konnectivity/agent.go @@ -0,0 +1,209 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type Agent struct { + resource *appsv1.DaemonSet + Client client.Client + Name string + tenantClient client.Client +} + +func (r *Agent) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Addons.Konnectivity.Agent.Name != r.resource.GetName() || + tenantControlPlane.Status.Addons.Konnectivity.Agent.Namespace != r.resource.GetNamespace() || + tenantControlPlane.Status.Addons.Konnectivity.Agent.RV != r.resource.ObjectMeta.ResourceVersion +} + +func (r *Agent) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *Agent) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.tenantClient.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *Agent) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: AgentName, + Namespace: kubeSystemNamespace, + }, + } + + client, err := NewClient(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + r.tenantClient = client + + return nil +} + +func (r *Agent) GetClient() client.Client { + return r.Client +} + +func (r *Agent) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *Agent) GetName() string { + return r.Name +} + +func (r *Agent) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{ + Name: r.resource.GetName(), + Namespace: r.resource.GetNamespace(), + RV: r.resource.ObjectMeta.ResourceVersion, + LastUpdate: metav1.Now(), + } + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.Enabled = false + tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{} + + return nil +} + +func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + address := tenantControlPlane.Spec.Addons.Konnectivity.ProxyHost + if address == "" { + address = tenantControlPlane.Spec.NetworkProfile.Address + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "k8s-app": AgentName, + "addonmanager.kubernetes.io/mode": "Reconcile", + }, + )) + + r.resource.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": AgentName, + }, + } + + r.resource.Spec.Template.SetLabels(utilities.MergeMaps( + r.resource.Spec.Template.GetLabels(), + map[string]string{ + "k8s-app": AgentName, + }, + )) + + r.resource.Spec.Template.Spec = corev1.PodSpec{ + PriorityClassName: "system-cluster-critical", + Tolerations: []corev1.Toleration{ + { + Key: "CriticalAddonsOnly", + Operator: "Exists", + }, + }, + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + Containers: []corev1.Container{ + { + Image: fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.AgentImage, tenantControlPlane.Spec.Addons.Konnectivity.Version), + Name: AgentName, + Command: []string{"/proxy-agent"}, + Args: []string{ + "-v=8", + "--logtostderr=true", + "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + fmt.Sprintf("--proxy-server-host=%s", address), + fmt.Sprintf("--proxy-server-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort), + "--admin-server-port=8133", + "--health-server-port=8134", + "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", + }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/var/run/secrets/tokens", + Name: agentTokenName, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8134), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 15, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + ServiceAccountName: AgentName, + DeprecatedServiceAccount: AgentName, + RestartPolicy: "Always", + DNSPolicy: "ClusterFirst", + TerminationGracePeriodSeconds: pointer.Int64(30), + SchedulerName: "default-scheduler", + SecurityContext: &corev1.PodSecurityContext{}, + Volumes: []corev1.Volume{ + { + Name: agentTokenName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: agentTokenName, + Audience: tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.Name, + ExpirationSeconds: pointer.Int64(3600), + }, + }, + }, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + }, + } + + return nil + } +} diff --git a/internal/resources/konnectivity/certificate_resource.go b/internal/resources/konnectivity/certificate_resource.go new file mode 100644 index 0000000..4bb0f4d --- /dev/null +++ b/internal/resources/konnectivity/certificate_resource.go @@ -0,0 +1,184 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "bytes" + "context" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "math/rand" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/crypto" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type CertificateResource struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string +} + +func (r *CertificateResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName != r.resource.GetName() || + tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion != r.resource.ResourceVersion +} + +func (r *CertificateResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *CertificateResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.Client.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *CertificateResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *CertificateResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *CertificateResource) GetClient() client.Client { + return r.Client +} + +func (r *CertificateResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *CertificateResource) GetName() string { + return r.Name +} + +func (r *CertificateResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.Certificate.LastUpdate = metav1.Now() + tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName = r.resource.GetName() + tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion = r.resource.ResourceVersion + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.Certificate = kamajiv1alpha1.CertificatePrivateKeyPairStatus{} + tenantControlPlane.Status.Addons.Konnectivity.Enabled = false + + return nil +} + +func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestCARV := tenantControlPlane.Status.Certificates.CA.ResourceVersion + actualCARV := r.resource.GetLabels()["latest-ca-rv"] + if latestCARV == actualCARV { + isValid, err := isCertificateAndKeyPairValid( + r.resource.Data[corev1.TLSCertKey], + r.resource.Data[corev1.TLSPrivateKeyKey], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", konnectivityCertAndKeyBaseName, err.Error())) + } + if isValid { + return nil + } + } + + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName} + secretCA := &corev1.Secret{} + if err := r.Client.Get(ctx, namespacedName, secretCA); err != nil { + return err + } + + ca := kubeadm.CertificatePrivateKeyPair{ + Name: kubeadmconstants.CACertAndKeyBaseName, + Certificate: secretCA.Data[kubeadmconstants.CACertName], + PrivateKey: secretCA.Data[kubeadmconstants.CAKeyName], + } + cert, privKey, err := getCertificateAndKeyPair(ca.Certificate, ca.PrivateKey) + if err != nil { + return err + } + + r.resource.Type = corev1.SecretTypeTLS + r.resource.Data = map[string][]byte{ + corev1.TLSCertKey: cert.Bytes(), + corev1.TLSPrivateKeyKey: privKey.Bytes(), + } + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-ca-rv": latestCARV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} + +func getCertificateAndKeyPair(caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) { + template := getCertTemplate() + + return crypto.GetCertificateAndKeyPair(template, caCert, caPrivKey) +} + +func isCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) { + return crypto.IsValidCertificateKeyPairBytes(cert, privKey) +} + +func getCertTemplate() *x509.Certificate { + serialNumber := big.NewInt(rand.Int63()) + + return &x509.Certificate{ + PublicKeyAlgorithm: x509.RSA, + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: CertCommonName, + Organization: []string{certOrganization}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(certExpirationDelayYears, 0, 0), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + x509.ExtKeyUsageCodeSigning, + }, + KeyUsage: x509.KeyUsageDigitalSignature, + } +} diff --git a/internal/resources/konnectivity/cluster_role_binding_resource.go b/internal/resources/konnectivity/cluster_role_binding_resource.go new file mode 100644 index 0000000..a9feaf4 --- /dev/null +++ b/internal/resources/konnectivity/cluster_role_binding_resource.go @@ -0,0 +1,119 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "context" + + rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type ClusterRoleBindingResource struct { + resource *rbacv1.ClusterRoleBinding + Client client.Client + Name string + tenantClient client.Client +} + +func (r *ClusterRoleBindingResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.Name != r.resource.GetName() || + tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.RV != r.resource.ObjectMeta.ResourceVersion +} + +func (r *ClusterRoleBindingResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *ClusterRoleBindingResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.tenantClient.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *ClusterRoleBindingResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: CertCommonName, + }, + } + + client, err := NewClient(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + r.tenantClient = client + + return nil +} + +func (r *ClusterRoleBindingResource) GetClient() client.Client { + return r.Client +} + +func (r *ClusterRoleBindingResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *ClusterRoleBindingResource) GetName() string { + return r.Name +} + +func (r *ClusterRoleBindingResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding = kamajiv1alpha1.ExternalKubernetesObjectStatus{ + Name: r.resource.GetName(), + RV: r.resource.ObjectMeta.ResourceVersion, + } + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding = kamajiv1alpha1.ExternalKubernetesObjectStatus{} + tenantControlPlane.Status.Addons.Konnectivity.Enabled = false + + return nil +} + +func (r *ClusterRoleBindingResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "kubernetes.io/cluster-service": "true", + "addonmanager.kubernetes.io/mode": "Reconcile", + }, + )) + + r.resource.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: roleAuthDelegator, + } + + r.resource.Subjects = []rbacv1.Subject{ + { + APIGroup: rbacv1.GroupName, + Kind: rbacv1.UserKind, + Name: CertCommonName, + }, + } + + return nil + } +} diff --git a/internal/resources/konnectivity/egress_selector_configuration_resource.go b/internal/resources/konnectivity/egress_selector_configuration_resource.go new file mode 100644 index 0000000..636c677 --- /dev/null +++ b/internal/resources/konnectivity/egress_selector_configuration_resource.go @@ -0,0 +1,118 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 +package konnectivity + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiserverv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type EgressSelectorConfigurationResource struct { + resource *corev1.ConfigMap + Client client.Client + Name string +} + +func (r *EgressSelectorConfigurationResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *EgressSelectorConfigurationResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *EgressSelectorConfigurationResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.Client.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *EgressSelectorConfigurationResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *EgressSelectorConfigurationResource) GetName() string { + return r.Name +} + +func (r *EgressSelectorConfigurationResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration != r.resource.GetName() +} + +func (r *EgressSelectorConfigurationResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration = r.resource.GetName() + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration = "" + + return nil +} + +func (r *EgressSelectorConfigurationResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error { + return func() error { + r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels())) + + configuration := &apiserverv1alpha1.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: egressSelectorConfigurationKind, + APIVersion: apiServerAPIVersion, + }, + EgressSelections: []apiserverv1alpha1.EgressSelection{ + { + Name: egressSelectorConfigurationName, + Connection: apiserverv1alpha1.Connection{ + ProxyProtocol: apiserverv1alpha1.ProtocolGRPC, + Transport: &apiserverv1alpha1.Transport{ + UDS: &apiserverv1alpha1.UDSTransport{ + UDSName: defaultUDSName, + }, + }, + }, + }, + }, + } + + yamlConfiguration, err := utilities.EncondeToYaml(configuration) + if err != nil { + return err + } + + r.resource.Data = map[string]string{ + "egress-selector-configuration.yaml": string(yamlConfiguration), + } + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} + +func (r *EgressSelectorConfigurationResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} diff --git a/internal/resources/konnectivity/kubeconfig_resource.go b/internal/resources/konnectivity/kubeconfig_resource.go new file mode 100644 index 0000000..27a117c --- /dev/null +++ b/internal/resources/konnectivity/kubeconfig_resource.go @@ -0,0 +1,173 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type KubeconfigResource struct { + resource *corev1.Secret + Client client.Client + Name string +} + +func (r *KubeconfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName != r.resource.GetName() +} + +func (r *KubeconfigResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *KubeconfigResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.Client.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *KubeconfigResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *KubeconfigResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *KubeconfigResource) GetClient() client.Client { + return r.Client +} + +func (r *KubeconfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *KubeconfigResource) GetName() string { + return r.Name +} + +func (r *KubeconfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.LastUpdate = metav1.Now() + tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName = r.resource.GetName() + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.Enabled = false + tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig = kamajiv1alpha1.KubeconfigStatus{} + + return nil +} + +func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestCARV := tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion + actualCARV := r.resource.GetLabels()["latest-certificate-rv"] + if latestCARV == actualCARV { + return nil + } + + caNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName} + secretCA := &corev1.Secret{} + if err := r.Client.Get(ctx, caNamespacedName, secretCA); err != nil { + return err + } + + certificateNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName} + secretCertificate := &corev1.Secret{} + if err := r.Client.Get(ctx, certificateNamespacedName, secretCertificate); err != nil { + return err + } + + userName := CertCommonName + clusterName := defaultClusterName + contextName := fmt.Sprintf("%s@%s", userName, clusterName) + + kubeconfig := &clientcmdapiv1.Config{ + Kind: "Config", + APIVersion: kubeconfigAPIVersion, + AuthInfos: []clientcmdapiv1.NamedAuthInfo{ + { + Name: userName, + AuthInfo: clientcmdapiv1.AuthInfo{ + ClientKeyData: secretCertificate.Data[corev1.TLSPrivateKeyKey], + ClientCertificateData: secretCertificate.Data[corev1.TLSCertKey], + }, + }, + }, + Clusters: []clientcmdapiv1.NamedCluster{ + { + Name: clusterName, + Cluster: clientcmdapiv1.Cluster{ + Server: r.getServer(*tenantControlPlane), + CertificateAuthorityData: secretCA.Data[kubeadmconstants.CACertName], + }, + }, + }, + Contexts: []clientcmdapiv1.NamedContext{ + { + Name: contextName, + Context: clientcmdapiv1.Context{ + Cluster: clusterName, + AuthInfo: userName, + }, + }, + }, + CurrentContext: contextName, + } + + kubeconfigBytes, err := utilities.EncondeToYaml(kubeconfig) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + konnectivityKubeconfigFileName: kubeconfigBytes, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-certificate-rv": latestCARV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} + +func (r *KubeconfigResource) getServer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("https://%s:%d", "localhost", tenantControlPlane.Spec.NetworkProfile.Port) +} diff --git a/internal/resources/konnectivity/service_account_resource.go b/internal/resources/konnectivity/service_account_resource.go new file mode 100644 index 0000000..57f88d4 --- /dev/null +++ b/internal/resources/konnectivity/service_account_resource.go @@ -0,0 +1,108 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type ServiceAccountResource struct { + resource *corev1.ServiceAccount + Client client.Client + Name string + tenantClient client.Client +} + +func (r *ServiceAccountResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount.Name != r.resource.GetName() || + tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount.Namespace != r.resource.GetNamespace() || + tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount.RV != r.resource.ObjectMeta.ResourceVersion +} + +func (r *ServiceAccountResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *ServiceAccountResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.tenantClient.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *ServiceAccountResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "konnectivity-agent", + Namespace: kubeSystemNamespace, + }, + } + + client, err := NewClient(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + r.tenantClient = client + + return nil +} + +func (r *ServiceAccountResource) GetClient() client.Client { + return r.Client +} + +func (r *ServiceAccountResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *ServiceAccountResource) GetName() string { + return r.Name +} + +func (r *ServiceAccountResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount = kamajiv1alpha1.ExternalKubernetesObjectStatus{ + Name: r.resource.GetName(), + Namespace: r.resource.GetNamespace(), + RV: r.resource.ObjectMeta.ResourceVersion, + } + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.Enabled = false + tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount = kamajiv1alpha1.ExternalKubernetesObjectStatus{} + + return nil +} + +func (r *ServiceAccountResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "kubernetes.io/cluster-service": "true", + "addonmanager.kubernetes.io/mode": "Reconcile", + }, + )) + + return nil + } +} diff --git a/internal/resources/konnectivity/service_resource.go b/internal/resources/konnectivity/service_resource.go new file mode 100644 index 0000000..03221ca --- /dev/null +++ b/internal/resources/konnectivity/service_resource.go @@ -0,0 +1,178 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +// ServiceResource must be the first Resource processed by the TenantControlPlane: +// when a TenantControlPlan is expecting a dynamic IP address, the Service will get it from the controller-manager. +type ServiceResource struct { + resource *corev1.Service + Client client.Client + Name string +} + +func (r *ServiceResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + if len(r.resource.Status.Conditions) != len(tenantControlPlane.Status.Addons.Konnectivity.Service.Conditions) { + return true + } + + resourceIngresses := tenantControlPlane.Status.Addons.Konnectivity.Service.LoadBalancer.Ingress + statusIngresses := r.resource.Status.LoadBalancer.Ingress + + if len(resourceIngresses) != len(statusIngresses) { + return true + } + + for i := 0; i < len(resourceIngresses); i++ { + if resourceIngresses[i].Hostname != statusIngresses[i].Hostname || + resourceIngresses[i].IP != statusIngresses[i].IP || + len(resourceIngresses[i].Ports) != len(statusIngresses[i].Ports) { + return true + } + + resourcePorts := resourceIngresses[i].Ports + statusPorts := statusIngresses[i].Ports + for j := 0; j < len(resourcePorts); j++ { + if resourcePorts[j].Port != statusPorts[j].Port || + resourcePorts[j].Protocol != statusPorts[j].Protocol { + return true + } + } + } + + return false +} + +func (r *ServiceResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Spec.Addons.Konnectivity != nil +} + +func (r *ServiceResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.Client.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *ServiceResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.Addons.Konnectivity != nil { + tenantControlPlane.Status.Addons.Konnectivity.Service.Name = r.resource.GetName() + tenantControlPlane.Status.Addons.Konnectivity.Service.Namespace = r.resource.GetNamespace() + tenantControlPlane.Status.Addons.Konnectivity.Service.Port = r.resource.Spec.Ports[0].Port + tenantControlPlane.Status.Addons.Konnectivity.Service.ServiceStatus = r.resource.Status + tenantControlPlane.Status.Addons.Konnectivity.Enabled = true + + return nil + } + + tenantControlPlane.Status.Addons.Konnectivity.Service = corev1.ServiceStatus{} + tenantControlPlane.Status.Addons.Konnectivity.Enabled = false + + return nil +} + +func (r *ServiceResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *ServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *ServiceResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error { + namespacedName := k8stypes.NamespacedName{Namespace: r.resource.GetNamespace(), Name: r.resource.GetName()} + address, _ := tenantControlPlane.GetAddress(ctx, r.Client, namespacedName, tenantControlPlane.Spec.Addons.Konnectivity.ProxyHost) + if address == "" { + address = tenantControlPlane.Spec.NetworkProfile.Address + } + + return func() error { + var servicePort corev1.ServicePort + if len(r.resource.Spec.Ports) > 0 { + servicePort = r.resource.Spec.Ports[0] + } + servicePort.Protocol = corev1.ProtocolTCP + servicePort.Port = tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort + servicePort.TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort)) + + r.resource.Spec.Ports = []corev1.ServicePort{servicePort} + r.resource.Spec.Selector = map[string]string{ + "kamaji.clastix.io/soot": tenantControlPlane.GetName(), + } + + labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Labels) + r.resource.SetLabels(labels) + + annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Annotations) + r.resource.SetAnnotations(annotations) + + isIP := false + + switch { + case utilities.IsValidIP(address): + isIP = true + case !utilities.IsValidHostname(address): + return fmt.Errorf("%s is not a valid address for Konnectivity proxy server.", address) + } + + switch tenantControlPlane.Spec.ControlPlane.Service.ServiceType { + case kamajiv1alpha1.ServiceTypeLoadBalancer: + r.resource.Spec.Type = corev1.ServiceTypeLoadBalancer + + if isIP { + r.resource.Spec.LoadBalancerIP = address + } + case kamajiv1alpha1.ServiceTypeNodePort: + r.resource.Spec.Type = corev1.ServiceTypeNodePort + r.resource.Spec.Ports[0].NodePort = tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort + + if isIP && tenantControlPlane.Spec.Addons.Konnectivity.AllowAddressAsExternalIP { + r.resource.Spec.ExternalIPs = []string{address} + } + default: + r.resource.Spec.Type = corev1.ServiceTypeClusterIP + + if isIP && tenantControlPlane.Spec.Addons.Konnectivity.AllowAddressAsExternalIP { + r.resource.Spec.ExternalIPs = []string{address} + } + } + + return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} + +func (r *ServiceResource) GetName() string { + return r.Name +} + +func (r *ServiceResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} diff --git a/internal/resources/konnectivity/utils.go b/internal/resources/konnectivity/utils.go new file mode 100644 index 0000000..ed9316c --- /dev/null +++ b/internal/resources/konnectivity/utils.go @@ -0,0 +1,87 @@ +package konnectivity + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + restclient "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig" +) + +// TODO: refactor and merge with /internal/resources/kubeadm_utils.go +// Logic is pretty close +// https://github.com/clastix/kamaji/issues/63 + +const ( + kubeconfigAdminKeyName = "admin.conf" + timeout = 10 // seconds + kubeSystemNamespace = "kube-system" +) + +type ExternalKubernetesResource interface { + GetClient() client.Client +} + +func NewClient(ctx context.Context, r ExternalKubernetesResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (client.Client, error) { + options := client.Options{} + config, err := getRESTClientConfig(ctx, r, tenantControlPlane) + if err != nil { + return nil, err + } + + return client.New(config, options) +} + +func getKubeconfigSecret(ctx context.Context, r ExternalKubernetesResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*corev1.Secret, error) { + kubeconfigSecretName := tenantControlPlane.Status.KubeConfig.Admin.SecretName + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: kubeconfigSecretName} + secret := &corev1.Secret{} + if err := r.GetClient().Get(ctx, namespacedName, secret); err != nil { + return nil, err + } + + return secret, nil +} + +func getKubeconfig(ctx context.Context, r ExternalKubernetesResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kubeconfigutil.Kubeconfig, error) { + secretKubeconfig, err := getKubeconfigSecret(ctx, r, tenantControlPlane) + if err != nil { + return nil, err + } + + bytes, ok := secretKubeconfig.Data[kubeconfigAdminKeyName] + if !ok { + return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeconfigAdminKeyName) + } + + return kubeconfigutil.GetKubeconfigFromBytes(bytes) +} + +func getRESTClientConfig(ctx context.Context, r ExternalKubernetesResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) { + kubeconfig, err := getKubeconfig(ctx, r, tenantControlPlane) + if err != nil { + return nil, err + } + + config := &restclient.Config{ + Host: fmt.Sprintf("https://%s:%d", getTenantControllerInternalFQDN(*tenantControlPlane), tenantControlPlane.Spec.NetworkProfile.Port), + TLSClientConfig: restclient.TLSClientConfig{ + CAData: kubeconfig.Clusters[0].Cluster.CertificateAuthorityData, + CertData: kubeconfig.AuthInfos[0].AuthInfo.ClientCertificateData, + KeyData: kubeconfig.AuthInfos[0].AuthInfo.ClientKeyData, + }, + Timeout: time.Second * timeout, + } + + return config, nil +} + +func getTenantControllerInternalFQDN(tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("%s.%s.svc.cluster.local", tenantControlPlane.GetName(), tenantControlPlane.GetNamespace()) +} diff --git a/internal/resources/kubeadm_addons.go b/internal/resources/kubeadm_addons.go index fa44547..c6d0184 100644 --- a/internal/resources/kubeadm_addons.go +++ b/internal/resources/kubeadm_addons.go @@ -39,11 +39,6 @@ type KubeadmAddonResource struct { } func (r *KubeadmAddonResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - addonSpec, err := r.getSpec(tenantControlPlane) - if err != nil { - return false - } - i, err := r.GetStatus(tenantControlPlane) if err != nil { return false @@ -54,7 +49,7 @@ func (r *KubeadmAddonResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1. return false } - return *addonSpec.Enabled == addonStatus.Enabled + return addonStatus.KubeadmConfigResourceVersion == r.kubeadmConfigResourceVersion } func (r *KubeadmAddonResource) SetKubeadmConfigResourceVersion(rv string) { @@ -71,7 +66,7 @@ func (r *KubeadmAddonResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1. return false } - return !*spec.Enabled + return spec == nil } func (r *KubeadmAddonResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { @@ -141,17 +136,11 @@ func (r *KubeadmAddonResource) UpdateTenantControlPlaneStatus(ctx context.Contex return err } - addonSpec, err := r.getSpec(tenantControlPlane) - if err != nil { - return err - } - status, ok := i.(*kamajiv1alpha1.AddonStatus) if !ok { return fmt.Errorf("error addon status") } - status.Enabled = *addonSpec.Enabled status.LastUpdate = metav1.Now() status.KubeadmConfigResourceVersion = r.kubeadmConfigResourceVersion @@ -172,9 +161,9 @@ func (r *KubeadmAddonResource) GetStatus(tenantControlPlane *kamajiv1alpha1.Tena func (r *KubeadmAddonResource) getSpec(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.AddonSpec, error) { switch r.KubeadmAddon { case AddonCoreDNS: - return &tenantControlPlane.Spec.Addons.CoreDNS, nil + return tenantControlPlane.Spec.Addons.CoreDNS, nil case AddonKubeProxy: - return &tenantControlPlane.Spec.Addons.KubeProxy, nil + return tenantControlPlane.Spec.Addons.KubeProxy, nil default: return nil, fmt.Errorf("%s has no spec", r.KubeadmAddon) } diff --git a/internal/resources/kubeadm_config.go b/internal/resources/kubeadm_config.go index 3088971..0b5b382 100644 --- a/internal/resources/kubeadm_config.go +++ b/internal/resources/kubeadm_config.go @@ -7,10 +7,8 @@ import ( "context" "fmt" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -23,8 +21,6 @@ import ( type KubeadmConfigResource struct { resource *corev1.ConfigMap Client client.Client - Scheme *runtime.Scheme - Log logr.Logger Name string Port int32 Domain string @@ -37,7 +33,7 @@ type KubeadmConfigResource struct { } func (r *KubeadmConfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - address, err := tenantControlPlane.GetAddress(ctx, r.Client) + address, err := getAddress(ctx, r.Client, *tenantControlPlane) if err != nil { return true } @@ -71,7 +67,7 @@ func (r *KubeadmConfigResource) getPrefixedName(tenantControlPlane *kamajiv1alph } func (r *KubeadmConfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { - address, err := tenantControlPlane.GetAddress(ctx, r.Client) + address, err := getAddress(ctx, r.Client, *tenantControlPlane) if err != nil { return controllerutil.OperationResultNone, err } @@ -84,7 +80,7 @@ func (r *KubeadmConfigResource) GetName() string { } func (r *KubeadmConfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { - address, _ := tenantControlPlane.GetAddress(ctx, r.Client) + address, _ := getAddress(ctx, r.Client, *tenantControlPlane) tenantControlPlane.Status.KubeadmConfig.LastUpdate = metav1.Now() tenantControlPlane.Status.KubeadmConfig.ResourceVersion = r.resource.ObjectMeta.ResourceVersion @@ -140,3 +136,7 @@ func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.Tenant return nil } } + +func getAddress(ctx context.Context, client client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) (string, error) { + return tenantControlPlane.GetControlPlaneAddress(ctx, client) +} diff --git a/internal/resources/kubeconfig.go b/internal/resources/kubeconfig.go index 5fb050a..79ff059 100644 --- a/internal/resources/kubeconfig.go +++ b/internal/resources/kubeconfig.go @@ -10,7 +10,6 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ctrl "sigs.k8s.io/controller-runtime" @@ -32,7 +31,6 @@ const ( type KubeconfigResource struct { resource *corev1.Secret Client client.Client - Scheme *runtime.Scheme Log logr.Logger Name string KubeConfigFileName string diff --git a/internal/utilities/utilities.go b/internal/utilities/utilities.go index 2a2c1a6..c91a7be 100644 --- a/internal/utilities/utilities.go +++ b/internal/utilities/utilities.go @@ -4,8 +4,12 @@ package utilities import ( + "bytes" "fmt" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/constants" ) @@ -42,3 +46,13 @@ func MergeMaps(maps ...map[string]string) map[string]string { func AddTenantPrefix(name string, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { return fmt.Sprintf("%s%s%s", tenantControlPlane.GetName(), separator, name) } + +// EncondeToYaml returns the given object in yaml format and the error. +func EncondeToYaml(o runtime.Object) ([]byte, error) { + scheme := runtime.NewScheme() + encoder := json.NewYAMLSerializer(json.SimpleMetaFactory{}, scheme, scheme) + buf := bytes.NewBuffer([]byte{}) + err := encoder.Encode(o, buf) + + return buf.Bytes(), err +}