// Copyright 2022 Clastix Labs // SPDX-License-Identifier: Apache-2.0 package controlplane import ( "fmt" "github.com/blang/semver" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" pointer "k8s.io/utils/ptr" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/utilities" ) const ( AgentName = "konnectivity-agent" CertCommonName = "system:konnectivity-server" konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml" konnectivityServerName = "konnectivity-server" konnectivityServerPath = "/run/konnectivity" egressSelectorConfigurationVolume = "egress-selector-configuration" konnectivityUDSVolume = "konnectivity-uds" konnectivityServerKubeconfigVolume = "konnectivity-server-kubeconfig" ) type Konnectivity struct { Scheme runtime.Scheme } func (k Konnectivity) serverVersion(tcpVersion, addonVersion string) string { if addonVersion != "" { return addonVersion } version, parsedErr := semver.ParseTolerant(tcpVersion) if parsedErr != nil { return "" } return fmt.Sprintf("v0.%d.0", version.Minor) } func (k Konnectivity) buildKonnectivityContainer(tcpVersion string, addon *kamajiv1alpha1.KonnectivitySpec, replicas int32, podSpec *corev1.PodSpec) { found, index := utilities.HasNamedContainer(podSpec.Containers, konnectivityServerName) if !found { index = len(podSpec.Containers) podSpec.Containers = append(podSpec.Containers, corev1.Container{}) } podSpec.Containers[index].Name = konnectivityServerName podSpec.Containers[index].Image = fmt.Sprintf("%s:%s", addon.KonnectivityServerSpec.Image, k.serverVersion(tcpVersion, addon.KonnectivityServerSpec.Version)) podSpec.Containers[index].Command = []string{"/proxy-server"} args := utilities.ArgsFromSliceToMap(addon.KonnectivityServerSpec.ExtraArgs) args["--uds-name"] = fmt.Sprintf("%s/konnectivity-server.socket", konnectivityServerPath) args["--cluster-cert"] = "/etc/kubernetes/pki/apiserver.crt" args["--cluster-key"] = "/etc/kubernetes/pki/apiserver.key" args["--mode"] = "grpc" args["--server-port"] = "0" args["--agent-port"] = fmt.Sprintf("%d", addon.KonnectivityServerSpec.Port) args["--admin-port"] = "8133" args["--health-port"] = "8134" args["--agent-namespace"] = "kube-system" args["--agent-service-account"] = AgentName args["--kubeconfig"] = "/etc/kubernetes/konnectivity-server.conf" args["--authentication-audience"] = CertCommonName args["--server-count"] = fmt.Sprintf("%d", replicas) podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) podSpec.Containers[index].LivenessProbe = &corev1.Probe{ InitialDelaySeconds: 30, TimeoutSeconds: 60, PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt32(8134), Scheme: corev1.URISchemeHTTP, }, }, } podSpec.Containers[index].Ports = []corev1.ContainerPort{ { Name: "agentport", ContainerPort: addon.KonnectivityServerSpec.Port, Protocol: corev1.ProtocolTCP, }, { Name: "adminport", ContainerPort: 8133, Protocol: corev1.ProtocolTCP, }, { Name: "healthport", ContainerPort: 8134, Protocol: corev1.ProtocolTCP, }, } podSpec.Containers[index].VolumeMounts = []corev1.VolumeMount{ { Name: "etc-kubernetes-pki", MountPath: "/etc/kubernetes/pki", ReadOnly: true, }, { Name: "konnectivity-server-kubeconfig", MountPath: "/etc/kubernetes/konnectivity-server.conf", SubPath: "konnectivity-server.conf", ReadOnly: true, }, { Name: konnectivityUDSVolume, MountPath: konnectivityServerPath, ReadOnly: false, }, } podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways podSpec.Containers[index].Resources = corev1.ResourceRequirements{ Limits: nil, Requests: nil, } if resources := addon.KonnectivityServerSpec.Resources; resources != nil { podSpec.Containers[index].Resources.Limits = resources.Limits podSpec.Containers[index].Resources.Requests = resources.Requests } } func (k Konnectivity) RemovingVolumeMounts(podSpec *corev1.PodSpec) { found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName) if !found { return } for _, volumeMountName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume, konnectivityServerKubeconfigVolume} { if ok, i := utilities.HasNamedVolumeMount(podSpec.Containers[index].VolumeMounts, volumeMountName); ok { var volumesMounts []corev1.VolumeMount volumesMounts = append(volumesMounts, podSpec.Containers[index].VolumeMounts[:i]...) volumesMounts = append(volumesMounts, podSpec.Containers[index].VolumeMounts[i+1:]...) podSpec.Containers[index].VolumeMounts = volumesMounts } } } func (k Konnectivity) RemovingVolumes(podSpec *corev1.PodSpec) { for _, volumeName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume} { if volumeFound, volumeIndex := utilities.HasNamedVolume(podSpec.Volumes, volumeName); volumeFound { var volumes []corev1.Volume volumes = append(volumes, podSpec.Volumes[:volumeIndex]...) volumes = append(volumes, podSpec.Volumes[volumeIndex+1:]...) podSpec.Volumes = volumes } } } func (k Konnectivity) RemovingKubeAPIServerContainerArg(podSpec *corev1.PodSpec) { if found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName); found { argsMap := utilities.ArgsFromSliceToMap(podSpec.Containers[index].Args) if utilities.ArgsRemoveFlag(argsMap, "--egress-selector-config-file") { podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(argsMap) } } } func (k Konnectivity) RemovingContainer(podSpec *corev1.PodSpec) { if found, index := utilities.HasNamedContainer(podSpec.Containers, konnectivityServerName); found { var containers []corev1.Container containers = append(containers, podSpec.Containers[:index]...) containers = append(containers, podSpec.Containers[index+1:]...) podSpec.Containers = containers } } func (k Konnectivity) buildVolumeMounts(podSpec *corev1.PodSpec) { found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName) if !found { return } // Adding the egress selector config file flag args := utilities.ArgsFromSliceToMap(podSpec.Containers[index].Args) utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath) podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) vFound, vIndex := false, 0 //nolint:wastedassign // Patching the volume mounts if vFound, vIndex = utilities.HasNamedVolumeMount(podSpec.Containers[index].VolumeMounts, konnectivityUDSVolume); !vFound { vIndex = len(podSpec.Containers[index].VolumeMounts) podSpec.Containers[index].VolumeMounts = append(podSpec.Containers[index].VolumeMounts, corev1.VolumeMount{}) } podSpec.Containers[index].VolumeMounts[vIndex].Name = konnectivityUDSVolume podSpec.Containers[index].VolumeMounts[vIndex].ReadOnly = false podSpec.Containers[index].VolumeMounts[vIndex].MountPath = konnectivityServerPath if vFound, vIndex = utilities.HasNamedVolumeMount(podSpec.Containers[index].VolumeMounts, egressSelectorConfigurationVolume); !vFound { vIndex = len(podSpec.Containers[index].VolumeMounts) podSpec.Containers[index].VolumeMounts = append(podSpec.Containers[index].VolumeMounts, corev1.VolumeMount{}) } podSpec.Containers[index].VolumeMounts[vIndex].Name = egressSelectorConfigurationVolume podSpec.Containers[index].VolumeMounts[vIndex].ReadOnly = false podSpec.Containers[index].VolumeMounts[vIndex].MountPath = "/etc/kubernetes/konnectivity/configurations" } func (k Konnectivity) buildVolumes(status kamajiv1alpha1.KonnectivityStatus, podSpec *corev1.PodSpec) { // Defining volumes for the UDS socket found, index := utilities.HasNamedVolume(podSpec.Volumes, konnectivityUDSVolume) if !found { index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } podSpec.Volumes[index].Name = konnectivityUDSVolume podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ Medium: "Memory", }, } // Defining volumes for the egress selector configuration found, index = utilities.HasNamedVolume(podSpec.Volumes, egressSelectorConfigurationVolume) if !found { podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) index = len(podSpec.Volumes) - 1 } podSpec.Volumes[index].Name = egressSelectorConfigurationVolume podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: status.ConfigMap.Name, }, DefaultMode: pointer.To(int32(420)), }, } // Defining volume for the Konnectivity kubeconfig found, index = utilities.HasNamedVolume(podSpec.Volumes, konnectivityServerKubeconfigVolume) if !found { podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) index = len(podSpec.Volumes) - 1 } podSpec.Volumes[index].Name = konnectivityServerKubeconfigVolume podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: status.Kubeconfig.SecretName, DefaultMode: pointer.To(int32(420)), }, } } func (k Konnectivity) Build(deployment *appsv1.Deployment, tenantControlPlane kamajiv1alpha1.TenantControlPlane) { k.buildKonnectivityContainer(tenantControlPlane.Spec.Kubernetes.Version, tenantControlPlane.Spec.Addons.Konnectivity, *tenantControlPlane.Spec.ControlPlane.Deployment.Replicas, &deployment.Spec.Template.Spec) k.buildVolumeMounts(&deployment.Spec.Template.Spec) k.buildVolumes(tenantControlPlane.Status.Addons.Konnectivity, &deployment.Spec.Template.Spec) k.Scheme.Default(deployment) }