From 11bf7a2dc486997603de9da5e150006dd10a18cf Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Tue, 28 Jul 2026 13:53:11 +0300 Subject: [PATCH] Add PodSecurityContext and Security Context to helm chart and cluster spec (#1058) * Add securityContext and podSecurityContext to the chart and cluster pods Signed-off-by: galal-hussein --- .../k3k/templates/crds/k3k.io_clusters.yaml | 470 +++++++++++++++++- charts/k3k/templates/deployment.yaml | 8 + charts/k3k/values.yaml | 8 + docs/crds/crds.adoc | 14 +- docs/crds/crds.md | 8 +- k3k-kubelet/provider/provider.go | 64 ++- pkg/apis/k3k.io/v1beta1/types.go | 22 +- .../k3k.io/v1beta1/zz_generated.deepcopy.go | 10 + pkg/controller/cluster/agent/shared.go | 10 + pkg/controller/cluster/agent/virtual.go | 9 + pkg/controller/cluster/cluster.go | 40 +- pkg/controller/cluster/server/server.go | 16 +- 12 files changed, 625 insertions(+), 54 deletions(-) diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index a44f6d72..15ed72f4 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -1340,6 +1340,235 @@ spec: description: Type specifies the persistence mode. type: string type: object + podSecurityContext: + description: |- + PodSecurityContext specifies custom PodSecurityContext to be added + to the agent and server pods of the cluster in virtual or shared mode. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object priorityClass: description: |- PriorityClass specifies the priorityClassName for server/agent pods. @@ -1447,8 +1676,8 @@ spec: type: array securityContext: description: |- - SecurityContext specifies custom SecurityContext to be added - to the agent and server pods of the cluster in virtual or shared mode. + SecurityContext specifies custom container SecurityContext to be added + to the agent and server containers of the cluster in virtual or shared mode. This option will override the SecurityContext set by default for virtual mode. properties: allowPrivilegeEscalation: @@ -3962,18 +4191,247 @@ spec: type: string description: nodeSelector is a node selector enforced by the active VirtualClusterPolicy. type: object + podSecurityContext: + description: |- + PodSecurityContext specifies custom SecurityContext to be added + to the agent and server pods of the cluster in virtual or shared mode. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object priorityClass: description: priorityClass is the priority class enforced by the active VirtualClusterPolicy. type: string runtimeClassName: description: |- - SecurityContext specifies custom SecurityContext to be added - to the agent and server pods of the cluster in virtual or shared mode. + RuntimeClassName specifies alternative runtime class for the + agent and server pods of the cluster in virtual or shared mode. type: string securityContext: description: |- - RuntimeClassName specifies alternative runtime class for the - agent and server pods of the cluster in virtual or shared mode. + SecurityContext specifies custom container SecurityContext to be added + to the agent and server containers of the cluster in virtual or shared mode. properties: allowPrivilegeEscalation: description: |- diff --git a/charts/k3k/templates/deployment.yaml b/charts/k3k/templates/deployment.yaml index 72e029a8..2a1860d8 100644 --- a/charts/k3k/templates/deployment.yaml +++ b/charts/k3k/templates/deployment.yaml @@ -15,6 +15,10 @@ spec: labels: {{- include "k3k.selectorLabels" . | nindent 8 }} spec: + {{- with .Values.controller.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} imagePullSecrets: {{- include "image.pullSecrets" (concat .Values.controller.imagePullSecrets .Values.global.imagePullSecrets) | nindent 8 }} containers: - image: "{{- include "controller.registry" .}}{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" @@ -54,4 +58,8 @@ spec: - containerPort: 8080 name: https protocol: TCP + {{- with .Values.controller.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} serviceAccountName: {{ include "k3k.serviceAccountName" . }} diff --git a/charts/k3k/values.yaml b/charts/k3k/values.yaml index 8ae4f307..c8921da6 100644 --- a/charts/k3k/values.yaml +++ b/charts/k3k/values.yaml @@ -52,6 +52,14 @@ controller: # memory: "100Mi" resources: {} + # podSecurityContext allows you to set custom securityContext for the controller + # deployment Pod spec + podSecurityContext: {} + + # securityContext allows you to set custom security context for the + # controller's Container + securityContext: {} + # configuration related to k3s server component in k3k server: imagePullSecrets: [] diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index eed155f9..881a3461 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -66,10 +66,12 @@ This includes both node affinity and pod affinity/anti-affinity rules. + | | | *`agentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy. + This includes both node affinity and pod affinity/anti-affinity rules. + | | | *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | sync is the SyncConfig enforced by the active VirtualClusterPolicy. + | | -| *`runtimeClassName`* __string__ | SecurityContext specifies custom SecurityContext to be added + -to the agent and server pods of the cluster in virtual or shared mode. + | | -| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | RuntimeClassName specifies alternative runtime class for the + +| *`runtimeClassName`* __string__ | RuntimeClassName specifies alternative runtime class for the + agent and server pods of the cluster in virtual or shared mode. + | | +| *`podSecurityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core[$$PodSecurityContext$$]__ | PodSecurityContext specifies custom SecurityContext to be added + +to the agent and server pods of the cluster in virtual or shared mode. + | | +| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | SecurityContext specifies custom container SecurityContext to be added + +to the agent and server containers of the cluster in virtual or shared mode. + | | | *`hostUsers`* __boolean__ | HostUsers sets the user namespace for server and agent pods. + If set to true or not present, the pod will be run in the host user namespace. + When set to false, a new userns is created for the pod. + @@ -227,8 +229,10 @@ are mirrored into the virtual cluster. + | | | *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. + | { } | | *`secretMounts`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretmount[$$SecretMount$$] array__ | SecretMounts specifies a list of secrets to mount into server and agent pods. + Each entry defines a secret and its mount path within the pods. + | | -| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | SecurityContext specifies custom SecurityContext to be added + -to the agent and server pods of the cluster in virtual or shared mode. + +| *`podSecurityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core[$$PodSecurityContext$$]__ | PodSecurityContext specifies custom PodSecurityContext to be added + +to the agent and server pods of the cluster in virtual or shared mode. + | | +| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | SecurityContext specifies custom container SecurityContext to be added + +to the agent and server containers of the cluster in virtual or shared mode. + This option will override the SecurityContext set by default for virtual mode. + | | | *`runtimeClassName`* __string__ | RuntimeClassName specifies alternative runtime class for the + agent and server pods of the cluster in virtual or shared mode. + | | diff --git a/docs/crds/crds.md b/docs/crds/crds.md index b6de132d..76373aa0 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -51,8 +51,9 @@ _Appears in:_ | `serverAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | serverAffinity is the affinity rules for server pods enforced by the active VirtualClusterPolicy.
This includes both node affinity and pod affinity/anti-affinity rules. | | | | `agentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy.
This includes both node affinity and pod affinity/anti-affinity rules. | | | | `sync` _[SyncConfig](#syncconfig)_ | sync is the SyncConfig enforced by the active VirtualClusterPolicy. | | | -| `runtimeClassName` _string_ | SecurityContext specifies custom SecurityContext to be added
to the agent and server pods of the cluster in virtual or shared mode. | | | -| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | RuntimeClassName specifies alternative runtime class for the
agent and server pods of the cluster in virtual or shared mode. | | | +| `runtimeClassName` _string_ | RuntimeClassName specifies alternative runtime class for the
agent and server pods of the cluster in virtual or shared mode. | | | +| `podSecurityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core)_ | PodSecurityContext specifies custom SecurityContext to be added
to the agent and server pods of the cluster in virtual or shared mode. | | | +| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | SecurityContext specifies custom container SecurityContext to be added
to the agent and server containers of the cluster in virtual or shared mode. | | | | `hostUsers` _boolean_ | HostUsers sets the user namespace for server and agent pods.
If set to true or not present, the pod will be run in the host user namespace.
When set to false, a new userns is created for the pod.
This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. | | | @@ -166,7 +167,8 @@ _Appears in:_ | `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. | | | | `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | | | `secretMounts` _[SecretMount](#secretmount) array_ | SecretMounts specifies a list of secrets to mount into server and agent pods.
Each entry defines a secret and its mount path within the pods. | | | -| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | SecurityContext specifies custom SecurityContext to be added
to the agent and server pods of the cluster in virtual or shared mode.
This option will override the SecurityContext set by default for virtual mode. | | | +| `podSecurityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core)_ | PodSecurityContext specifies custom PodSecurityContext to be added
to the agent and server pods of the cluster in virtual or shared mode. | | | +| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | SecurityContext specifies custom container SecurityContext to be added
to the agent and server containers of the cluster in virtual or shared mode.
This option will override the SecurityContext set by default for virtual mode. | | | | `runtimeClassName` _string_ | RuntimeClassName specifies alternative runtime class for the
agent and server pods of the cluster in virtual or shared mode. | | | | `hostUsers` _boolean_ | HostUsers sets the user namespace for server and agent pods.
If set to true or not present, the pod will be run in the host user namespace.
When set to false, a new userns is created for the pod.
This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. | | | diff --git a/k3k-kubelet/provider/provider.go b/k3k-kubelet/provider/provider.go index c685807e..9a1908e9 100644 --- a/k3k-kubelet/provider/provider.go +++ b/k3k-kubelet/provider/provider.go @@ -23,7 +23,6 @@ import ( "k8s.io/client-go/tools/portforward" "k8s.io/client-go/tools/remotecommand" "k8s.io/client-go/transport/spdy" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -493,7 +492,7 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error { } // inject networking information to the pod including the virtual cluster controlplane endpoint - configureNetworking(hostPod, virtualPod.Name, virtualPod.Namespace, p.serverIP, p.dnsIP) + configureNetworking(hostPod, &virtualPod, p.serverIP, p.dnsIP) // set ownerReference to the cluster object if err := controllerutil.SetControllerReference(&cluster, hostPod, p.Host.Client.Scheme()); err != nil { @@ -501,6 +500,11 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error { return err } + // harden coreDNS pod + if isCoreDNSPod(virtualPod) { + hardenCoreDNS(hostPod) + } + if err := p.Host.Client.Create(ctx, hostPod); err != nil { logger.Error(err, "Error creating pod on host cluster") return err @@ -836,12 +840,42 @@ func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) { return retPods, nil } +const nobodyUID = int64(65534) + +func isCoreDNSPod(pod corev1.Pod) bool { + return pod.Namespace == metav1.NamespaceSystem && pod.Labels["k8s-app"] == "kube-dns" +} + +// hardenCoreDNS will add security contexts needed for CoreDNS pod to run in +// restricted PSS mode, this is not possible by modifying the k3s manifests. +func hardenCoreDNS(pod *corev1.Pod) { + for i := range pod.Spec.Containers { + securityContext := pod.Spec.Containers[i].SecurityContext + if securityContext == nil { + securityContext = &corev1.SecurityContext{} + pod.Spec.Containers[i].SecurityContext = securityContext + } + + securityContext.AllowPrivilegeEscalation = new(false) + securityContext.Capabilities = &corev1.Capabilities{ + Add: []corev1.Capability{"NET_BIND_SERVICE"}, + Drop: []corev1.Capability{"ALL"}, + } + securityContext.RunAsNonRoot = new(true) + securityContext.RunAsUser = new(nobodyUID) + securityContext.RunAsGroup = new(nobodyUID) + securityContext.SeccompProfile = &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + } + } +} + // configureNetworking will inject network information to each pod to connect them to the // virtual cluster api server, as well as confiugre DNS information to connect them to the // synced coredns on the host cluster. -func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP string) { +func configureNetworking(hostPod, virtualPod *corev1.Pod, serverIP, dnsIP string) { // inject serverIP to hostalias for the pod - pod.Spec.HostAliases = append(pod.Spec.HostAliases, corev1.HostAlias{ + hostPod.Spec.HostAliases = append(hostPod.Spec.HostAliases, corev1.HostAlias{ IP: serverIP, Hostnames: []string{ "kubernetes", @@ -853,21 +887,21 @@ func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP }) // injecting cluster DNS IP to the pods except for coredns pod - if !strings.HasPrefix(podName, "coredns") && pod.Spec.DNSConfig == nil { - pod.Spec.DNSPolicy = corev1.DNSNone - pod.Spec.DNSConfig = &corev1.PodDNSConfig{ + if !isCoreDNSPod(*virtualPod) && hostPod.Spec.DNSConfig == nil { + hostPod.Spec.DNSPolicy = corev1.DNSNone + hostPod.Spec.DNSConfig = &corev1.PodDNSConfig{ Nameservers: []string{ dnsIP, }, Searches: []string{ - podNamespace + ".svc.cluster.local", + virtualPod.Namespace + ".svc.cluster.local", "svc.cluster.local", "cluster.local", }, Options: []corev1.PodDNSConfigOption{ { Name: "ndots", - Value: ptr.To("5"), + Value: new("5"), }, }, } @@ -881,18 +915,18 @@ func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP } // inject networking information to the pod's environment variables - for i := range pod.Spec.Containers { - pod.Spec.Containers[i].Env = mergeEnvVars(pod.Spec.Containers[i].Env, updatedEnvVars) + for i := range hostPod.Spec.Containers { + hostPod.Spec.Containers[i].Env = mergeEnvVars(hostPod.Spec.Containers[i].Env, updatedEnvVars) } // handle init containers as well - for i := range pod.Spec.InitContainers { - pod.Spec.InitContainers[i].Env = mergeEnvVars(pod.Spec.InitContainers[i].Env, updatedEnvVars) + for i := range hostPod.Spec.InitContainers { + hostPod.Spec.InitContainers[i].Env = mergeEnvVars(hostPod.Spec.InitContainers[i].Env, updatedEnvVars) } // handle ephemeral containers as well - for i := range pod.Spec.EphemeralContainers { - pod.Spec.EphemeralContainers[i].Env = mergeEnvVars(pod.Spec.EphemeralContainers[i].Env, updatedEnvVars) + for i := range hostPod.Spec.EphemeralContainers { + hostPod.Spec.EphemeralContainers[i].Env = mergeEnvVars(hostPod.Spec.EphemeralContainers[i].Env, updatedEnvVars) } } diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index f7464f8f..3c8eda82 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -213,8 +213,14 @@ type ClusterSpec struct { // +optional SecretMounts []SecretMount `json:"secretMounts,omitempty"` - // SecurityContext specifies custom SecurityContext to be added + // PodSecurityContext specifies custom PodSecurityContext to be added // to the agent and server pods of the cluster in virtual or shared mode. + // + // +optional + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + + // SecurityContext specifies custom container SecurityContext to be added + // to the agent and server containers of the cluster in virtual or shared mode. // This option will override the SecurityContext set by default for virtual mode. // // +optional @@ -683,14 +689,20 @@ type AppliedPolicy struct { // +optional Sync *SyncConfig `json:"sync,omitempty"` - // SecurityContext specifies custom SecurityContext to be added - // to the agent and server pods of the cluster in virtual or shared mode. + // RuntimeClassName specifies alternative runtime class for the + // agent and server pods of the cluster in virtual or shared mode. // // +optional RuntimeClassName *string `json:"runtimeClassName,omitempty"` - // RuntimeClassName specifies alternative runtime class for the - // agent and server pods of the cluster in virtual or shared mode. + // PodSecurityContext specifies custom SecurityContext to be added + // to the agent and server pods of the cluster in virtual or shared mode. + // + // +optional + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + + // SecurityContext specifies custom container SecurityContext to be added + // to the agent and server containers of the cluster in virtual or shared mode. // // +optional SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` diff --git a/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go index fd29ccb3..b945e83b 100644 --- a/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go @@ -60,6 +60,11 @@ func (in *AppliedPolicy) DeepCopyInto(out *AppliedPolicy) { *out = new(string) **out = **in } + if in.PodSecurityContext != nil { + in, out := &in.PodSecurityContext, &out.PodSecurityContext + *out = new(v1.PodSecurityContext) + (*in).DeepCopyInto(*out) + } if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext *out = new(v1.SecurityContext) @@ -257,6 +262,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.PodSecurityContext != nil { + in, out := &in.PodSecurityContext, &out.PodSecurityContext + *out = new(v1.PodSecurityContext) + (*in).DeepCopyInto(*out) + } if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext *out = new(v1.SecurityContext) diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index 91f70935..6f0e9f4a 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -251,6 +251,16 @@ func (s *SharedAgent) podSpec(ctx context.Context) corev1.PodSpec { podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{Name: imagePullSecret}) } + // pod security context + podSecurityContext := s.cluster.Spec.PodSecurityContext + if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.PodSecurityContext != nil { + log.V(1).Info("Using container pod securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name) + podSecurityContext = s.cluster.Status.Policy.PodSecurityContext + } + + podSpec.SecurityContext = podSecurityContext + + // container security context securityContext := s.cluster.Spec.SecurityContext if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.SecurityContext != nil { log.V(1).Info("Using securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name) diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 9c0862d0..18c7d2d9 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -299,6 +299,15 @@ func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) corev1.P podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{Name: imagePullSecret}) } + // pod security context + podSecurityContext := v.cluster.Spec.PodSecurityContext + if v.cluster.Status.Policy != nil && v.cluster.Status.Policy.PodSecurityContext != nil { + log.V(1).Info("Using container pod securityContext configuration from policy", "policyName", v.cluster.Status.PolicyName, "clusterName", v.cluster.Name) + podSecurityContext = v.cluster.Status.Policy.PodSecurityContext + } + + podSpec.SecurityContext = podSecurityContext + securityContext := v.cluster.Spec.SecurityContext if v.cluster.Status.Policy != nil && v.cluster.Status.Policy.SecurityContext != nil { log.V(1).Info("Using securityContext configuration from policy", "policyName", v.cluster.Status.PolicyName, "clusterName", v.cluster.Name) diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 066b51e7..f3679806 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -422,7 +422,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus serviceIP := service.Spec.ClusterIP - if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil { + if err := c.ensureClusterConfigs(ctx, cluster, s, serviceIP); err != nil { return err } @@ -534,37 +534,43 @@ func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster return err } -func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1beta1.Cluster, server *server.Server, serviceIP string) error { - // create init node config +func (c *ClusterReconciler) ensureClusterConfigs(ctx context.Context, cluster *v1beta1.Cluster, server *server.Server, serviceIP string) error { + // init node config initServerConfig, err := server.Config(true, serviceIP) if err != nil { return err } - if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Client.Scheme()); err != nil { + currentInitServerConfig := initServerConfig.DeepCopy() + if _, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentInitServerConfig, func() error { + if err := controllerutil.SetControllerReference(cluster, currentInitServerConfig, c.Client.Scheme()); err != nil { + return err + } + + currentInitServerConfig.Data = initServerConfig.Data + + return nil + }); err != nil { return err } - if err := c.Client.Create(ctx, initServerConfig); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - - // create servers configuration + // servers configuration serverConfig, err := server.Config(false, serviceIP) if err != nil { return err } - if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Client.Scheme()); err != nil { - return err - } - - if err := c.Client.Create(ctx, serverConfig); err != nil { - if !apierrors.IsAlreadyExists(err) { + currentServerConfig := serverConfig.DeepCopy() + if _, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerConfig, func() error { + if err := controllerutil.SetControllerReference(cluster, currentServerConfig, c.Client.Scheme()); err != nil { return err } + + currentServerConfig.Data = serverConfig.Data + + return nil + }); err != nil { + return err } return nil diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 8ea304cb..057d6d3e 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -11,7 +11,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -249,18 +248,29 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo }, }, } + // virtual mode runs an embedded kubelet inside the server pod and therefore // requires Privileged. shared and hcp modes are agentless (no kubelet) and // run unprivileged. if s.mode == string(v1beta1.VirtualClusterMode) { podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{ - Privileged: ptr.To(true), + Privileged: new(true), } } + // pod security context + podSecurityContext := s.cluster.Spec.PodSecurityContext + if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.PodSecurityContext != nil { + log.V(1).Info("Using container pod securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name) + podSecurityContext = s.cluster.Status.Policy.PodSecurityContext + } + + podSpec.SecurityContext = podSecurityContext + + // container security context securityContext := s.cluster.Spec.SecurityContext if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.SecurityContext != nil { - log.V(1).Info("Using securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name) + log.V(1).Info("Using container securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name) securityContext = s.cluster.Status.Policy.SecurityContext }