Add PodSecurityContext and Security Context to helm chart and cluster spec (#1058)

* Add securityContext and podSecurityContext to the chart and cluster pods

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

charts/k3k/templates/crds/k3k.io_clusters.yaml

@@ -1340,6 +1340,235 @@ spec:
                     description: Type specifies the persistence mode.
                     type: string
                 type: object
+              podSecurityContext:
+                description: |-
+                  PodSecurityContext specifies custom PodSecurityContext to be added
+                  to the agent and server pods of the cluster in virtual or shared mode.
+                properties:
+                  appArmorProfile:
+                    description: |-
+                      appArmorProfile is the AppArmor options to use by the containers in this pod.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    properties:
+                      localhostProfile:
+                        description: |-
+                          localhostProfile indicates a profile loaded on the node that should be used.
+                          The profile must be preconfigured on the node to work.
+                          Must match the loaded name of the profile.
+                          Must be set if and only if type is "Localhost".
+                        type: string
+                      type:
+                        description: |-
+                          type indicates which kind of AppArmor profile will be applied.
+                          Valid options are:
+                            Localhost - a profile pre-loaded on the node.
+                            RuntimeDefault - the container runtime's default profile.
+                            Unconfined - no AppArmor enforcement.
+                        type: string
+                    required:
+                    - type
+                    type: object
+                  fsGroup:
+                    description: |-
+                      A special supplemental group that applies to all containers in a pod.
+                      Some volume types allow the Kubelet to change the ownership of that volume
+                      to be owned by the pod:
+
+                      1. The owning GID will be the FSGroup
+                      2. The setgid bit is set (new files created in the volume will be owned by FSGroup)
+                      3. The permission bits are OR'd with rw-rw----
+
+                      If unset, the Kubelet will not modify the ownership and permissions of any volume.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    format: int64
+                    type: integer
+                  fsGroupChangePolicy:
+                    description: |-
+                      fsGroupChangePolicy defines behavior of changing ownership and permission of the volume
+                      before being exposed inside Pod. This field will only apply to
+                      volume types which support fsGroup based ownership(and permissions).
+                      It will have no effect on ephemeral volume types such as: secret, configmaps
+                      and emptydir.
+                      Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    type: string
+                  runAsGroup:
+                    description: |-
+                      The GID to run the entrypoint of the container process.
+                      Uses runtime default if unset.
+                      May also be set in SecurityContext.  If set in both SecurityContext and
+                      PodSecurityContext, the value specified in SecurityContext takes precedence
+                      for that container.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    format: int64
+                    type: integer
+                  runAsNonRoot:
+                    description: |-
+                      Indicates that the container must run as a non-root user.
+                      If true, the Kubelet will validate the image at runtime to ensure that it
+                      does not run as UID 0 (root) and fail to start the container if it does.
+                      If unset or false, no such validation will be performed.
+                      May also be set in SecurityContext.  If set in both SecurityContext and
+                      PodSecurityContext, the value specified in SecurityContext takes precedence.
+                    type: boolean
+                  runAsUser:
+                    description: |-
+                      The UID to run the entrypoint of the container process.
+                      Defaults to user specified in image metadata if unspecified.
+                      May also be set in SecurityContext.  If set in both SecurityContext and
+                      PodSecurityContext, the value specified in SecurityContext takes precedence
+                      for that container.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    format: int64
+                    type: integer
+                  seLinuxChangePolicy:
+                    description: |-
+                      seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                      It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                      Valid values are "MountOption" and "Recursive".
+
+                      "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                      This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                      "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                      This requires all Pods that share the same volume to use the same SELinux label.
+                      It is not possible to share the same volume among privileged and unprivileged Pods.
+                      Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                      whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                      CSIDriver instance. Other volumes are always re-labelled recursively.
+                      "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                      If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                      If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                      and "Recursive" for all other volumes.
+
+                      This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                      All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    type: string
+                  seLinuxOptions:
+                    description: |-
+                      The SELinux context to be applied to all containers.
+                      If unspecified, the container runtime will allocate a random SELinux context for each
+                      container.  May also be set in SecurityContext.  If set in
+                      both SecurityContext and PodSecurityContext, the value specified in SecurityContext
+                      takes precedence for that container.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    properties:
+                      level:
+                        description: Level is SELinux level label that applies to the container.
+                        type: string
+                      role:
+                        description: Role is a SELinux role label that applies to the container.
+                        type: string
+                      type:
+                        description: Type is a SELinux type label that applies to the container.
+                        type: string
+                      user:
+                        description: User is a SELinux user label that applies to the container.
+                        type: string
+                    type: object
+                  seccompProfile:
+                    description: |-
+                      The seccomp options to use by the containers in this pod.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    properties:
+                      localhostProfile:
+                        description: |-
+                          localhostProfile indicates a profile defined in a file on the node should be used.
+                          The profile must be preconfigured on the node to work.
+                          Must be a descending path, relative to the kubelet's configured seccomp profile location.
+                          Must be set if type is "Localhost". Must NOT be set for any other type.
+                        type: string
+                      type:
+                        description: |-
+                          type indicates which kind of seccomp profile will be applied.
+                          Valid options are:
+
+                          Localhost - a profile defined in a file on the node should be used.
+                          RuntimeDefault - the container runtime default profile should be used.
+                          Unconfined - no profile should be applied.
+                        type: string
+                    required:
+                    - type
+                    type: object
+                  supplementalGroups:
+                    description: |-
+                      A list of groups applied to the first process run in each container, in
+                      addition to the container's primary GID and fsGroup (if specified).  If
+                      the SupplementalGroupsPolicy feature is enabled, the
+                      supplementalGroupsPolicy field determines whether these are in addition
+                      to or instead of any group memberships defined in the container image.
+                      If unspecified, no additional groups are added, though group memberships
+                      defined in the container image may still be used, depending on the
+                      supplementalGroupsPolicy field.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    items:
+                      format: int64
+                      type: integer
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  supplementalGroupsPolicy:
+                    description: |-
+                      Defines how supplemental groups of the first container processes are calculated.
+                      Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                      (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                      and the container runtime must implement support for this feature.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    type: string
+                  sysctls:
+                    description: |-
+                      Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
+                      sysctls (by the container runtime) might fail to launch.
+                      Note that this field cannot be set when spec.os.name is windows.
+                    items:
+                      description: Sysctl defines a kernel parameter to be set
+                      properties:
+                        name:
+                          description: Name of a property to set
+                          type: string
+                        value:
+                          description: Value of a property to set
+                          type: string
+                      required:
+                      - name
+                      - value
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  windowsOptions:
+                    description: |-
+                      The Windows specific settings applied to all containers.
+                      If unspecified, the options within a container's SecurityContext will be used.
+                      If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
+                      Note that this field cannot be set when spec.os.name is linux.
+                    properties:
+                      gmsaCredentialSpec:
+                        description: |-
+                          GMSACredentialSpec is where the GMSA admission webhook
+                          (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
+                          GMSA credential spec named by the GMSACredentialSpecName field.
+                        type: string
+                      gmsaCredentialSpecName:
+                        description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
+                        type: string
+                      hostProcess:
+                        description: |-
+                          HostProcess determines if a container should be run as a 'Host Process' container.
+                          All of a Pod's containers must have the same effective HostProcess value
+                          (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).
+                          In addition, if HostProcess is true then HostNetwork must also be set to true.
+                        type: boolean
+                      runAsUserName:
+                        description: |-
+                          The UserName in Windows to run the entrypoint of the container process.
+                          Defaults to the user specified in image metadata if unspecified.
+                          May also be set in PodSecurityContext. If set in both SecurityContext and
+                          PodSecurityContext, the value specified in SecurityContext takes precedence.
+                        type: string
+                    type: object
+                type: object
               priorityClass:
                 description: |-
                   PriorityClass specifies the priorityClassName for server/agent pods.
@@ -1447,8 +1676,8 @@ spec:
                 type: array
               securityContext:
                 description: |-
-                  SecurityContext specifies custom SecurityContext to be added
-                  to the agent and server pods of the cluster in virtual or shared mode.
+                  SecurityContext specifies custom container SecurityContext to be added
+                  to the agent and server containers of the cluster in virtual or shared mode.
                   This option will override the SecurityContext set by default for virtual mode.
                 properties:
                   allowPrivilegeEscalation:
@@ -3962,18 +4191,247 @@ spec:
                       type: string
                     description: nodeSelector is a node selector enforced by the active VirtualClusterPolicy.
                     type: object
+                  podSecurityContext:
+                    description: |-
+                      PodSecurityContext specifies custom SecurityContext to be added
+                      to the agent and server pods of the cluster in virtual or shared mode.
+                    properties:
+                      appArmorProfile:
+                        description: |-
+                          appArmorProfile is the AppArmor options to use by the containers in this pod.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        properties:
+                          localhostProfile:
+                            description: |-
+                              localhostProfile indicates a profile loaded on the node that should be used.
+                              The profile must be preconfigured on the node to work.
+                              Must match the loaded name of the profile.
+                              Must be set if and only if type is "Localhost".
+                            type: string
+                          type:
+                            description: |-
+                              type indicates which kind of AppArmor profile will be applied.
+                              Valid options are:
+                                Localhost - a profile pre-loaded on the node.
+                                RuntimeDefault - the container runtime's default profile.
+                                Unconfined - no AppArmor enforcement.
+                            type: string
+                        required:
+                        - type
+                        type: object
+                      fsGroup:
+                        description: |-
+                          A special supplemental group that applies to all containers in a pod.
+                          Some volume types allow the Kubelet to change the ownership of that volume
+                          to be owned by the pod:
+
+                          1. The owning GID will be the FSGroup
+                          2. The setgid bit is set (new files created in the volume will be owned by FSGroup)
+                          3. The permission bits are OR'd with rw-rw----
+
+                          If unset, the Kubelet will not modify the ownership and permissions of any volume.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        format: int64
+                        type: integer
+                      fsGroupChangePolicy:
+                        description: |-
+                          fsGroupChangePolicy defines behavior of changing ownership and permission of the volume
+                          before being exposed inside Pod. This field will only apply to
+                          volume types which support fsGroup based ownership(and permissions).
+                          It will have no effect on ephemeral volume types such as: secret, configmaps
+                          and emptydir.
+                          Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        type: string
+                      runAsGroup:
+                        description: |-
+                          The GID to run the entrypoint of the container process.
+                          Uses runtime default if unset.
+                          May also be set in SecurityContext.  If set in both SecurityContext and
+                          PodSecurityContext, the value specified in SecurityContext takes precedence
+                          for that container.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        format: int64
+                        type: integer
+                      runAsNonRoot:
+                        description: |-
+                          Indicates that the container must run as a non-root user.
+                          If true, the Kubelet will validate the image at runtime to ensure that it
+                          does not run as UID 0 (root) and fail to start the container if it does.
+                          If unset or false, no such validation will be performed.
+                          May also be set in SecurityContext.  If set in both SecurityContext and
+                          PodSecurityContext, the value specified in SecurityContext takes precedence.
+                        type: boolean
+                      runAsUser:
+                        description: |-
+                          The UID to run the entrypoint of the container process.
+                          Defaults to user specified in image metadata if unspecified.
+                          May also be set in SecurityContext.  If set in both SecurityContext and
+                          PodSecurityContext, the value specified in SecurityContext takes precedence
+                          for that container.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        format: int64
+                        type: integer
+                      seLinuxChangePolicy:
+                        description: |-
+                          seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                          It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                          Valid values are "MountOption" and "Recursive".
+
+                          "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                          This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                          "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                          This requires all Pods that share the same volume to use the same SELinux label.
+                          It is not possible to share the same volume among privileged and unprivileged Pods.
+                          Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                          whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                          CSIDriver instance. Other volumes are always re-labelled recursively.
+                          "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                          If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                          If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                          and "Recursive" for all other volumes.
+
+                          This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                          All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        type: string
+                      seLinuxOptions:
+                        description: |-
+                          The SELinux context to be applied to all containers.
+                          If unspecified, the container runtime will allocate a random SELinux context for each
+                          container.  May also be set in SecurityContext.  If set in
+                          both SecurityContext and PodSecurityContext, the value specified in SecurityContext
+                          takes precedence for that container.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        properties:
+                          level:
+                            description: Level is SELinux level label that applies to the container.
+                            type: string
+                          role:
+                            description: Role is a SELinux role label that applies to the container.
+                            type: string
+                          type:
+                            description: Type is a SELinux type label that applies to the container.
+                            type: string
+                          user:
+                            description: User is a SELinux user label that applies to the container.
+                            type: string
+                        type: object
+                      seccompProfile:
+                        description: |-
+                          The seccomp options to use by the containers in this pod.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        properties:
+                          localhostProfile:
+                            description: |-
+                              localhostProfile indicates a profile defined in a file on the node should be used.
+                              The profile must be preconfigured on the node to work.
+                              Must be a descending path, relative to the kubelet's configured seccomp profile location.
+                              Must be set if type is "Localhost". Must NOT be set for any other type.
+                            type: string
+                          type:
+                            description: |-
+                              type indicates which kind of seccomp profile will be applied.
+                              Valid options are:
+
+                              Localhost - a profile defined in a file on the node should be used.
+                              RuntimeDefault - the container runtime default profile should be used.
+                              Unconfined - no profile should be applied.
+                            type: string
+                        required:
+                        - type
+                        type: object
+                      supplementalGroups:
+                        description: |-
+                          A list of groups applied to the first process run in each container, in
+                          addition to the container's primary GID and fsGroup (if specified).  If
+                          the SupplementalGroupsPolicy feature is enabled, the
+                          supplementalGroupsPolicy field determines whether these are in addition
+                          to or instead of any group memberships defined in the container image.
+                          If unspecified, no additional groups are added, though group memberships
+                          defined in the container image may still be used, depending on the
+                          supplementalGroupsPolicy field.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        items:
+                          format: int64
+                          type: integer
+                        type: array
+                        x-kubernetes-list-type: atomic
+                      supplementalGroupsPolicy:
+                        description: |-
+                          Defines how supplemental groups of the first container processes are calculated.
+                          Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                          (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                          and the container runtime must implement support for this feature.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        type: string
+                      sysctls:
+                        description: |-
+                          Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
+                          sysctls (by the container runtime) might fail to launch.
+                          Note that this field cannot be set when spec.os.name is windows.
+                        items:
+                          description: Sysctl defines a kernel parameter to be set
+                          properties:
+                            name:
+                              description: Name of a property to set
+                              type: string
+                            value:
+                              description: Value of a property to set
+                              type: string
+                          required:
+                          - name
+                          - value
+                          type: object
+                        type: array
+                        x-kubernetes-list-type: atomic
+                      windowsOptions:
+                        description: |-
+                          The Windows specific settings applied to all containers.
+                          If unspecified, the options within a container's SecurityContext will be used.
+                          If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
+                          Note that this field cannot be set when spec.os.name is linux.
+                        properties:
+                          gmsaCredentialSpec:
+                            description: |-
+                              GMSACredentialSpec is where the GMSA admission webhook
+                              (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
+                              GMSA credential spec named by the GMSACredentialSpecName field.
+                            type: string
+                          gmsaCredentialSpecName:
+                            description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
+                            type: string
+                          hostProcess:
+                            description: |-
+                              HostProcess determines if a container should be run as a 'Host Process' container.
+                              All of a Pod's containers must have the same effective HostProcess value
+                              (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).
+                              In addition, if HostProcess is true then HostNetwork must also be set to true.
+                            type: boolean
+                          runAsUserName:
+                            description: |-
+                              The UserName in Windows to run the entrypoint of the container process.
+                              Defaults to the user specified in image metadata if unspecified.
+                              May also be set in PodSecurityContext. If set in both SecurityContext and
+                              PodSecurityContext, the value specified in SecurityContext takes precedence.
+                            type: string
+                        type: object
+                    type: object
                   priorityClass:
                     description: priorityClass is the priority class enforced by the active VirtualClusterPolicy.
                     type: string
                   runtimeClassName:
                     description: |-
-                      SecurityContext specifies custom SecurityContext to be added
-                      to the agent and server pods of the cluster in virtual or shared mode.
+                      RuntimeClassName specifies alternative runtime class for the
+                      agent and server pods of the cluster in virtual or shared mode.
                     type: string
                   securityContext:
                     description: |-
-                      RuntimeClassName specifies alternative runtime class for the
-                      agent and server pods of the cluster in virtual or shared mode.
+                      SecurityContext specifies custom container SecurityContext to be added
+                      to the agent and server containers of the cluster in virtual or shared mode.
                     properties:
                       allowPrivilegeEscalation:
                         description: |-

charts/k3k/templates/deployment.yaml

@@ -15,6 +15,10 @@ spec:
       labels:
         {{- include "k3k.selectorLabels" . | nindent 8 }}
     spec:
+      {{- with .Values.controller.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       imagePullSecrets: {{- include "image.pullSecrets" (concat .Values.controller.imagePullSecrets .Values.global.imagePullSecrets) | nindent 8 }}
       containers:
         - image: "{{- include "controller.registry" .}}{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}"
@@ -54,4 +58,8 @@ spec:
           - containerPort: 8080
             name: https
             protocol: TCP
+          {{- with .Values.controller.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
       serviceAccountName: {{ include "k3k.serviceAccountName" . }}

charts/k3k/values.yaml

@@ -52,6 +52,14 @@ controller:
   #     memory: "100Mi"
   resources: {}
 
+  # podSecurityContext allows you to set custom securityContext for the controller
+  # deployment Pod spec
+  podSecurityContext: {}
+
+  # securityContext allows you to set custom security context for the
+  # controller's Container
+  securityContext: {}
+
 # configuration related to k3s server component in k3k
 server:
   imagePullSecrets: []

docs/crds/crds.adoc

@@ -66,10 +66,12 @@ This includes both node affinity and pod affinity/anti-affinity rules. + |  |
 | *`agentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy. +
 This includes both node affinity and pod affinity/anti-affinity rules. + |  | 
 | *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | sync is the SyncConfig enforced by the active VirtualClusterPolicy. + |  | 
-| *`runtimeClassName`* __string__ | SecurityContext specifies custom SecurityContext to be added +
-to the agent and server pods of the cluster in virtual or shared mode. + |  | 
-| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | RuntimeClassName specifies alternative runtime class for the +
+| *`runtimeClassName`* __string__ | RuntimeClassName specifies alternative runtime class for the +
 agent and server pods of the cluster in virtual or shared mode. + |  | 
+| *`podSecurityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core[$$PodSecurityContext$$]__ | PodSecurityContext specifies custom SecurityContext to be added +
+to the agent and server pods of the cluster in virtual or shared mode. + |  | 
+| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | SecurityContext specifies custom container SecurityContext to be added +
+to the agent and server containers of the cluster in virtual or shared mode. + |  | 
 | *`hostUsers`* __boolean__ | HostUsers sets the user namespace for server and agent pods. +
 If set to true or not present, the pod will be run in the host user namespace. +
 When set to false, a new userns is created for the pod. +
@@ -227,8 +229,10 @@ are mirrored into the virtual cluster. + |  |
 | *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. + | {  } | 
 | *`secretMounts`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretmount[$$SecretMount$$] array__ | SecretMounts specifies a list of secrets to mount into server and agent pods. +
 Each entry defines a secret and its mount path within the pods. + |  | 
-| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | SecurityContext specifies custom SecurityContext to be added +
-to the agent and server pods of the cluster in virtual or shared mode. +
+| *`podSecurityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core[$$PodSecurityContext$$]__ | PodSecurityContext specifies custom PodSecurityContext to be added +
+to the agent and server pods of the cluster in virtual or shared mode. + |  | 
+| *`securityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core[$$SecurityContext$$]__ | SecurityContext specifies custom container SecurityContext to be added +
+to the agent and server containers of the cluster in virtual or shared mode. +
 This option will override the SecurityContext set by default for virtual mode. + |  | 
 | *`runtimeClassName`* __string__ | RuntimeClassName specifies alternative runtime class for the +
 agent and server pods of the cluster in virtual or shared mode. + |  | 

docs/crds/crds.md

@@ -51,8 +51,9 @@ _Appears in:_
 | `serverAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | serverAffinity is the affinity rules for server pods enforced by the active VirtualClusterPolicy.<br />This includes both node affinity and pod affinity/anti-affinity rules. |  |  |
 | `agentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy.<br />This includes both node affinity and pod affinity/anti-affinity rules. |  |  |
 | `sync` _[SyncConfig](#syncconfig)_ | sync is the SyncConfig enforced by the active VirtualClusterPolicy. |  |  |
-| `runtimeClassName` _string_ | SecurityContext specifies custom SecurityContext to be added<br />to the agent and server pods of the cluster in virtual or shared mode. |  |  |
-| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | RuntimeClassName specifies alternative runtime class for the<br />agent and server pods of the cluster in virtual or shared mode. |  |  |
+| `runtimeClassName` _string_ | RuntimeClassName specifies alternative runtime class for the<br />agent and server pods of the cluster in virtual or shared mode. |  |  |
+| `podSecurityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core)_ | PodSecurityContext specifies custom SecurityContext to be added<br />to the agent and server pods of the cluster in virtual or shared mode. |  |  |
+| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | SecurityContext specifies custom container SecurityContext to be added<br />to the agent and server containers of the cluster in virtual or shared mode. |  |  |
 | `hostUsers` _boolean_ | HostUsers sets the user namespace for server and agent pods.<br />If set to true or not present, the pod will be run in the host user namespace.<br />When set to false, a new userns is created for the pod.<br />This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. |  |  |
 
 
@@ -166,7 +167,8 @@ _Appears in:_
 | `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. |  |  |
 | `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{  \} |  |
 | `secretMounts` _[SecretMount](#secretmount) array_ | SecretMounts specifies a list of secrets to mount into server and agent pods.<br />Each entry defines a secret and its mount path within the pods. |  |  |
-| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | SecurityContext specifies custom SecurityContext to be added<br />to the agent and server pods of the cluster in virtual or shared mode.<br />This option will override the SecurityContext set by default for virtual mode. |  |  |
+| `podSecurityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#podsecuritycontext-v1-core)_ | PodSecurityContext specifies custom PodSecurityContext to be added<br />to the agent and server pods of the cluster in virtual or shared mode. |  |  |
+| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#securitycontext-v1-core)_ | SecurityContext specifies custom container SecurityContext to be added<br />to the agent and server containers of the cluster in virtual or shared mode.<br />This option will override the SecurityContext set by default for virtual mode. |  |  |
 | `runtimeClassName` _string_ | RuntimeClassName specifies alternative runtime class for the<br />agent and server pods of the cluster in virtual or shared mode. |  |  |
 | `hostUsers` _boolean_ | HostUsers sets the user namespace for server and agent pods.<br />If set to true or not present, the pod will be run in the host user namespace.<br />When set to false, a new userns is created for the pod.<br />This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. |  |  |
 

k3k-kubelet/provider/provider.go

@@ -23,7 +23,6 @@ import (
 	"k8s.io/client-go/tools/portforward"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/client-go/transport/spdy"
-	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -493,7 +492,7 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
 	}
 
 	// inject networking information to the pod including the virtual cluster controlplane endpoint
-	configureNetworking(hostPod, virtualPod.Name, virtualPod.Namespace, p.serverIP, p.dnsIP)
+	configureNetworking(hostPod, &virtualPod, p.serverIP, p.dnsIP)
 
 	// set ownerReference to the cluster object
 	if err := controllerutil.SetControllerReference(&cluster, hostPod, p.Host.Client.Scheme()); err != nil {
@@ -501,6 +500,11 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
 		return err
 	}
 
+	// harden coreDNS pod
+	if isCoreDNSPod(virtualPod) {
+		hardenCoreDNS(hostPod)
+	}
+
 	if err := p.Host.Client.Create(ctx, hostPod); err != nil {
 		logger.Error(err, "Error creating pod on host cluster")
 		return err
@@ -836,12 +840,42 @@ func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) {
 	return retPods, nil
 }
 
+const nobodyUID = int64(65534)
+
+func isCoreDNSPod(pod corev1.Pod) bool {
+	return pod.Namespace == metav1.NamespaceSystem && pod.Labels["k8s-app"] == "kube-dns"
+}
+
+// hardenCoreDNS will add security contexts needed for CoreDNS pod to run in
+// restricted PSS mode, this is not possible by modifying the k3s manifests.
+func hardenCoreDNS(pod *corev1.Pod) {
+	for i := range pod.Spec.Containers {
+		securityContext := pod.Spec.Containers[i].SecurityContext
+		if securityContext == nil {
+			securityContext = &corev1.SecurityContext{}
+			pod.Spec.Containers[i].SecurityContext = securityContext
+		}
+
+		securityContext.AllowPrivilegeEscalation = new(false)
+		securityContext.Capabilities = &corev1.Capabilities{
+			Add:  []corev1.Capability{"NET_BIND_SERVICE"},
+			Drop: []corev1.Capability{"ALL"},
+		}
+		securityContext.RunAsNonRoot = new(true)
+		securityContext.RunAsUser = new(nobodyUID)
+		securityContext.RunAsGroup = new(nobodyUID)
+		securityContext.SeccompProfile = &corev1.SeccompProfile{
+			Type: corev1.SeccompProfileTypeRuntimeDefault,
+		}
+	}
+}
+
 // configureNetworking will inject network information to each pod to connect them to the
 // virtual cluster api server, as well as confiugre DNS information to connect them to the
 // synced coredns on the host cluster.
-func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP string) {
+func configureNetworking(hostPod, virtualPod *corev1.Pod, serverIP, dnsIP string) {
 	// inject serverIP to hostalias for the pod
-	pod.Spec.HostAliases = append(pod.Spec.HostAliases, corev1.HostAlias{
+	hostPod.Spec.HostAliases = append(hostPod.Spec.HostAliases, corev1.HostAlias{
 		IP: serverIP,
 		Hostnames: []string{
 			"kubernetes",
@@ -853,21 +887,21 @@ func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP
 	})
 
 	// injecting cluster DNS IP to the pods except for coredns pod
-	if !strings.HasPrefix(podName, "coredns") && pod.Spec.DNSConfig == nil {
-		pod.Spec.DNSPolicy = corev1.DNSNone
-		pod.Spec.DNSConfig = &corev1.PodDNSConfig{
+	if !isCoreDNSPod(*virtualPod) && hostPod.Spec.DNSConfig == nil {
+		hostPod.Spec.DNSPolicy = corev1.DNSNone
+		hostPod.Spec.DNSConfig = &corev1.PodDNSConfig{
 			Nameservers: []string{
 				dnsIP,
 			},
 			Searches: []string{
-				podNamespace + ".svc.cluster.local",
+				virtualPod.Namespace + ".svc.cluster.local",
 				"svc.cluster.local",
 				"cluster.local",
 			},
 			Options: []corev1.PodDNSConfigOption{
 				{
 					Name:  "ndots",
-					Value: ptr.To("5"),
+					Value: new("5"),
 				},
 			},
 		}
@@ -881,18 +915,18 @@ func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP
 	}
 
 	// inject networking information to the pod's environment variables
-	for i := range pod.Spec.Containers {
-		pod.Spec.Containers[i].Env = mergeEnvVars(pod.Spec.Containers[i].Env, updatedEnvVars)
+	for i := range hostPod.Spec.Containers {
+		hostPod.Spec.Containers[i].Env = mergeEnvVars(hostPod.Spec.Containers[i].Env, updatedEnvVars)
 	}
 
 	// handle init containers as well
-	for i := range pod.Spec.InitContainers {
-		pod.Spec.InitContainers[i].Env = mergeEnvVars(pod.Spec.InitContainers[i].Env, updatedEnvVars)
+	for i := range hostPod.Spec.InitContainers {
+		hostPod.Spec.InitContainers[i].Env = mergeEnvVars(hostPod.Spec.InitContainers[i].Env, updatedEnvVars)
 	}
 
 	// handle ephemeral containers as well
-	for i := range pod.Spec.EphemeralContainers {
-		pod.Spec.EphemeralContainers[i].Env = mergeEnvVars(pod.Spec.EphemeralContainers[i].Env, updatedEnvVars)
+	for i := range hostPod.Spec.EphemeralContainers {
+		hostPod.Spec.EphemeralContainers[i].Env = mergeEnvVars(hostPod.Spec.EphemeralContainers[i].Env, updatedEnvVars)
 	}
 }
 

pkg/apis/k3k.io/v1beta1/types.go

@@ -213,8 +213,14 @@ type ClusterSpec struct {
 	// +optional
 	SecretMounts []SecretMount `json:"secretMounts,omitempty"`
 
-	// SecurityContext specifies custom SecurityContext to be added
+	// PodSecurityContext specifies custom PodSecurityContext to be added
 	// to the agent and server pods of the cluster in virtual or shared mode.
+	//
+	// +optional
+	PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
+
+	// SecurityContext specifies custom container SecurityContext to be added
+	// to the agent and server containers of the cluster in virtual or shared mode.
 	// This option will override the SecurityContext set by default for virtual mode.
 	//
 	// +optional
@@ -683,14 +689,20 @@ type AppliedPolicy struct {
 	// +optional
 	Sync *SyncConfig `json:"sync,omitempty"`
 
-	// SecurityContext specifies custom SecurityContext to be added
-	// to the agent and server pods of the cluster in virtual or shared mode.
+	// RuntimeClassName specifies alternative runtime class for the
+	// agent and server pods of the cluster in virtual or shared mode.
 	//
 	// +optional
 	RuntimeClassName *string `json:"runtimeClassName,omitempty"`
 
-	// RuntimeClassName specifies alternative runtime class for the
-	// agent and server pods of the cluster in virtual or shared mode.
+	// PodSecurityContext specifies custom SecurityContext to be added
+	// to the agent and server pods of the cluster in virtual or shared mode.
+	//
+	// +optional
+	PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
+
+	// SecurityContext specifies custom container SecurityContext to be added
+	// to the agent and server containers of the cluster in virtual or shared mode.
 	//
 	// +optional
 	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`

pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go

@@ -60,6 +60,11 @@ func (in *AppliedPolicy) DeepCopyInto(out *AppliedPolicy) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.PodSecurityContext != nil {
+		in, out := &in.PodSecurityContext, &out.PodSecurityContext
+		*out = new(v1.PodSecurityContext)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.SecurityContext != nil {
 		in, out := &in.SecurityContext, &out.SecurityContext
 		*out = new(v1.SecurityContext)
@@ -257,6 +262,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.PodSecurityContext != nil {
+		in, out := &in.PodSecurityContext, &out.PodSecurityContext
+		*out = new(v1.PodSecurityContext)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.SecurityContext != nil {
 		in, out := &in.SecurityContext, &out.SecurityContext
 		*out = new(v1.SecurityContext)

pkg/controller/cluster/agent/shared.go

@@ -251,6 +251,16 @@ func (s *SharedAgent) podSpec(ctx context.Context) corev1.PodSpec {
 		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{Name: imagePullSecret})
 	}
 
+	// pod security context
+	podSecurityContext := s.cluster.Spec.PodSecurityContext
+	if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.PodSecurityContext != nil {
+		log.V(1).Info("Using container pod securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)
+		podSecurityContext = s.cluster.Status.Policy.PodSecurityContext
+	}
+
+	podSpec.SecurityContext = podSecurityContext
+
+	// container security context
 	securityContext := s.cluster.Spec.SecurityContext
 	if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.SecurityContext != nil {
 		log.V(1).Info("Using securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)

pkg/controller/cluster/agent/virtual.go

@@ -299,6 +299,15 @@ func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) corev1.P
 		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{Name: imagePullSecret})
 	}
 
+	// pod security context
+	podSecurityContext := v.cluster.Spec.PodSecurityContext
+	if v.cluster.Status.Policy != nil && v.cluster.Status.Policy.PodSecurityContext != nil {
+		log.V(1).Info("Using container pod securityContext configuration from policy", "policyName", v.cluster.Status.PolicyName, "clusterName", v.cluster.Name)
+		podSecurityContext = v.cluster.Status.Policy.PodSecurityContext
+	}
+
+	podSpec.SecurityContext = podSecurityContext
+
 	securityContext := v.cluster.Spec.SecurityContext
 	if v.cluster.Status.Policy != nil && v.cluster.Status.Policy.SecurityContext != nil {
 		log.V(1).Info("Using securityContext configuration from policy", "policyName", v.cluster.Status.PolicyName, "clusterName", v.cluster.Name)

pkg/controller/cluster/cluster.go

@@ -422,7 +422,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
 
 	serviceIP := service.Spec.ClusterIP
 
-	if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil {
+	if err := c.ensureClusterConfigs(ctx, cluster, s, serviceIP); err != nil {
 		return err
 	}
 
@@ -534,37 +534,43 @@ func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster
 	return err
 }
 
-func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1beta1.Cluster, server *server.Server, serviceIP string) error {
-	// create init node config
+func (c *ClusterReconciler) ensureClusterConfigs(ctx context.Context, cluster *v1beta1.Cluster, server *server.Server, serviceIP string) error {
+	// init node config
 	initServerConfig, err := server.Config(true, serviceIP)
 	if err != nil {
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Client.Scheme()); err != nil {
+	currentInitServerConfig := initServerConfig.DeepCopy()
+	if _, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentInitServerConfig, func() error {
+		if err := controllerutil.SetControllerReference(cluster, currentInitServerConfig, c.Client.Scheme()); err != nil {
+			return err
+		}
+
+		currentInitServerConfig.Data = initServerConfig.Data
+
+		return nil
+	}); err != nil {
 		return err
 	}
 
-	if err := c.Client.Create(ctx, initServerConfig); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			return err
-		}
-	}
-
-	// create servers configuration
+	// servers configuration
 	serverConfig, err := server.Config(false, serviceIP)
 	if err != nil {
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Client.Scheme()); err != nil {
-		return err
-	}
-
-	if err := c.Client.Create(ctx, serverConfig); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
+	currentServerConfig := serverConfig.DeepCopy()
+	if _, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerConfig, func() error {
+		if err := controllerutil.SetControllerReference(cluster, currentServerConfig, c.Client.Scheme()); err != nil {
 			return err
 		}
+
+		currentServerConfig.Data = serverConfig.Data
+
+		return nil
+	}); err != nil {
+		return err
 	}
 
 	return nil

pkg/controller/cluster/server/server.go

@@ -11,7 +11,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -249,18 +248,29 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 			},
 		},
 	}
+
 	// virtual mode runs an embedded kubelet inside the server pod and therefore
 	// requires Privileged. shared and hcp modes are agentless (no kubelet) and
 	// run unprivileged.
 	if s.mode == string(v1beta1.VirtualClusterMode) {
 		podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{
-			Privileged: ptr.To(true),
+			Privileged: new(true),
 		}
 	}
 
+	// pod security context
+	podSecurityContext := s.cluster.Spec.PodSecurityContext
+	if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.PodSecurityContext != nil {
+		log.V(1).Info("Using container pod securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)
+		podSecurityContext = s.cluster.Status.Policy.PodSecurityContext
+	}
+
+	podSpec.SecurityContext = podSecurityContext
+
+	// container security context
 	securityContext := s.cluster.Spec.SecurityContext
 	if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.SecurityContext != nil {
-		log.V(1).Info("Using securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)
+		log.V(1).Info("Using container securityContext configuration from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)
 		securityContext = s.cluster.Status.Policy.SecurityContext
 	}