SecretMounts feature and private registries (#570)

* Add SecretMounts field

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

pkg/controller/cluster/agent/virtual.go

@@ -13,6 +13,7 @@ import (
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/cluster/mounts"
 )
 
 const (
@@ -98,6 +99,15 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
 			"mode":    "virtual",
 		},
 	}
+	podSpec := v.podSpec(image, name)
+
+	if len(v.cluster.Spec.SecretMounts) > 0 {
+		vols, volMounts := mounts.BuildSecretsMountsVolumes(v.cluster.Spec.SecretMounts, "agent")
+
+		podSpec.Volumes = append(podSpec.Volumes, vols...)
+
+		podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volMounts...)
+	}
 
 	deployment := &apps.Deployment{
 		TypeMeta: metav1.TypeMeta{
@@ -116,7 +126,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: selector.MatchLabels,
 				},
-				Spec: v.podSpec(image, name, v.cluster.Spec.AgentArgs, &selector),
+				Spec: podSpec,
 			},
 		},
 	}
@@ -124,9 +134,10 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
 	return v.ensureObject(ctx, deployment)
 }
 
-func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelector *metav1.LabelSelector) v1.PodSpec {
+func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec {
 	var limit v1.ResourceList
 
+	args := v.cluster.Spec.AgentArgs
 	args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
 
 	podSpec := v1.PodSpec{

pkg/controller/cluster/cluster.go

@@ -43,7 +43,6 @@ import (
 )
 
 const (
-	namePrefix           = "k3k"
 	clusterController    = "k3k-cluster-controller"
 	clusterFinalizerName = "cluster.k3k.io/finalizer"
 	ClusterInvalidName   = "system"

pkg/controller/cluster/cluster_test.go

@@ -2,12 +2,14 @@ package cluster_test
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -295,6 +297,164 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
 					Expect(err).To(HaveOccurred())
 				})
 			})
+			When("adding addons to the cluster", func() {
+				It("will create a statefulset with the correct addon volumes and volume mounts", func() {
+					// Create the addon secret first
+					addonSecret := &corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "test-addon",
+							Namespace: namespace,
+						},
+						Data: map[string][]byte{
+							"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: test-cm\n"),
+						},
+					}
+					Expect(k8sClient.Create(ctx, addonSecret)).To(Succeed())
+
+					// Create the cluster with an addon referencing the secret
+					cluster := &v1beta1.Cluster{
+						ObjectMeta: metav1.ObjectMeta{
+							GenerateName: "cluster-",
+							Namespace:    namespace,
+						},
+						Spec: v1beta1.ClusterSpec{
+							Addons: []v1beta1.Addon{
+								{
+									SecretRef: "test-addon",
+								},
+							},
+						},
+					}
+					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())
+
+					// Wait for the statefulset to be created and verify volumes/mounts
+					var statefulSet appsv1.StatefulSet
+					statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server")
+
+					Eventually(func() error {
+						return k8sClient.Get(ctx, client.ObjectKey{
+							Name:      statefulSetName,
+							Namespace: cluster.Namespace,
+						}, &statefulSet)
+					}).
+						WithTimeout(time.Second * 30).
+						WithPolling(time.Second).
+						Should(Succeed())
+
+					// Verify the addon volume exists
+					var addonVolume *corev1.Volume
+					for i := range statefulSet.Spec.Template.Spec.Volumes {
+						v := &statefulSet.Spec.Template.Spec.Volumes[i]
+						if v.Name == "addon-test-addon" {
+							addonVolume = v
+							break
+						}
+					}
+					Expect(addonVolume).NotTo(BeNil(), "addon volume should exist")
+					Expect(addonVolume.VolumeSource.Secret).NotTo(BeNil())
+					Expect(addonVolume.VolumeSource.Secret.SecretName).To(Equal("test-addon"))
+
+					// Verify the addon volume mount exists in the first container
+					containers := statefulSet.Spec.Template.Spec.Containers
+					Expect(containers).NotTo(BeEmpty())
+
+					var addonMount *corev1.VolumeMount
+					for i := range containers[0].VolumeMounts {
+						m := &containers[0].VolumeMounts[i]
+						if m.Name == "addon-test-addon" {
+							addonMount = m
+							break
+						}
+					}
+					Expect(addonMount).NotTo(BeNil(), "addon volume mount should exist")
+					Expect(addonMount.MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/test-addon"))
+					Expect(addonMount.ReadOnly).To(BeTrue())
+				})
+
+				It("will create volumes for multiple addons in the correct order", func() {
+					// Create multiple addon secrets
+					addonSecret1 := &corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "addon-one",
+							Namespace: namespace,
+						},
+						Data: map[string][]byte{
+							"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cm-one\n"),
+						},
+					}
+					addonSecret2 := &corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "addon-two",
+							Namespace: namespace,
+						},
+						Data: map[string][]byte{
+							"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cm-two\n"),
+						},
+					}
+					Expect(k8sClient.Create(ctx, addonSecret1)).To(Succeed())
+					Expect(k8sClient.Create(ctx, addonSecret2)).To(Succeed())
+
+					// Create the cluster with multiple addons in specific order
+					cluster := &v1beta1.Cluster{
+						ObjectMeta: metav1.ObjectMeta{
+							GenerateName: "cluster-",
+							Namespace:    namespace,
+						},
+						Spec: v1beta1.ClusterSpec{
+							Addons: []v1beta1.Addon{
+								{SecretRef: "addon-one"},
+								{SecretRef: "addon-two"},
+							},
+						},
+					}
+					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())
+
+					// Wait for the statefulset to be created
+					var statefulSet appsv1.StatefulSet
+					statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server")
+
+					Eventually(func() error {
+						return k8sClient.Get(ctx, client.ObjectKey{
+							Name:      statefulSetName,
+							Namespace: cluster.Namespace,
+						}, &statefulSet)
+					}).
+						WithTimeout(time.Second * 30).
+						WithPolling(time.Second).
+						Should(Succeed())
+
+					// Verify both addon volumes exist and are in the correct order
+					volumes := statefulSet.Spec.Template.Spec.Volumes
+
+					// Extract only addon volumes (those starting with "addon-")
+					var addonVolumes []corev1.Volume
+					for _, v := range volumes {
+						if strings.HasPrefix(v.Name, "addon-") {
+							addonVolumes = append(addonVolumes, v)
+						}
+					}
+					Expect(addonVolumes).To(HaveLen(2))
+					Expect(addonVolumes[0].Name).To(Equal("addon-addon-one"))
+					Expect(addonVolumes[1].Name).To(Equal("addon-addon-two"))
+
+					// Verify both addon volume mounts exist and are in the correct order
+					containers := statefulSet.Spec.Template.Spec.Containers
+					Expect(containers).NotTo(BeEmpty())
+
+					// Extract only addon mounts (those starting with "addon-")
+					var addonMounts []corev1.VolumeMount
+					for _, m := range containers[0].VolumeMounts {
+						if strings.HasPrefix(m.Name, "addon-") {
+							addonMounts = append(addonMounts, m)
+						}
+					}
+					Expect(addonMounts).To(HaveLen(2))
+					Expect(addonMounts[0].Name).To(Equal("addon-addon-one"))
+					Expect(addonMounts[0].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-one"))
+					Expect(addonMounts[1].Name).To(Equal("addon-addon-two"))
+					Expect(addonMounts[1].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-two"))
+				})
+			})
 		})
 	})
 })

pkg/controller/cluster/mounts/mounts.go

@@ -0,0 +1,60 @@
+package mounts
+
+import (
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
+)
+
+func BuildSecretsMountsVolumes(secretMounts []v1beta1.SecretMount, role string) ([]v1.Volume, []v1.VolumeMount) {
+	var (
+		vols      []v1.Volume
+		volMounts []v1.VolumeMount
+	)
+
+	for _, secretMount := range secretMounts {
+		if secretMount.SecretName == "" || secretMount.MountPath == "" {
+			continue
+		}
+
+		if secretMount.Role == role || secretMount.Role == "" || secretMount.Role == "all" {
+			vol, volMount := buildSecretMountVolume(secretMount)
+
+			vols = append(vols, vol)
+			volMounts = append(volMounts, volMount)
+		}
+	}
+
+	return vols, volMounts
+}
+
+func buildSecretMountVolume(secretMount v1beta1.SecretMount) (v1.Volume, v1.VolumeMount) {
+	projectedVolSources := []v1.VolumeProjection{
+		{
+			Secret: &v1.SecretProjection{
+				LocalObjectReference: v1.LocalObjectReference{
+					Name: secretMount.SecretName,
+				},
+				Items:    secretMount.Items,
+				Optional: secretMount.Optional,
+			},
+		},
+	}
+
+	vol := v1.Volume{
+		Name: secretMount.SecretName,
+		VolumeSource: v1.VolumeSource{
+			Projected: &v1.ProjectedVolumeSource{
+				Sources: projectedVolSources,
+			},
+		},
+	}
+
+	volMount := v1.VolumeMount{
+		Name:      secretMount.SecretName,
+		MountPath: secretMount.MountPath,
+		SubPath:   secretMount.SubPath,
+	}
+
+	return vol, volMount
+}

pkg/controller/cluster/mounts/mounts_test.go

@@ -0,0 +1,523 @@
+package mounts
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
+)
+
+func Test_BuildSecretMountsVolume(t *testing.T) {
+	type args struct {
+		secretMounts []v1beta1.SecretMount
+		role         string
+	}
+
+	type expectedVolumes struct {
+		vols      []v1.Volume
+		volMounts []v1.VolumeMount
+	}
+
+	tests := []struct {
+		name         string
+		args         args
+		expectedData expectedVolumes
+	}{
+		{
+			name: "empty secret mounts",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{},
+				role:         "server",
+			},
+			expectedData: expectedVolumes{
+				vols:      nil,
+				volMounts: nil,
+			},
+		},
+		{
+			name: "nil secret mounts",
+			args: args{
+				secretMounts: nil,
+				role:         "server",
+			},
+			expectedData: expectedVolumes{
+				vols:      nil,
+				volMounts: nil,
+			},
+		},
+		{
+			name: "single secret mount with no role specified defaults to all",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+						},
+						MountPath: "/mount-dir-1",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-1", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-1", "/mount-dir-1", ""),
+				},
+			},
+		},
+		{
+			name: "multiple secrets mounts with no role specified defaults to all",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+						},
+						MountPath: "/mount-dir-1",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-2",
+						},
+						MountPath: "/mount-dir-2",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-1", nil),
+					expectedVolume("secret-2", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-1", "/mount-dir-1", ""),
+					expectedVolumeMount("secret-2", "/mount-dir-2", ""),
+				},
+			},
+		},
+		{
+			name: "single secret mount with items",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+							Items: []v1.KeyToPath{
+								{
+									Key:  "key-1",
+									Path: "path-1",
+								},
+							},
+						},
+						MountPath: "/mount-dir-1",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-1", []v1.KeyToPath{{Key: "key-1", Path: "path-1"}}),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-1", "/mount-dir-1", ""),
+				},
+			},
+		},
+		{
+			name: "multiple secret mounts with items",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+							Items: []v1.KeyToPath{
+								{
+									Key:  "key-1",
+									Path: "path-1",
+								},
+							},
+						},
+						MountPath: "/mount-dir-1",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-2",
+							Items: []v1.KeyToPath{
+								{
+									Key:  "key-2",
+									Path: "path-2",
+								},
+							},
+						},
+						MountPath: "/mount-dir-2",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-1", []v1.KeyToPath{{Key: "key-1", Path: "path-1"}}),
+					expectedVolume("secret-2", []v1.KeyToPath{{Key: "key-2", Path: "path-2"}}),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-1", "/mount-dir-1", ""),
+					expectedVolumeMount("secret-2", "/mount-dir-2", ""),
+				},
+			},
+		},
+		{
+			name: "user will specify the order",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "z-secret",
+						},
+						MountPath: "/z",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "a-secret",
+						},
+						MountPath: "/a",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "m-secret",
+						},
+						MountPath: "/m",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("z-secret", nil),
+					expectedVolume("a-secret", nil),
+					expectedVolume("m-secret", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("z-secret", "/z", ""),
+					expectedVolumeMount("a-secret", "/a", ""),
+					expectedVolumeMount("m-secret", "/m", ""),
+				},
+			},
+		},
+		{
+			name: "skip entries with empty secret name",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						MountPath: "/mount-dir-1",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-2",
+						},
+						MountPath: "/mount-dir-2",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-2", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-2", "/mount-dir-2", ""),
+				},
+			},
+		},
+		{
+			name: "skip entries with empty mount path",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+						},
+						MountPath: "",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-2",
+						},
+						MountPath: "/mount-dir-2",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-2", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-2", "/mount-dir-2", ""),
+				},
+			},
+		},
+		{
+			name: "secret mount with subPath",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+						},
+						MountPath: "/etc/rancher/k3s/registries.yaml",
+						SubPath:   "registries.yaml",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-1", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-1", "/etc/rancher/k3s/registries.yaml", "registries.yaml"),
+				},
+			},
+		},
+		// Role-based filtering tests
+		{
+			name: "role server includes only server and all roles",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "server-secret",
+						},
+						MountPath: "/server",
+						Role:      "server",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "agent-secret",
+						},
+						MountPath: "/agent",
+						Role:      "agent",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "all-secret",
+						},
+						MountPath: "/all",
+						Role:      "all",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("server-secret", nil),
+					expectedVolume("all-secret", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("server-secret", "/server", ""),
+					expectedVolumeMount("all-secret", "/all", ""),
+				},
+			},
+		},
+		{
+			name: "role agent includes only agent and all roles",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "server-secret",
+						},
+						MountPath: "/server",
+						Role:      "server",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "agent-secret",
+						},
+						MountPath: "/agent",
+						Role:      "agent",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "all-secret",
+						},
+						MountPath: "/all",
+						Role:      "all",
+					},
+				},
+				role: "agent",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("agent-secret", nil),
+					expectedVolume("all-secret", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("agent-secret", "/agent", ""),
+					expectedVolumeMount("all-secret", "/all", ""),
+				},
+			},
+		},
+		{
+			name: "empty role in secret mount defaults to all",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "no-role-secret",
+						},
+						MountPath: "/no-role",
+						Role:      "",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "server-secret",
+						},
+						MountPath: "/server",
+						Role:      "server",
+					},
+				},
+				role: "agent",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("no-role-secret", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("no-role-secret", "/no-role", ""),
+				},
+			},
+		},
+		{
+			name: "mixed roles with server filter",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "registry-config",
+						},
+						MountPath: "/etc/rancher/k3s/registries.yaml",
+						SubPath:   "registries.yaml",
+						Role:      "all",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "server-config",
+						},
+						MountPath: "/etc/server",
+						Role:      "server",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "agent-config",
+						},
+						MountPath: "/etc/agent",
+						Role:      "agent",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("registry-config", nil),
+					expectedVolume("server-config", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("registry-config", "/etc/rancher/k3s/registries.yaml", "registries.yaml"),
+					expectedVolumeMount("server-config", "/etc/server", ""),
+				},
+			},
+		},
+		{
+			name: "all secrets have role all",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-1",
+						},
+						MountPath: "/secret-1",
+						Role:      "all",
+					},
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "secret-2",
+						},
+						MountPath: "/secret-2",
+						Role:      "all",
+					},
+				},
+				role: "server",
+			},
+			expectedData: expectedVolumes{
+				vols: []v1.Volume{
+					expectedVolume("secret-1", nil),
+					expectedVolume("secret-2", nil),
+				},
+				volMounts: []v1.VolumeMount{
+					expectedVolumeMount("secret-1", "/secret-1", ""),
+					expectedVolumeMount("secret-2", "/secret-2", ""),
+				},
+			},
+		},
+		{
+			name: "no secrets match agent role",
+			args: args{
+				secretMounts: []v1beta1.SecretMount{
+					{
+						SecretVolumeSource: v1.SecretVolumeSource{
+							SecretName: "server-only",
+						},
+						MountPath: "/server-only",
+						Role:      "server",
+					},
+				},
+				role: "agent",
+			},
+			expectedData: expectedVolumes{
+				vols:      nil,
+				volMounts: nil,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			vols, volMounts := BuildSecretsMountsVolumes(tt.args.secretMounts, tt.args.role)
+
+			assert.Equal(t, tt.expectedData.vols, vols)
+			assert.Equal(t, tt.expectedData.volMounts, volMounts)
+		})
+	}
+}
+
+func expectedVolume(name string, items []v1.KeyToPath) v1.Volume {
+	return v1.Volume{
+		Name: name,
+		VolumeSource: v1.VolumeSource{
+			Projected: &v1.ProjectedVolumeSource{
+				Sources: []v1.VolumeProjection{
+					{Secret: &v1.SecretProjection{
+						LocalObjectReference: v1.LocalObjectReference{
+							Name: name,
+						},
+						Items: items,
+					}},
+				},
+			},
+		},
+	}
+}
+
+func expectedVolumeMount(name, mountPath, subPath string) v1.VolumeMount {
+	return v1.VolumeMount{
+		Name:      name,
+		MountPath: mountPath,
+		SubPath:   subPath,
+	}
+}

pkg/controller/cluster/server/config.go

@@ -81,7 +81,7 @@ func serverOptions(cluster *v1beta1.Cluster, token string) string {
 	}
 
 	if cluster.Spec.Mode != agent.VirtualNodeMode {
-		opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage"
+		opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n"
 	}
 	// TODO: Add extra args to the options
 

pkg/controller/cluster/server/server.go

@@ -21,13 +21,13 @@ import (
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
+	"github.com/rancher/k3k/pkg/controller/cluster/mounts"
 )
 
 const (
-	k3kSystemNamespace = "k3k-system"
-	serverName         = "server"
-	configName         = "server-config"
-	initConfigName     = "init-server-config"
+	serverName     = "server"
+	configName     = "server-config"
+	initConfigName = "init-server-config"
 )
 
 // Server
@@ -279,67 +279,31 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 		volumeMounts []v1.VolumeMount
 	)
 
-	for _, addon := range s.cluster.Spec.Addons {
-		namespace := k3kSystemNamespace
-		if addon.SecretNamespace != "" {
-			namespace = addon.SecretNamespace
-		}
-
-		nn := types.NamespacedName{
-			Name:      addon.SecretRef,
-			Namespace: namespace,
-		}
-
-		var addons v1.Secret
-		if err := s.client.Get(ctx, nn, &addons); err != nil {
-			return nil, err
-		}
-
-		clusterAddons := v1.Secret{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Secret",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      addons.Name,
-				Namespace: s.cluster.Namespace,
-			},
-			Data: make(map[string][]byte, len(addons.Data)),
-		}
-		for k, v := range addons.Data {
-			clusterAddons.Data[k] = v
-		}
-
-		if err := s.client.Create(ctx, &clusterAddons); err != nil {
-			return nil, err
-		}
-
-		name := "varlibrancherk3smanifests" + addon.SecretRef
-		volume := v1.Volume{
-			Name: name,
-			VolumeSource: v1.VolumeSource{
-				Secret: &v1.SecretVolumeSource{
-					SecretName: addon.SecretRef,
-				},
-			},
-		}
-		volumes = append(volumes, volume)
-
-		volumeMount := v1.VolumeMount{
-			Name:      name,
-			MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
-			// changes to this part of the filesystem shouldn't be done manually. The secret should be updated instead.
-			ReadOnly: true,
-		}
-		volumeMounts = append(volumeMounts, volumeMount)
-	}
-
-	if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled {
-		vols, mounts, err := s.loadCACertBundle(ctx)
+	if len(s.cluster.Spec.Addons) > 0 {
+		addonsVols, addonsMounts, err := s.buildAddonsVolumes(ctx)
 		if err != nil {
 			return nil, err
 		}
 
+		volumes = append(volumes, addonsVols...)
+
+		volumeMounts = append(volumeMounts, addonsMounts...)
+	}
+
+	if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled {
+		vols, mounts, err := s.buildCABundleVolumes(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		volumes = append(volumes, vols...)
+
+		volumeMounts = append(volumeMounts, mounts...)
+	}
+
+	if len(s.cluster.Spec.SecretMounts) > 0 {
+		vols, mounts := mounts.BuildSecretsMountsVolumes(s.cluster.Spec.SecretMounts, "server")
+
 		volumes = append(volumes, vols...)
 
 		volumeMounts = append(volumeMounts, mounts...)
@@ -441,7 +405,7 @@ func (s *Server) setupStartCommand() (string, error) {
 	return output.String(), nil
 }
 
-func (s *Server) loadCACertBundle(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
+func (s *Server) buildCABundleVolumes(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
 	if s.cluster.Spec.CustomCAs == nil {
 		return nil, nil, fmt.Errorf("customCAs not found")
 	}
@@ -533,6 +497,71 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou
 	return volume, mounts
 }
 
+func (s *Server) buildAddonsVolumes(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
+	var (
+		volumes []v1.Volume
+		mounts  []v1.VolumeMount
+	)
+
+	for _, addon := range s.cluster.Spec.Addons {
+		namespace := s.cluster.Namespace
+		if addon.SecretNamespace != "" {
+			namespace = addon.SecretNamespace
+		}
+
+		nn := types.NamespacedName{
+			Name:      addon.SecretRef,
+			Namespace: namespace,
+		}
+
+		var addons v1.Secret
+		if err := s.client.Get(ctx, nn, &addons); err != nil {
+			return nil, nil, err
+		}
+
+		// skip creating the addon secret if it already exists and in the same namespace as the cluster
+		if namespace != s.cluster.Namespace {
+			clusterAddons := v1.Secret{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      addons.Name,
+					Namespace: s.cluster.Namespace,
+				},
+				Data: addons.Data,
+			}
+
+			if _, err := controllerutil.CreateOrUpdate(ctx, s.client, &clusterAddons, func() error {
+				return controllerutil.SetOwnerReference(s.cluster, &clusterAddons, s.client.Scheme())
+			}); err != nil {
+				return nil, nil, err
+			}
+		}
+
+		name := "addon-" + addon.SecretRef
+		volume := v1.Volume{
+			Name: name,
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName: addon.SecretRef,
+				},
+			},
+		}
+		volumes = append(volumes, volume)
+
+		volumeMount := v1.VolumeMount{
+			Name:      name,
+			MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
+			ReadOnly:  true,
+		}
+		mounts = append(mounts, volumeMount)
+	}
+
+	return volumes, mounts, nil
+}
+
 func sortedKeys(keyMap map[string]string) []string {
 	keys := make([]string, 0, len(keyMap))