From c1b7da4c72771cae546a92be418aa6c78385d211 Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Mon, 26 Jan 2026 21:47:40 +0200 Subject: [PATCH] SecretMounts feature and private registries (#570) * Add SecretMounts field Signed-off-by: galal-hussein --- .../k3k/templates/crds/k3k.io_clusters.yaml | 131 ++++- docs/crds/crds.adoc | 54 +- docs/crds/crds.md | 26 +- pkg/apis/k3k.io/v1beta1/types.go | 37 +- .../k3k.io/v1beta1/zz_generated.deepcopy.go | 23 + pkg/controller/cluster/agent/virtual.go | 15 +- pkg/controller/cluster/cluster.go | 1 - pkg/controller/cluster/cluster_test.go | 160 ++++++ pkg/controller/cluster/mounts/mounts.go | 60 ++ pkg/controller/cluster/mounts/mounts_test.go | 523 ++++++++++++++++++ pkg/controller/cluster/server/config.go | 2 +- pkg/controller/cluster/server/server.go | 153 ++--- tests/cluster_addons_test.go | 235 ++++++++ tests/cluster_certs_test.go | 10 +- tests/cluster_persistence_test.go | 15 +- tests/cluster_registry_test.go | 160 ++++++ tests/common_test.go | 19 +- tests/testdata/addons/nginx.yaml | 9 + tests/testdata/registry/certs/ca.crt | 33 ++ tests/testdata/registry/certs/ca.key | 52 ++ tests/testdata/registry/certs/registry.crt | 35 ++ tests/testdata/registry/certs/registry.key | 52 ++ tests/testdata/registry/config.yml | 28 + tests/testdata/registry/registries.yaml | 8 + tests/tests_suite_test.go | 240 ++++++++ 25 files changed, 1956 insertions(+), 125 deletions(-) create mode 100644 pkg/controller/cluster/mounts/mounts.go create mode 100644 pkg/controller/cluster/mounts/mounts_test.go create mode 100644 tests/cluster_addons_test.go create mode 100644 tests/cluster_registry_test.go create mode 100644 tests/testdata/addons/nginx.yaml create mode 100644 tests/testdata/registry/certs/ca.crt create mode 100644 tests/testdata/registry/certs/ca.key create mode 100644 tests/testdata/registry/certs/registry.crt create mode 100644 tests/testdata/registry/certs/registry.key create mode 100644 tests/testdata/registry/config.yml create mode 100644 tests/testdata/registry/registries.yaml diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index e036f7b..51a4c36 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -241,10 +241,9 @@ spec: properties: secretName: description: |- - SecretName specifies the name of an existing secret to use. - The controller expects specific keys inside based on the credential type: - - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - - For ServiceAccountTokenKey: 'tls.key'. + The secret must contain specific keys based on the credential type: + - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + - For the ServiceAccountToken signing key: `tls.key`. type: string required: - secretName @@ -255,10 +254,9 @@ spec: properties: secretName: description: |- - SecretName specifies the name of an existing secret to use. - The controller expects specific keys inside based on the credential type: - - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - - For ServiceAccountTokenKey: 'tls.key'. + The secret must contain specific keys based on the credential type: + - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + - For the ServiceAccountToken signing key: `tls.key`. type: string required: - secretName @@ -269,10 +267,9 @@ spec: properties: secretName: description: |- - SecretName specifies the name of an existing secret to use. - The controller expects specific keys inside based on the credential type: - - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - - For ServiceAccountTokenKey: 'tls.key'. + The secret must contain specific keys based on the credential type: + - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + - For the ServiceAccountToken signing key: `tls.key`. type: string required: - secretName @@ -283,10 +280,9 @@ spec: properties: secretName: description: |- - SecretName specifies the name of an existing secret to use. - The controller expects specific keys inside based on the credential type: - - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - - For ServiceAccountTokenKey: 'tls.key'. + The secret must contain specific keys based on the credential type: + - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + - For the ServiceAccountToken signing key: `tls.key`. type: string required: - secretName @@ -296,10 +292,9 @@ spec: properties: secretName: description: |- - SecretName specifies the name of an existing secret to use. - The controller expects specific keys inside based on the credential type: - - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - - For ServiceAccountTokenKey: 'tls.key'. + The secret must contain specific keys based on the credential type: + - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + - For the ServiceAccountToken signing key: `tls.key`. type: string required: - secretName @@ -310,10 +305,9 @@ spec: properties: secretName: description: |- - SecretName specifies the name of an existing secret to use. - The controller expects specific keys inside based on the credential type: - - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - - For ServiceAccountTokenKey: 'tls.key'. + The secret must contain specific keys based on the credential type: + - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + - For the ServiceAccountToken signing key: `tls.key`. type: string required: - secretName @@ -456,6 +450,95 @@ spec: PriorityClass specifies the priorityClassName for server/agent pods. In "shared" mode, this also applies to workloads. type: string + secretMounts: + description: |- + SecretMounts specifies a list of secrets to mount into server and agent pods. + Each entry defines a secret and its mount path within the pods. + items: + description: |- + SecretMount defines a secret to be mounted into server or agent pods, + allowing for custom configurations, certificates, or other sensitive data. + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + mountPath: + description: |- + MountPath is the path within server and agent pods where the + secret contents will be mounted. + type: string + optional: + description: optional field specify whether the Secret or its + keys must be defined + type: boolean + role: + description: |- + Role is the type of the k3k pod that will be used to mount the secret. + This can be 'server', 'agent', or 'all' (for both). + enum: + - server + - agent + - all + type: string + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + subPath: + description: |- + SubPath is an optional path within the secret to mount instead of the root. + When specified, only the specified key from the secret will be mounted as a file + at MountPath, keeping the parent directory writable. + type: string + type: object + type: array serverArgs: description: |- ServerArgs specifies ordered key-value pairs for K3s server pods. diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index 6cf8024..c91ee96 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -182,6 +182,8 @@ Example: ["--node-name=my-agent-node"] + | | are mirrored into the virtual cluster. + | | | *`customCAs`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-customcas[$$CustomCAs$$]__ | CustomCAs specifies the cert/key pairs for custom CA certificates. + | | | *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. + | { } | +| *`secretMounts`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretmount[$$SecretMount$$] array__ | SecretMounts specifies a list of secrets to mount into server and agent pods. + +Each entry defines a secret and its mount path within the pods. + | | |=== @@ -226,10 +228,9 @@ _Appears In:_ [cols="25a,55a,10a,10a", options="header"] |=== | Field | Description | Default | Validation -| *`secretName`* __string__ | SecretName specifies the name of an existing secret to use. + -The controller expects specific keys inside based on the credential type: + -- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. + -- For ServiceAccountTokenKey: 'tls.key'. + | | +| *`secretName`* __string__ | The secret must contain specific keys based on the credential type: + +- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + +- For the ServiceAccountToken signing key: `tls.key`. + | | |=== @@ -494,6 +495,51 @@ then all resources of the given type will be synced. + | | |=== +[id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretmount"] +=== SecretMount + + + +SecretMount defines a secret to be mounted into server or agent pods, +allowing for custom configurations, certificates, or other sensitive data. + + + +_Appears In:_ + +* xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clusterspec[$$ClusterSpec$$] + +[cols="25a,55a,10a,10a", options="header"] +|=== +| Field | Description | Default | Validation +| *`secretName`* __string__ | secretName is the name of the secret in the pod's namespace to use. + +More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + | | +| *`items`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#keytopath-v1-core[$$KeyToPath$$] array__ | items If unspecified, each key-value pair in the Data field of the referenced + +Secret will be projected into the volume as a file whose name is the + +key and content is the value. If specified, the listed keys will be + +projected into the specified paths, and unlisted keys will not be + +present. If a key is specified which is not present in the Secret, + +the volume setup will error unless it is marked optional. Paths must be + +relative and may not contain the '..' path or start with '..'. + | | +| *`defaultMode`* __integer__ | defaultMode is Optional: mode bits used to set permissions on created files by default. + +Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + +YAML accepts both octal and decimal values, JSON requires decimal values + +for mode bits. Defaults to 0644. + +Directories within the path are not affected by this setting. + +This might be in conflict with other options that affect the file + +mode, like fsGroup, and the result can be other mode bits set. + | | +| *`optional`* __boolean__ | optional field specify whether the Secret or its keys must be defined + | | +| *`mountPath`* __string__ | MountPath is the path within server and agent pods where the + +secret contents will be mounted. + | | +| *`subPath`* __string__ | SubPath is an optional path within the secret to mount instead of the root. + +When specified, only the specified key from the secret will be mounted as a file + +at MountPath, keeping the parent directory writable. + | | +| *`role`* __string__ | Role is the type of the k3k pod that will be used to mount the secret. + +This can be 'server', 'agent', or 'all' (for both). + | | Enum: [server agent all] + + +|=== + + [id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretsyncconfig"] === SecretSyncConfig diff --git a/docs/crds/crds.md b/docs/crds/crds.md index c74c7c2..419d5ab 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -135,6 +135,7 @@ _Appears in:_ | `mirrorHostNodes` _boolean_ | MirrorHostNodes controls whether node objects from the host cluster
are mirrored into the virtual cluster. | | | | `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. | | | | `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | | +| `secretMounts` _[SecretMount](#secretmount) array_ | SecretMounts specifies a list of secrets to mount into server and agent pods.
Each entry defines a secret and its mount path within the pods. | | | @@ -170,7 +171,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `secretName` _string_ | SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'. | | | +| `secretName` _string_ | The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`. | | | #### CredentialSources @@ -377,6 +378,29 @@ _Appears in:_ | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty
then all resources of the given type will be synced. | | | +#### SecretMount + + + +SecretMount defines a secret to be mounted into server or agent pods, +allowing for custom configurations, certificates, or other sensitive data. + + + +_Appears in:_ +- [ClusterSpec](#clusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretName` _string_ | secretName is the name of the secret in the pod's namespace to use.
More info: https://kubernetes.io/docs/concepts/storage/volumes#secret | | | +| `items` _[KeyToPath](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#keytopath-v1-core) array_ | items If unspecified, each key-value pair in the Data field of the referenced
Secret will be projected into the volume as a file whose name is the
key and content is the value. If specified, the listed keys will be
projected into the specified paths, and unlisted keys will not be
present. If a key is specified which is not present in the Secret,
the volume setup will error unless it is marked optional. Paths must be
relative and may not contain the '..' path or start with '..'. | | | +| `defaultMode` _integer_ | defaultMode is Optional: mode bits used to set permissions on created files by default.
Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.
YAML accepts both octal and decimal values, JSON requires decimal values
for mode bits. Defaults to 0644.
Directories within the path are not affected by this setting.
This might be in conflict with other options that affect the file
mode, like fsGroup, and the result can be other mode bits set. | | | +| `optional` _boolean_ | optional field specify whether the Secret or its keys must be defined | | | +| `mountPath` _string_ | MountPath is the path within server and agent pods where the
secret contents will be mounted. | | | +| `subPath` _string_ | SubPath is an optional path within the secret to mount instead of the root.
When specified, only the specified key from the secret will be mounted as a file
at MountPath, keeping the parent directory writable. | | | +| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.
This can be 'server', 'agent', or 'all' (for both). | | Enum: [server agent all]
| + + #### SecretSyncConfig diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 6980bc4..627cc7a 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -185,6 +185,36 @@ type ClusterSpec struct { // +kubebuilder:default={} // +optional Sync *SyncConfig `json:"sync,omitempty"` + + // SecretMounts specifies a list of secrets to mount into server and agent pods. + // Each entry defines a secret and its mount path within the pods. + // + // +optional + SecretMounts []SecretMount `json:"secretMounts,omitempty"` +} + +// SecretMount defines a secret to be mounted into server or agent pods, +// allowing for custom configurations, certificates, or other sensitive data. +type SecretMount struct { + // Embeds SecretName, Items, DefaultMode, and Optional + corev1.SecretVolumeSource `json:",inline"` + // MountPath is the path within server and agent pods where the + // secret contents will be mounted. + // + // +optional + MountPath string `json:"mountPath,omitempty"` + // SubPath is an optional path within the secret to mount instead of the root. + // When specified, only the specified key from the secret will be mounted as a file + // at MountPath, keeping the parent directory writable. + // + // +optional + SubPath string `json:"subPath,omitempty"` + // Role is the type of the k3k pod that will be used to mount the secret. + // This can be 'server', 'agent', or 'all' (for both). + // + // +optional + // +kubebuilder:validation:Enum=server;agent;all + Role string `json:"role,omitempty"` } // SyncConfig will contain the resources that should be synced from virtual cluster to host cluster. @@ -470,10 +500,9 @@ type CredentialSources struct { // CredentialSource defines where to get a credential from. // It can represent either a TLS key pair or a single private key. type CredentialSource struct { - // SecretName specifies the name of an existing secret to use. - // The controller expects specific keys inside based on the credential type: - // - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. - // - For ServiceAccountTokenKey: 'tls.key'. + // The secret must contain specific keys based on the credential type: + // - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. + // - For the ServiceAccountToken signing key: `tls.key`. SecretName string `json:"secretName"` } diff --git a/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go index a624e9c..a21d300 100644 --- a/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go @@ -173,6 +173,13 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(SyncConfig) (*in).DeepCopyInto(*out) } + if in.SecretMounts != nil { + in, out := &in.SecretMounts, &out.SecretMounts + *out = make([]SecretMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. @@ -479,6 +486,22 @@ func (in *PriorityClassSyncConfig) DeepCopy() *PriorityClassSyncConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretMount) DeepCopyInto(out *SecretMount) { + *out = *in + in.SecretVolumeSource.DeepCopyInto(&out.SecretVolumeSource) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretMount. +func (in *SecretMount) DeepCopy() *SecretMount { + if in == nil { + return nil + } + out := new(SecretMount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretSyncConfig) DeepCopyInto(out *SecretSyncConfig) { *out = *in diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 1656ddf..15a7304 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -13,6 +13,7 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/mounts" ) const ( @@ -98,6 +99,15 @@ func (v *VirtualAgent) deployment(ctx context.Context) error { "mode": "virtual", }, } + podSpec := v.podSpec(image, name) + + if len(v.cluster.Spec.SecretMounts) > 0 { + vols, volMounts := mounts.BuildSecretsMountsVolumes(v.cluster.Spec.SecretMounts, "agent") + + podSpec.Volumes = append(podSpec.Volumes, vols...) + + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volMounts...) + } deployment := &apps.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -116,7 +126,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error { ObjectMeta: metav1.ObjectMeta{ Labels: selector.MatchLabels, }, - Spec: v.podSpec(image, name, v.cluster.Spec.AgentArgs, &selector), + Spec: podSpec, }, }, } @@ -124,9 +134,10 @@ func (v *VirtualAgent) deployment(ctx context.Context) error { return v.ensureObject(ctx, deployment) } -func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelector *metav1.LabelSelector) v1.PodSpec { +func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec { var limit v1.ResourceList + args := v.cluster.Spec.AgentArgs args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...) podSpec := v1.PodSpec{ diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 6ed1697..7495721 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -43,7 +43,6 @@ import ( ) const ( - namePrefix = "k3k" clusterController = "k3k-cluster-controller" clusterFinalizerName = "cluster.k3k.io/finalizer" ClusterInvalidName = "system" diff --git a/pkg/controller/cluster/cluster_test.go b/pkg/controller/cluster/cluster_test.go index 6099217..5074578 100644 --- a/pkg/controller/cluster/cluster_test.go +++ b/pkg/controller/cluster/cluster_test.go @@ -2,12 +2,14 @@ package cluster_test import ( "context" + "strings" "time" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -295,6 +297,164 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu Expect(err).To(HaveOccurred()) }) }) + When("adding addons to the cluster", func() { + It("will create a statefulset with the correct addon volumes and volume mounts", func() { + // Create the addon secret first + addonSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-addon", + Namespace: namespace, + }, + Data: map[string][]byte{ + "manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: test-cm\n"), + }, + } + Expect(k8sClient.Create(ctx, addonSecret)).To(Succeed()) + + // Create the cluster with an addon referencing the secret + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cluster-", + Namespace: namespace, + }, + Spec: v1beta1.ClusterSpec{ + Addons: []v1beta1.Addon{ + { + SecretRef: "test-addon", + }, + }, + }, + } + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + + // Wait for the statefulset to be created and verify volumes/mounts + var statefulSet appsv1.StatefulSet + statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server") + + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{ + Name: statefulSetName, + Namespace: cluster.Namespace, + }, &statefulSet) + }). + WithTimeout(time.Second * 30). + WithPolling(time.Second). + Should(Succeed()) + + // Verify the addon volume exists + var addonVolume *corev1.Volume + for i := range statefulSet.Spec.Template.Spec.Volumes { + v := &statefulSet.Spec.Template.Spec.Volumes[i] + if v.Name == "addon-test-addon" { + addonVolume = v + break + } + } + Expect(addonVolume).NotTo(BeNil(), "addon volume should exist") + Expect(addonVolume.VolumeSource.Secret).NotTo(BeNil()) + Expect(addonVolume.VolumeSource.Secret.SecretName).To(Equal("test-addon")) + + // Verify the addon volume mount exists in the first container + containers := statefulSet.Spec.Template.Spec.Containers + Expect(containers).NotTo(BeEmpty()) + + var addonMount *corev1.VolumeMount + for i := range containers[0].VolumeMounts { + m := &containers[0].VolumeMounts[i] + if m.Name == "addon-test-addon" { + addonMount = m + break + } + } + Expect(addonMount).NotTo(BeNil(), "addon volume mount should exist") + Expect(addonMount.MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/test-addon")) + Expect(addonMount.ReadOnly).To(BeTrue()) + }) + + It("will create volumes for multiple addons in the correct order", func() { + // Create multiple addon secrets + addonSecret1 := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "addon-one", + Namespace: namespace, + }, + Data: map[string][]byte{ + "manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cm-one\n"), + }, + } + addonSecret2 := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "addon-two", + Namespace: namespace, + }, + Data: map[string][]byte{ + "manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cm-two\n"), + }, + } + Expect(k8sClient.Create(ctx, addonSecret1)).To(Succeed()) + Expect(k8sClient.Create(ctx, addonSecret2)).To(Succeed()) + + // Create the cluster with multiple addons in specific order + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cluster-", + Namespace: namespace, + }, + Spec: v1beta1.ClusterSpec{ + Addons: []v1beta1.Addon{ + {SecretRef: "addon-one"}, + {SecretRef: "addon-two"}, + }, + }, + } + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + + // Wait for the statefulset to be created + var statefulSet appsv1.StatefulSet + statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server") + + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{ + Name: statefulSetName, + Namespace: cluster.Namespace, + }, &statefulSet) + }). + WithTimeout(time.Second * 30). + WithPolling(time.Second). + Should(Succeed()) + + // Verify both addon volumes exist and are in the correct order + volumes := statefulSet.Spec.Template.Spec.Volumes + + // Extract only addon volumes (those starting with "addon-") + var addonVolumes []corev1.Volume + for _, v := range volumes { + if strings.HasPrefix(v.Name, "addon-") { + addonVolumes = append(addonVolumes, v) + } + } + Expect(addonVolumes).To(HaveLen(2)) + Expect(addonVolumes[0].Name).To(Equal("addon-addon-one")) + Expect(addonVolumes[1].Name).To(Equal("addon-addon-two")) + + // Verify both addon volume mounts exist and are in the correct order + containers := statefulSet.Spec.Template.Spec.Containers + Expect(containers).NotTo(BeEmpty()) + + // Extract only addon mounts (those starting with "addon-") + var addonMounts []corev1.VolumeMount + for _, m := range containers[0].VolumeMounts { + if strings.HasPrefix(m.Name, "addon-") { + addonMounts = append(addonMounts, m) + } + } + Expect(addonMounts).To(HaveLen(2)) + Expect(addonMounts[0].Name).To(Equal("addon-addon-one")) + Expect(addonMounts[0].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-one")) + Expect(addonMounts[1].Name).To(Equal("addon-addon-two")) + Expect(addonMounts[1].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-two")) + }) + }) }) }) }) diff --git a/pkg/controller/cluster/mounts/mounts.go b/pkg/controller/cluster/mounts/mounts.go new file mode 100644 index 0000000..0974c14 --- /dev/null +++ b/pkg/controller/cluster/mounts/mounts.go @@ -0,0 +1,60 @@ +package mounts + +import ( + v1 "k8s.io/api/core/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +func BuildSecretsMountsVolumes(secretMounts []v1beta1.SecretMount, role string) ([]v1.Volume, []v1.VolumeMount) { + var ( + vols []v1.Volume + volMounts []v1.VolumeMount + ) + + for _, secretMount := range secretMounts { + if secretMount.SecretName == "" || secretMount.MountPath == "" { + continue + } + + if secretMount.Role == role || secretMount.Role == "" || secretMount.Role == "all" { + vol, volMount := buildSecretMountVolume(secretMount) + + vols = append(vols, vol) + volMounts = append(volMounts, volMount) + } + } + + return vols, volMounts +} + +func buildSecretMountVolume(secretMount v1beta1.SecretMount) (v1.Volume, v1.VolumeMount) { + projectedVolSources := []v1.VolumeProjection{ + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: secretMount.SecretName, + }, + Items: secretMount.Items, + Optional: secretMount.Optional, + }, + }, + } + + vol := v1.Volume{ + Name: secretMount.SecretName, + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: projectedVolSources, + }, + }, + } + + volMount := v1.VolumeMount{ + Name: secretMount.SecretName, + MountPath: secretMount.MountPath, + SubPath: secretMount.SubPath, + } + + return vol, volMount +} diff --git a/pkg/controller/cluster/mounts/mounts_test.go b/pkg/controller/cluster/mounts/mounts_test.go new file mode 100644 index 0000000..7823b8e --- /dev/null +++ b/pkg/controller/cluster/mounts/mounts_test.go @@ -0,0 +1,523 @@ +package mounts + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + v1 "k8s.io/api/core/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +func Test_BuildSecretMountsVolume(t *testing.T) { + type args struct { + secretMounts []v1beta1.SecretMount + role string + } + + type expectedVolumes struct { + vols []v1.Volume + volMounts []v1.VolumeMount + } + + tests := []struct { + name string + args args + expectedData expectedVolumes + }{ + { + name: "empty secret mounts", + args: args{ + secretMounts: []v1beta1.SecretMount{}, + role: "server", + }, + expectedData: expectedVolumes{ + vols: nil, + volMounts: nil, + }, + }, + { + name: "nil secret mounts", + args: args{ + secretMounts: nil, + role: "server", + }, + expectedData: expectedVolumes{ + vols: nil, + volMounts: nil, + }, + }, + { + name: "single secret mount with no role specified defaults to all", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + }, + MountPath: "/mount-dir-1", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-1", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-1", "/mount-dir-1", ""), + }, + }, + }, + { + name: "multiple secrets mounts with no role specified defaults to all", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + }, + MountPath: "/mount-dir-1", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-2", + }, + MountPath: "/mount-dir-2", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-1", nil), + expectedVolume("secret-2", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-1", "/mount-dir-1", ""), + expectedVolumeMount("secret-2", "/mount-dir-2", ""), + }, + }, + }, + { + name: "single secret mount with items", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + Items: []v1.KeyToPath{ + { + Key: "key-1", + Path: "path-1", + }, + }, + }, + MountPath: "/mount-dir-1", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-1", []v1.KeyToPath{{Key: "key-1", Path: "path-1"}}), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-1", "/mount-dir-1", ""), + }, + }, + }, + { + name: "multiple secret mounts with items", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + Items: []v1.KeyToPath{ + { + Key: "key-1", + Path: "path-1", + }, + }, + }, + MountPath: "/mount-dir-1", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-2", + Items: []v1.KeyToPath{ + { + Key: "key-2", + Path: "path-2", + }, + }, + }, + MountPath: "/mount-dir-2", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-1", []v1.KeyToPath{{Key: "key-1", Path: "path-1"}}), + expectedVolume("secret-2", []v1.KeyToPath{{Key: "key-2", Path: "path-2"}}), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-1", "/mount-dir-1", ""), + expectedVolumeMount("secret-2", "/mount-dir-2", ""), + }, + }, + }, + { + name: "user will specify the order", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "z-secret", + }, + MountPath: "/z", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "a-secret", + }, + MountPath: "/a", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "m-secret", + }, + MountPath: "/m", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("z-secret", nil), + expectedVolume("a-secret", nil), + expectedVolume("m-secret", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("z-secret", "/z", ""), + expectedVolumeMount("a-secret", "/a", ""), + expectedVolumeMount("m-secret", "/m", ""), + }, + }, + }, + { + name: "skip entries with empty secret name", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + MountPath: "/mount-dir-1", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-2", + }, + MountPath: "/mount-dir-2", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-2", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-2", "/mount-dir-2", ""), + }, + }, + }, + { + name: "skip entries with empty mount path", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + }, + MountPath: "", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-2", + }, + MountPath: "/mount-dir-2", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-2", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-2", "/mount-dir-2", ""), + }, + }, + }, + { + name: "secret mount with subPath", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + }, + MountPath: "/etc/rancher/k3s/registries.yaml", + SubPath: "registries.yaml", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-1", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-1", "/etc/rancher/k3s/registries.yaml", "registries.yaml"), + }, + }, + }, + // Role-based filtering tests + { + name: "role server includes only server and all roles", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "server-secret", + }, + MountPath: "/server", + Role: "server", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "agent-secret", + }, + MountPath: "/agent", + Role: "agent", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "all-secret", + }, + MountPath: "/all", + Role: "all", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("server-secret", nil), + expectedVolume("all-secret", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("server-secret", "/server", ""), + expectedVolumeMount("all-secret", "/all", ""), + }, + }, + }, + { + name: "role agent includes only agent and all roles", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "server-secret", + }, + MountPath: "/server", + Role: "server", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "agent-secret", + }, + MountPath: "/agent", + Role: "agent", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "all-secret", + }, + MountPath: "/all", + Role: "all", + }, + }, + role: "agent", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("agent-secret", nil), + expectedVolume("all-secret", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("agent-secret", "/agent", ""), + expectedVolumeMount("all-secret", "/all", ""), + }, + }, + }, + { + name: "empty role in secret mount defaults to all", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "no-role-secret", + }, + MountPath: "/no-role", + Role: "", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "server-secret", + }, + MountPath: "/server", + Role: "server", + }, + }, + role: "agent", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("no-role-secret", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("no-role-secret", "/no-role", ""), + }, + }, + }, + { + name: "mixed roles with server filter", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "registry-config", + }, + MountPath: "/etc/rancher/k3s/registries.yaml", + SubPath: "registries.yaml", + Role: "all", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "server-config", + }, + MountPath: "/etc/server", + Role: "server", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "agent-config", + }, + MountPath: "/etc/agent", + Role: "agent", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("registry-config", nil), + expectedVolume("server-config", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("registry-config", "/etc/rancher/k3s/registries.yaml", "registries.yaml"), + expectedVolumeMount("server-config", "/etc/server", ""), + }, + }, + }, + { + name: "all secrets have role all", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-1", + }, + MountPath: "/secret-1", + Role: "all", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "secret-2", + }, + MountPath: "/secret-2", + Role: "all", + }, + }, + role: "server", + }, + expectedData: expectedVolumes{ + vols: []v1.Volume{ + expectedVolume("secret-1", nil), + expectedVolume("secret-2", nil), + }, + volMounts: []v1.VolumeMount{ + expectedVolumeMount("secret-1", "/secret-1", ""), + expectedVolumeMount("secret-2", "/secret-2", ""), + }, + }, + }, + { + name: "no secrets match agent role", + args: args{ + secretMounts: []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "server-only", + }, + MountPath: "/server-only", + Role: "server", + }, + }, + role: "agent", + }, + expectedData: expectedVolumes{ + vols: nil, + volMounts: nil, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vols, volMounts := BuildSecretsMountsVolumes(tt.args.secretMounts, tt.args.role) + + assert.Equal(t, tt.expectedData.vols, vols) + assert.Equal(t, tt.expectedData.volMounts, volMounts) + }) + } +} + +func expectedVolume(name string, items []v1.KeyToPath) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + {Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + Items: items, + }}, + }, + }, + }, + } +} + +func expectedVolumeMount(name, mountPath, subPath string) v1.VolumeMount { + return v1.VolumeMount{ + Name: name, + MountPath: mountPath, + SubPath: subPath, + } +} diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 7603bdc..a1dcd03 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -81,7 +81,7 @@ func serverOptions(cluster *v1beta1.Cluster, token string) string { } if cluster.Spec.Mode != agent.VirtualNodeMode { - opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage" + opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n" } // TODO: Add extra args to the options diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 5e92ebb..3d47707 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -21,13 +21,13 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" "github.com/rancher/k3k/pkg/controller/cluster/agent" + "github.com/rancher/k3k/pkg/controller/cluster/mounts" ) const ( - k3kSystemNamespace = "k3k-system" - serverName = "server" - configName = "server-config" - initConfigName = "init-server-config" + serverName = "server" + configName = "server-config" + initConfigName = "init-server-config" ) // Server @@ -279,67 +279,31 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) volumeMounts []v1.VolumeMount ) - for _, addon := range s.cluster.Spec.Addons { - namespace := k3kSystemNamespace - if addon.SecretNamespace != "" { - namespace = addon.SecretNamespace - } - - nn := types.NamespacedName{ - Name: addon.SecretRef, - Namespace: namespace, - } - - var addons v1.Secret - if err := s.client.Get(ctx, nn, &addons); err != nil { - return nil, err - } - - clusterAddons := v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: addons.Name, - Namespace: s.cluster.Namespace, - }, - Data: make(map[string][]byte, len(addons.Data)), - } - for k, v := range addons.Data { - clusterAddons.Data[k] = v - } - - if err := s.client.Create(ctx, &clusterAddons); err != nil { - return nil, err - } - - name := "varlibrancherk3smanifests" + addon.SecretRef - volume := v1.Volume{ - Name: name, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: addon.SecretRef, - }, - }, - } - volumes = append(volumes, volume) - - volumeMount := v1.VolumeMount{ - Name: name, - MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef, - // changes to this part of the filesystem shouldn't be done manually. The secret should be updated instead. - ReadOnly: true, - } - volumeMounts = append(volumeMounts, volumeMount) - } - - if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled { - vols, mounts, err := s.loadCACertBundle(ctx) + if len(s.cluster.Spec.Addons) > 0 { + addonsVols, addonsMounts, err := s.buildAddonsVolumes(ctx) if err != nil { return nil, err } + volumes = append(volumes, addonsVols...) + + volumeMounts = append(volumeMounts, addonsMounts...) + } + + if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled { + vols, mounts, err := s.buildCABundleVolumes(ctx) + if err != nil { + return nil, err + } + + volumes = append(volumes, vols...) + + volumeMounts = append(volumeMounts, mounts...) + } + + if len(s.cluster.Spec.SecretMounts) > 0 { + vols, mounts := mounts.BuildSecretsMountsVolumes(s.cluster.Spec.SecretMounts, "server") + volumes = append(volumes, vols...) volumeMounts = append(volumeMounts, mounts...) @@ -441,7 +405,7 @@ func (s *Server) setupStartCommand() (string, error) { return output.String(), nil } -func (s *Server) loadCACertBundle(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) { +func (s *Server) buildCABundleVolumes(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) { if s.cluster.Spec.CustomCAs == nil { return nil, nil, fmt.Errorf("customCAs not found") } @@ -533,6 +497,71 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou return volume, mounts } +func (s *Server) buildAddonsVolumes(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) { + var ( + volumes []v1.Volume + mounts []v1.VolumeMount + ) + + for _, addon := range s.cluster.Spec.Addons { + namespace := s.cluster.Namespace + if addon.SecretNamespace != "" { + namespace = addon.SecretNamespace + } + + nn := types.NamespacedName{ + Name: addon.SecretRef, + Namespace: namespace, + } + + var addons v1.Secret + if err := s.client.Get(ctx, nn, &addons); err != nil { + return nil, nil, err + } + + // skip creating the addon secret if it already exists and in the same namespace as the cluster + if namespace != s.cluster.Namespace { + clusterAddons := v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: addons.Name, + Namespace: s.cluster.Namespace, + }, + Data: addons.Data, + } + + if _, err := controllerutil.CreateOrUpdate(ctx, s.client, &clusterAddons, func() error { + return controllerutil.SetOwnerReference(s.cluster, &clusterAddons, s.client.Scheme()) + }); err != nil { + return nil, nil, err + } + } + + name := "addon-" + addon.SecretRef + volume := v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: addon.SecretRef, + }, + }, + } + volumes = append(volumes, volume) + + volumeMount := v1.VolumeMount{ + Name: name, + MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef, + ReadOnly: true, + } + mounts = append(mounts, volumeMount) + } + + return volumes, mounts, nil +} + func sortedKeys(keyMap map[string]string) []string { keys := make([]string, 0, len(keyMap)) diff --git a/tests/cluster_addons_test.go b/tests/cluster_addons_test.go new file mode 100644 index 0000000..977efc2 --- /dev/null +++ b/tests/cluster_addons_test.go @@ -0,0 +1,235 @@ +package k3k_test + +import ( + "context" + "os" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + addonsTestsLabel = "addons" + addonsSecretName = "k3s-addons" + secretMountManifestMountPath = "/var/lib/rancher/k3s/server/manifests/nginx.yaml" + addonManifestMountPath = "/var/lib/rancher/k3s/server/manifests/k3s-addons/nginx.yaml" +) + +var _ = When("a cluster with secretMounts configuration is used to load addons", Label("e2e"), Label(addonsTestsLabel), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + ctx := context.Background() + + namespace := NewNamespace() + + // Create the addon secret + err := createAddonSecret(ctx, namespace.Name) + Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + cluster.Spec.SecretMounts = []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: addonsSecretName, + }, + MountPath: secretMountManifestMountPath, + SubPath: "nginx.yaml", + }, + } + + CreateCluster(cluster) + + virtualClient, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: virtualClient, + } + }) + + It("will load the addon manifest in server pod", func() { + ctx := context.Background() + + serverPods := listServerPods(ctx, virtualCluster) + + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] + + addonContent, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, secretMountManifestMountPath) + Expect(err).To(Not(HaveOccurred())) + + addonTestFile, err := os.ReadFile("testdata/addons/nginx.yaml") + Expect(err).To(Not(HaveOccurred())) + Expect(addonContent).To(Equal(addonTestFile)) + }) + + It("will deploy the addon pod in the virtual cluster", func() { + ctx := context.Background() + + Eventually(func(g Gomega) { + nginxPod, err := virtualCluster.Client.CoreV1().Pods("default").Get(ctx, "nginx-addon", metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(nginxPod.Status.Phase).To(Equal(v1.PodRunning)) + }). + WithTimeout(time.Minute * 3). + WithPolling(time.Second * 5). + Should(Succeed()) + }) +}) + +var _ = When("a cluster with addon configuration is used with addons secret in the same namespace", Label("e2e"), Label(addonsTestsLabel), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + ctx := context.Background() + + namespace := NewNamespace() + + // Create the addon secret + err := createAddonSecret(ctx, namespace.Name) + Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + cluster.Spec.Addons = []v1beta1.Addon{ + { + SecretNamespace: namespace.Name, + SecretRef: addonsSecretName, + }, + } + + CreateCluster(cluster) + + virtualClient, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: virtualClient, + } + + serverPods := listServerPods(ctx, virtualCluster) + + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] + + addonContent, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, addonManifestMountPath) + Expect(err).To(Not(HaveOccurred())) + + addonTestFile, err := os.ReadFile("testdata/addons/nginx.yaml") + Expect(err).To(Not(HaveOccurred())) + Expect(addonContent).To(Equal(addonTestFile)) + + Eventually(func(g Gomega) { + nginxPod, err := virtualCluster.Client.CoreV1().Pods("default").Get(ctx, "nginx-addon", metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(nginxPod.Status.Phase).To(Equal(v1.PodRunning)) + }). + WithTimeout(time.Minute * 3). + WithPolling(time.Second * 5). + Should(Succeed()) + }) +}) + +var _ = When("a cluster with addon configuration is used with addons secret in the different namespace", Label("e2e"), Label(addonsTestsLabel), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + ctx := context.Background() + + namespace := NewNamespace() + secretNamespace := NewNamespace() + + // Create the addon secret + err := createAddonSecret(ctx, secretNamespace.Name) + Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name, secretNamespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + cluster.Spec.Addons = []v1beta1.Addon{ + { + SecretNamespace: secretNamespace.Name, + SecretRef: addonsSecretName, + }, + } + + CreateCluster(cluster) + + virtualClient, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: virtualClient, + } + }) + + It("will load the addon manifest in server pod and deploys the pod", func() { + ctx := context.Background() + + serverPods := listServerPods(ctx, virtualCluster) + + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] + + addonContent, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, addonManifestMountPath) + Expect(err).To(Not(HaveOccurred())) + + addonTestFile, err := os.ReadFile("testdata/addons/nginx.yaml") + Expect(err).To(Not(HaveOccurred())) + Expect(addonContent).To(Equal(addonTestFile)) + + Eventually(func(g Gomega) { + nginxPod, err := virtualCluster.Client.CoreV1().Pods("default").Get(ctx, "nginx-addon", metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(nginxPod.Status.Phase).To(Equal(v1.PodRunning)) + }). + WithTimeout(time.Minute * 3). + WithPolling(time.Second * 5). + Should(Succeed()) + }) +}) + +func createAddonSecret(ctx context.Context, namespace string) error { + addonContent, err := os.ReadFile("testdata/addons/nginx.yaml") + if err != nil { + return err + } + + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: addonsSecretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "nginx.yaml": addonContent, + }, + } + + return k8sClient.Create(ctx, secret) +} diff --git a/tests/cluster_certs_test.go b/tests/cluster_certs_test.go index e1cafbb..37932a8 100644 --- a/tests/cluster_certs_test.go +++ b/tests/cluster_certs_test.go @@ -5,8 +5,6 @@ import ( "os" "strings" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" . "github.com/onsi/ginkgo/v2" @@ -98,12 +96,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce It("will load the custom certs in the server pod", func() { ctx := context.Background() - labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" - serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) + serverPods := listServerPods(ctx, virtualCluster) - Expect(len(serverPods.Items)).To(Equal(1)) - serverPod := serverPods.Items[0] + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] // check server-ca.crt serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt" diff --git a/tests/cluster_persistence_test.go b/tests/cluster_persistence_test.go index 9b8b70c..359993d 100644 --- a/tests/cluster_persistence_test.go +++ b/tests/cluster_persistence_test.go @@ -59,12 +59,10 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per _, err := virtualCluster.Client.ServerVersion() Expect(err).To(Not(HaveOccurred())) - labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" - serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) + serverPods := listServerPods(ctx, virtualCluster) - Expect(len(serverPods.Items)).To(Equal(1)) - serverPod := serverPods.Items[0] + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name) @@ -75,10 +73,9 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per // check that the server pods restarted Eventually(func() any { - serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - Expect(len(serverPods.Items)).To(Equal(1)) - return serverPods.Items[0].DeletionTimestamp + serverPods := listServerPods(ctx, virtualCluster) + Expect(len(serverPods)).To(Equal(1)) + return serverPods[0].DeletionTimestamp }). WithTimeout(time.Minute). WithPolling(time.Second * 5). diff --git a/tests/cluster_registry_test.go b/tests/cluster_registry_test.go new file mode 100644 index 0000000..a875151 --- /dev/null +++ b/tests/cluster_registry_test.go @@ -0,0 +1,160 @@ +package k3k_test + +import ( + "context" + "os" + "time" + + "k8s.io/kubernetes/pkg/api/v1/pod" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller/policy" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = When("a cluster with private registry configuration is used", Label("e2e"), Label(registryTestsLabel), func() { + var virtualCluster *VirtualCluster + BeforeEach(func() { + ctx := context.Background() + + vcp := &v1beta1.VirtualClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "policy-", + }, + Spec: v1beta1.VirtualClusterPolicySpec{ + AllowedMode: v1beta1.VirtualClusterMode, + DisableNetworkPolicy: true, + }, + } + Expect(k8sClient.Create(ctx, vcp)).To(Succeed()) + + namespace := NewNamespace() + + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace) + Expect(err).To(Not(HaveOccurred())) + + namespace.Labels = map[string]string{ + policy.PolicyNameLabelKey: vcp.Name, + } + Expect(k8sClient.Update(ctx, namespace)).To(Succeed()) + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + Expect(k8sClient.Delete(ctx, vcp)).To(Succeed()) + }) + + err = privateRegistry(ctx, namespace.Name) + Expect(err).ToNot(HaveOccurred()) + + cluster := NewCluster(namespace.Name) + + // configure the cluster with the private registry secrets using SecretMounts + // Using subPath allows mounting individual files while keeping parent directories writable + cluster.Spec.SecretMounts = []v1beta1.SecretMount{ + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "k3s-registry-config", + }, + MountPath: "/etc/rancher/k3s/registries.yaml", + SubPath: "registries.yaml", + }, + { + SecretVolumeSource: v1.SecretVolumeSource{ + SecretName: "private-registry-ca-cert", + }, + MountPath: "/etc/rancher/k3s/tls/ca.crt", + SubPath: "tls.crt", + }, + } + + cluster.Spec.Mode = v1beta1.VirtualClusterMode + + // airgap the k3k-server pod + err = buildRegistryNetPolicy(ctx, cluster.Namespace) + Expect(err).ToNot(HaveOccurred()) + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + }) + + It("will be load the registries.yaml and crts in server pod", func() { + ctx := context.Background() + + serverPods := listServerPods(ctx, virtualCluster) + + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] + + // check registries.yaml + registriesConfigPath := "/etc/rancher/k3s/registries.yaml" + registriesConfig, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, registriesConfigPath) + Expect(err).To(Not(HaveOccurred())) + + registriesConfigTestFile, err := os.ReadFile("testdata/registry/registries.yaml") + Expect(err).To(Not(HaveOccurred())) + Expect(registriesConfig).To(Equal(registriesConfigTestFile)) + + // check ca.crt + CACrtPath := "/etc/rancher/k3s/tls/ca.crt" + CACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, CACrtPath) + Expect(err).To(Not(HaveOccurred())) + + CACrtTestFile, err := os.ReadFile("testdata/registry/certs/ca.crt") + Expect(err).To(Not(HaveOccurred())) + Expect(CACrt).To(Equal(CACrtTestFile)) + }) + It("will only pull images from mirrored docker.io registry", func() { + ctx := context.Background() + + // make sure that any pod using docker.io mirror works + virtualCluster.NewNginxPod("") + + // creating a pod with image that uses any registry other than docker.io should fail + // for example public.ecr.aws/docker/library/alpine:latest + alpinePod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "alpine-", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "alpine", + Image: "public.ecr.aws/docker/library/alpine:latest", + }}, + }, + } + + By("Creating Alpine Pod and making sure its failing to start") + var err error + alpinePod, err = virtualCluster.Client.CoreV1().Pods(alpinePod.Namespace).Create(ctx, alpinePod, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + // check that the alpine Pod is failing to pull the image + Eventually(func(g Gomega) { + alpinePod, err = virtualCluster.Client.CoreV1().Pods(alpinePod.Namespace).Get(ctx, alpinePod.Name, metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + + status, _ := pod.GetContainerStatus(alpinePod.Status.ContainerStatuses, "alpine") + state := status.State.Waiting + g.Expect(state).NotTo(BeNil()) + + g.Expect(state.Reason).To(BeEquivalentTo("ImagePullBackOff")) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) +}) diff --git a/tests/common_test.go b/tests/common_test.go index 12ba88d..ea6307c 100644 --- a/tests/common_test.go +++ b/tests/common_test.go @@ -397,26 +397,25 @@ func (c *VirtualCluster) ExecCmd(pod *v1.Pod, command string) (string, string, e func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) { GinkgoHelper() - labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" - serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) + serverPods := listServerPods(ctx, virtualCluster) - Expect(len(serverPods.Items)).To(Equal(1)) - serverPod := serverPods.Items[0] + Expect(len(serverPods)).To(Equal(1)) + serverPod := serverPods[0] GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name) - err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, metav1.DeleteOptions{}) + err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) By("Deleting server pod") // check that the server pods restarted Eventually(func() any { - serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - Expect(len(serverPods.Items)).To(Equal(1)) - return serverPods.Items[0].DeletionTimestamp + serverPods := listServerPods(ctx, virtualCluster) + + Expect(len(serverPods)).To(Equal(1)) + + return serverPods[0].DeletionTimestamp }).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil()) } diff --git a/tests/testdata/addons/nginx.yaml b/tests/testdata/addons/nginx.yaml new file mode 100644 index 0000000..fa0a583 --- /dev/null +++ b/tests/testdata/addons/nginx.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-addon + namespace: default +spec: + containers: + - name: nginx + image: nginx:latest diff --git a/tests/testdata/registry/certs/ca.crt b/tests/testdata/registry/certs/ca.crt new file mode 100644 index 0000000..a68175a --- /dev/null +++ b/tests/testdata/registry/certs/ca.crt @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFuzCCA6OgAwIBAgIUI2MIXZDFl1X+tVkpbeYv78eGB5swDQYJKoZIhvcNAQEL +BQAwbTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBUxvY2FsMQ4wDAYDVQQHDAVMb2Nh +bDESMBAGA1UECgwJUHJpdmF0ZUNBMQwwCgYDVQQLDANEZXYxHDAaBgNVBAMME1By +aXZhdGUtUmVnaXN0cnktQ0EwHhcNMjUxMTI2MTIxMDIyWhcNMzUxMTI0MTIxMDIy +WjBtMQswCQYDVQQGEwJVUzEOMAwGA1UECAwFTG9jYWwxDjAMBgNVBAcMBUxvY2Fs +MRIwEAYDVQQKDAlQcml2YXRlQ0ExDDAKBgNVBAsMA0RldjEcMBoGA1UEAwwTUHJp +dmF0ZS1SZWdpc3RyeS1DQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +AIwwuetZ0bpDmYkdP1HjQpPVjatiQHyedyulKBfPo9/uVE+HVoPe3Ku4QQgSmhKs +hzYE0g7CoM/Kr1EdCYS0RkJhLp+Mfdqwcn2+QXH0PHqdJU3ocutmxRB0xObXyXBv +2Nf+0yKNY36E096wgcJKIl8eNrONmFbPmNl7PjsIvnilQBUSUbfecxUjDcmbZxXq +dSSntMPN/twPi9FzZGMGhZzHD0+Wf+9V0dfF1L1P7rFtmdEtPbBEH6wZFpKl8qzH +O3aXxznLbRSiVZ8PP4zdEKXGcXU8bWZ0NeGPOfgQhsBZvN2auV+esb1McK+7ZzOK +P5dPuzmlCCa8F7+P8wYgHm3W5dCkwsRqvu+ht4Z2sw/Gj1Uot7hunW2jO1PWoegs +W4kGD2EFHqmYU8j3RzxppY0YvZxlSVMiQDHE11ZKHrvu0p7EGsZnAqntAH73jlEg +kt7ZhAS+AMs+evIGAFL5C6dZc8C99uiXiANausrBlVQUk068kR7W2mVy8omet2mQ +DCkmnLJnrxgq5BwPKKvqMDs6dl+idYirxiX33dhildTrkX07DIOwy0nBTdBz8O5s +PIhIA5maVZt8g8w+YpNLEJSMI7NJ1YlgIJ3P9JBOk47hlNp3gw7WlkUxW5XXS1jU +qUWUDHi8grjL9WxSbENXIY4aGIu5ag/xqgQzKGkWcr1nAgMBAAGjUzBRMB0GA1Ud +DgQWBBQYh4H6KFRZXYi9/pzeZUTgxlWQ3jAfBgNVHSMEGDAWgBQYh4H6KFRZXYi9 +/pzeZUTgxlWQ3jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBt +IGJhMJsvDFXWO5pRMjA633k4zquyYBfSt1lTmH5S91SWW8sAickNVvnpBvmsG5lg +GAA1kdF9b7+yrcWGK3TldQobZVpU46MBoiqWPY8tHqbM/0x+UwHHPTwbNa5JLjxT +5ECsZa5wdeACiLt0dFv4CJ8GIcGK7k8QaoxvolEPcxbEpas7bJF9cHNZH3TEwhIb +kC6Q+4+Y2r0pxDW/2uqFpL2RSzk4kJDH3K2y3ywnkSsM6QTDXlaBKijU5bXvrq7v +ZIAX75w+dSHtj4oCWFm/jgVyS+KrAhWbjwxwRMLku0/603OQCAv7c1oEbLh58fLb +zfOOPGkpDvNP5rTwacIyW0P0/GJpmwFjaJYRJgmmCwM7S0qhhJfuVp2d7oZvkkRT +vRpNqpR813ge6T3pnXpdBA9NofTiIsxJ18CGDaHBvDBR+MAsPjmCskIsf9T3fGY/ +fAzuYd8qgE2jIuyZzBLIwARq+zKSzBwgLbfnYRprbp62Qi0OQRYOqxgfP0grw29S +k++Wtfd1zr8OCX/8CkC1P85ipzUoF3G7W1Cadn4aesfxvHPKQlRlpHnt7RImwJMZ +k5QP/2igzjjGwvlGb97Br25dgjiJmQlY3IjqNpzt8uIrCK65vQSwpioq7coJWlTZ +feU/+JrEL8lsa4jkMMeYnW4IJTRqq4Aqoe+e+vz8bQ== +-----END CERTIFICATE----- diff --git a/tests/testdata/registry/certs/ca.key b/tests/testdata/registry/certs/ca.key new file mode 100644 index 0000000..14841f0 --- /dev/null +++ b/tests/testdata/registry/certs/ca.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCMMLnrWdG6Q5mJ +HT9R40KT1Y2rYkB8nncrpSgXz6Pf7lRPh1aD3tyruEEIEpoSrIc2BNIOwqDPyq9R +HQmEtEZCYS6fjH3asHJ9vkFx9Dx6nSVN6HLrZsUQdMTm18lwb9jX/tMijWN+hNPe +sIHCSiJfHjazjZhWz5jZez47CL54pUAVElG33nMVIw3Jm2cV6nUkp7TDzf7cD4vR +c2RjBoWcxw9Pln/vVdHXxdS9T+6xbZnRLT2wRB+sGRaSpfKsxzt2l8c5y20UolWf +Dz+M3RClxnF1PG1mdDXhjzn4EIbAWbzdmrlfnrG9THCvu2czij+XT7s5pQgmvBe/ +j/MGIB5t1uXQpMLEar7vobeGdrMPxo9VKLe4bp1toztT1qHoLFuJBg9hBR6pmFPI +90c8aaWNGL2cZUlTIkAxxNdWSh677tKexBrGZwKp7QB+945RIJLe2YQEvgDLPnry +BgBS+QunWXPAvfbol4gDWrrKwZVUFJNOvJEe1tplcvKJnrdpkAwpJpyyZ68YKuQc +Dyir6jA7OnZfonWIq8Yl993YYpXU65F9OwyDsMtJwU3Qc/DubDyISAOZmlWbfIPM +PmKTSxCUjCOzSdWJYCCdz/SQTpOO4ZTad4MO1pZFMVuV10tY1KlFlAx4vIK4y/Vs +UmxDVyGOGhiLuWoP8aoEMyhpFnK9ZwIDAQABAoICACjieQZLTp/84QUc83+FQMBu +kn9+CwKNEII5C2VOWCORlSMQfEm/MCogdU7OZgK2MESvyTcmydFv8gs85a6/CJKJ +VxiO15F0zh8f4mRCb3Tu6Zc8CG/gq+4tr9MG8aeJ5vqvRZIZHAAk6slSPrWT+0w0 +Oo3I6LnAl3otuCttVGdJAlRi4FQ4WuW6MGYwnTLGCt3izxQfuokhO4ydE5TRrRvY +7f0vDiaVp7o+5tlDO4ChTy+y+v+yDm6ZbnzcStbaz9u5Tg/r5OcUpNXbk5QYUKeY +JTSkp98uWxxqMeTHpRTp1uvmGNPrKzji1yZZCDL+yabuSNL571OknWRvrdeGfHjr +lK68EnznaXO5aI7/85fVl+eeU+l3lb3iVkKTmGoS9/wbJwaCy92++P9H0hfO96Zj +2ssIRNds1+3/fLrOR9ctkIHQBze/lp3OfPqglVcbx9CH9jfKp0hgC4d6osSd6cqW +bZAyz5AZb/3tiCbEhsRXgXfJ4YhaNCn7Pug9tR6YUAz6KlsfqrwpJODBYnISIO/E +kKze14lwMtYB5F1RV9plb1GeI3uVA0IY9RcENyLF4bDl5GzeiBMPhEQ3ZTCgz2Uy +AtJphdZekqXJivOhS4D8eRFBVSXrOoC3V40QYBM4DYmvlPOxRG2vxpwOrrtlOUY7 +ix6tJXUb16YQWCNkMKPZAoIBAQDEsqHsGN1bJDzjci/sRKbrXvdBVvR80qt1yrfF +5CkPkbTZTnTZ8OSPGhy4icoJ0IxoXjhQidMYI/JyDES49sX373ITD9eTLD2KJGL9 +kLhoUbeKsC9oFq0hmsprmmjWZ8N6rn9ONE7HEt5PM9HS7sCS8+mzBpaLJoNWgg// +00y1brjj00/g8tbfkTxEfEaeG4Artmeeq7HXm51Na1sft6XHUYTiEJ5DO+/0S45i +orf2GhHxwawoVj/WFZIpjKOA0CmncKJ3VhiVekkGMUvnP5fJaASy//AsssQ86h58 +ELUiRO/tecwLI3X4FYklsOO+LqNrh4BbOlkI9fdVbwIIEvHDAoIBAQC2dMg/N5JO +DkzbVQRZIiIsZoCr7Z6mHOywZ5wpPS6E0anOxBaJTpPormX7gqpVcg5lzj3M0uYX +wQywO81dUAQkrCIri0cHnS93/5oWTlcBzVg51DmeTuhDsVlVOmFwroStCVeqZwOB +rDdOteoYkRt6lHYR3jA4WtLqsghFrpl6xagqJozB4qGZUQ3PXz1xOPaByQRwAbcd +ghZ8QngKnFe1T50WhX8r3UoFD3AqbkBcApLzwOf0Twf4QBMb+49oGZMmfZ4tnTCu +q1zPANnB0PRZVhsMr+tfc9M5GPCPVHfzQzAXnCdkgI8oTniANNDl6laj4wa64ne9 +H1ZwiI7fR8eNAoIBABQKMw8P1XWUspNlrdY/hFYUndJNXqlc+VUN6z1BKqHIcYl2 +Qdd2gILH4Uc32pq3Yaa8erZR5GzgNLJD57iEg9Tn01J32bnH1xk87czxsqgGM1Hw +81OCg+8Ziyf9WlMFzVexcYzxLVmA5Z9iIy1/X6VZLmUr9aiFqvnkVGb3Cyis+C9V +9xxvAU9Tx7UeiD9Rg/RwKAx1Z7AUzaj2mBkaJ8yv1H8HvGgTMjZMgFwyQdXUACIG +XljZuLVCC1sqVfoouyWxBwxrfCO2irwTx6zuwLMnYtst0jVrnSyrmaGAPkQYi+1A +7HXyDfHRl+B8LifRLpsk+gHRZwLPtHxCzA0wiOsCggEBAJvVJmqH5hdws0fpVuth +8doGOgOd0ZCCx8zq0T+Pl7ms8OE+LRlc2Ysz2Lp1oVGVNqLRAYt83TSQl2u1x/LY +spE3y39xV1szbyWIU2yVwE4zuhS6I/QH5Oxb/raCRFLfW0YG4q8RiLcqBZreWHBf +Dx8kyar9ICYhvF7ja5lIRKHNS5GklzfJfsfZqHfjGjEnu7Kho36emG1FfDro8mnt +miOrObnQjwtB10R3KQ+0Vpe/Qw+ZRQMutNncr/WIZ7U7kqifRYgj5z5n8b6DNXkK +JIhguH2fiuJdpJvxpxRjyockbWDc5/A4tQxx6Q1nDrwv54vWDRt07VvD9inrGEuv +nMkCggEBAK3Qud7mRdXT1GyVT2htguwH0EY+4czXy1mWZvIfZmd9rXzcKrYt/R08 +q6Ym4liEzAyZmVJACGF5f9GyKriBinjXoCsIU9qLSsggCZhW48Ma2HXNx53g0u1U +xvIMcWgR/aMfjV2uu0keKCucP+ZCheHYq0Tn67KnhdtyhZiv9rPMVJ2IamDKC8ov +Tr/7ibSsW8xx84e/4rl310w/9CmuDtAP76aOWQ3zg5P0fq68mAKNPOhwlmHGNg7P +udczhHjEsAOiLth5A50ewf4SbnEjSTR8NYMKeIqxcgc/a74E20B84mB8+255LdsZ +OyNDZebtWToMbZu8LqzYns+6z16TmKA= +-----END PRIVATE KEY----- diff --git a/tests/testdata/registry/certs/registry.crt b/tests/testdata/registry/certs/registry.crt new file mode 100644 index 0000000..245102e --- /dev/null +++ b/tests/testdata/registry/certs/registry.crt @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGFDCCA/ygAwIBAgIUOuZ9BopS9seAhiWlSMwUSuT5KWswDQYJKoZIhvcNAQEL +BQAwbTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBUxvY2FsMQ4wDAYDVQQHDAVMb2Nh +bDESMBAGA1UECgwJUHJpdmF0ZUNBMQwwCgYDVQQLDANEZXYxHDAaBgNVBAMME1By +aXZhdGUtUmVnaXN0cnktQ0EwHhcNMjUxMTI3MTE0ODI2WhcNMzUxMTI1MTE0ODI2 +WjBwMQswCQYDVQQGEwJVUzEOMAwGA1UECAwFTG9jYWwxDjAMBgNVBAcMBUxvY2Fs +MRgwFgYDVQQKDA9Qcml2YXRlUmVnaXN0cnkxDDAKBgNVBAsMA0RldjEZMBcGA1UE +AwwQcHJpdmF0ZS1yZWdpc3RyeTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC +ggIBAKKJWOvCWPuv3uRjKuRdMFigshFtRz1iFeezrs0GIefHTDX4oSV29QpeQD4Y +eURghpZhVAhE6kaLOCGIiJId5u5l4SGGHdjAt4653Px1a2TjXU2n42GK/b9Q4Mzo +KpcyyK7Rzjh5gKT1moEOgSa1hAXnnzfK6qtThFIamMg9b9xFCSEIU3kZ1us3BN1t +JjjkML27aCnEfXYlZPDVSyvV6dGG2xnI4MDiP1P5FkY10p/ryPMTbqCQpJaamwGX +xmmtieEEkXahO/xAug2cgTMz9TFmZHagvmZH7h/625MAAKMlPJvLoYM1E0q219nS +7TZ85H80ymV7G/otISnsThwX6G2gyS/s5ZbJAuKzNBAjfBl2fXOSIU+YVycbUAr4 +2Wzn9rhGIh4972LYTuISAG3BjymFGsUAOxXaFcZwlGOtc1ORqLI4d5XK5yBBW+a7 ++1bf9RfWq3KCDmhXgMDCaolHSMnVH3pyzkSuNfZB20Ic4cae7+TnHCUMvKBZ+H5o +N/V0Uo4WNC+tHxtTmGfjdKmvWgAzc8CaD+xtDHcJgRo45wQDBd0TaBR8XPSF9iDy +vnYDRv2cKHpfICDfvDUgN4mvHkABBj0ELQopSMq60nc8WvG6ftd2twoEMOuSekOx +PvDWQCuzhG/Rx2eAckdMIZgPAYiSjA/pCDI8SQXx3EVhW/3vAgMBAAGjgagwgaUw +HwYDVR0jBBgwFoAUGIeB+ihUWV2Ivf6c3mVE4MZVkN4wCQYDVR0TBAIwADALBgNV +HQ8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwNgYDVR0RBC8wLYIQcHJpdmF0 +ZS1yZWdpc3RyeYIIcmVnaXN0cnmCCWxvY2FsaG9zdIcEfwAAATAdBgNVHQ4EFgQU +NL/k9cTjQNh6X5LfvYKTyxhccFAwDQYJKoZIhvcNAQELBQADggIBAGD8ZKPjCwhT +2UtNOVf4AXp9GRwEet1r1Q+jO/5suOMm3JsL+ohB00pZaWXdHfMpHvP/SMSXCdl5 +fZTqXenTGTrxlItllLinJKjYtTYLH2LUyel8dOIhH+cM8KW/rMjwqnP/urWR+GA9 +5t4wq6QZxz3cCoQbPSCRXBcuzR/b2/ebEILkyHGoixpUHsU9GjVaKHU+uqWwlUa5 +RQYEgM7sCFhcdaxk2YXlBmqajOvUN/TW5Wxv0IkHZlyqg15DLUeZT+toCSrDJRh9 +7zk6S3spTJL+R0P3NVDpvUeMXvo3NSJGlpUXDSqflIxs9V0FSD/Jj+koorW0TaDT +m2G0UrWE8FVuppOI4RkwN/iMBu5m5qM6OJhknZnzJEfgnX9UX+e2g89+JjT9OoVu +8s2kh0bdk+motkV7Vyfjh/jtpdesW+qkIO0RFZ2yKNIA13Tl0zu9okx0TI+Wt8FD +XRzO74dOpWwHjwCOQDaBa8Fy1cxxs+WtEMqjC36SPQzfiW9JaGMFRI9PZei7VNiY +crwOTu8wYjalXXUC8RWtXXUZQjcCp5tNPuCyJBTym2oh+uLu2mimgqYJ4KNEwNKB +21bAipI5hyY+b6AtZ/qHPxv+czL8sPhL4W/tZPCe++WWkQgC8sRa4aVrKnc14cIT +986BhJJ9j0EhPPMBOIbVo0cc75HQSQHE +-----END CERTIFICATE----- diff --git a/tests/testdata/registry/certs/registry.key b/tests/testdata/registry/certs/registry.key new file mode 100644 index 0000000..285145b --- /dev/null +++ b/tests/testdata/registry/certs/registry.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCiiVjrwlj7r97k +YyrkXTBYoLIRbUc9YhXns67NBiHnx0w1+KEldvUKXkA+GHlEYIaWYVQIROpGizgh +iIiSHebuZeEhhh3YwLeOudz8dWtk411Np+Nhiv2/UODM6CqXMsiu0c44eYCk9ZqB +DoEmtYQF5583yuqrU4RSGpjIPW/cRQkhCFN5GdbrNwTdbSY45DC9u2gpxH12JWTw +1Usr1enRhtsZyODA4j9T+RZGNdKf68jzE26gkKSWmpsBl8ZprYnhBJF2oTv8QLoN +nIEzM/UxZmR2oL5mR+4f+tuTAACjJTyby6GDNRNKttfZ0u02fOR/NMplexv6LSEp +7E4cF+htoMkv7OWWyQLiszQQI3wZdn1zkiFPmFcnG1AK+Nls5/a4RiIePe9i2E7i +EgBtwY8phRrFADsV2hXGcJRjrXNTkaiyOHeVyucgQVvmu/tW3/UX1qtygg5oV4DA +wmqJR0jJ1R96cs5ErjX2QdtCHOHGnu/k5xwlDLygWfh+aDf1dFKOFjQvrR8bU5hn +43Spr1oAM3PAmg/sbQx3CYEaOOcEAwXdE2gUfFz0hfYg8r52A0b9nCh6XyAg37w1 +IDeJrx5AAQY9BC0KKUjKutJ3PFrxun7XdrcKBDDrknpDsT7w1kArs4Rv0cdngHJH +TCGYDwGIkowP6QgyPEkF8dxFYVv97wIDAQABAoICAEH/66+wN1ncTHIJIr2gaaVT +e3tAGJGAZsyzVePC/bmUYAn6b9U6vL39D7EnVvbBC2W9F9ZTxZ3nol9bhblvkvpz +PDvUrgH6H49BQc7yDy3kdVq3NcnCGs+5E8+g5sqGwJ7caxTbobVaVebZ8O+6/WU4 +bJrHNwti2nRMgIWvDOEw10gmjV67c14H9V3EmKS5ZGFm3CE5vIhhHt/8fI3MSynd +zNJnk3w/Yt/CYZ0Y9fIiWHL8DQv+MBdHqHG5I8R9x2Mr67V0O1tvHR2x03TrQEFT +BrB1DVuTEcrCnq7ObXPSBw5sXaVdw/uuy2+UCub5R/+vfBBBMVchRDo1znHx81sL +ByxnSm3FSnXYuI+v/k1BzTZeZCtPodo57s7w1bL9pkz6HZgob3ZNV5C1gqWz+dyv +gznB8xkZkhUu4783nBY4zgUZAm+eptjhSGX5gqDZ+lwLBHVkm9Ir1Yu4Jo153IVq +tJkTpPsq6E0kMKw3k75b39g4hu7iO2pzZ5bu8iqEcjnkUG47fHyRzPf1VopteW3Q +u3qMgnyO7G6IRYbcy0xKeumN4qtDh0w0PANOCkvqYpYk4kD1u3a4HYygJ+423VK/ +SFJvNnMJuAsWruElaFF5bkmSqVIkfbDBZTmw/BtHfA7hf5J1o59LmkOJX8XvKouF +yK11dWuXh+wDgAg2AakBAoIBAQDUYOMFn6jUk2UC8RYUCxhVQmM32BM9xySkjsWb +zNMe9Onmc9jJu6eeCORl/IAZIr1Vc+vND0zs52FOVg1zfTX4X5SFpMall+fLcfIl +MDTJag7bcZX9jJnOaDSMK4jn44wrZDZBjf5F/sLcmcFT+qECnLEI6nqLcU2vBeAH +/KtJzxDLmmFsdgH8oSsiU6dIHbHsBPNtRgP3Xgkv7iuFBBJNwQfQWjysvz+jyC48 +Q8oVtJkICLeMn8A2I0a9LaROXLAQi1HV5+PtvbZnC8NzOksgKWrb8uaqQNonOCG2 +t/Ck7jKLeYn7Eu/Omln1FQDAf2jA1ZjffbSyxAJ9OlGSbENPAoIBAQDD67YUuls8 +z+OMzKZ6lgJSWDDVKcg59bEBk6pCL09zQ4C4KE3wL0eq9cf7yctZFZHES+kebDUp +QS9K3kX0vpyIMtteykLDqjBq0dYYbExos/7i+sCIf0dmqPaW9cyJoEZAUYdMFuvx +vofQP4lm0hSRc+ijXjEXzS69ZeD/NzxABCPrRCppCYbKRa3IfitEh2xjIArHniW0 +Wpur5iHRnCHxiNMtKE4wezGPQWJlslTrs2cVt5S5hkyKxfHBsPyrVSbZBFlF9vBI +mLFy4vxkfb3xVjQR6Ezt97aeG71VyC5yvZ1K16TznvzDGwELcd/Fycjaj5vNcp+U +89ZJdXk7xnNhAoIBAFdssseD29n1+uTlHXOOxauDMpiwZ+tMaPcclpf2Dwp1QzvM +gHc6ultBydN5x7mRJWNh3rWBEOeMr++xWMQrzOW7YsZI+ET+bTrAYy+P0or/D7Kh +5V6EXGQtXUQ+P5NFhlPuYq9FpmBl6Q0qdfz99P3ARtgmvd9c+t+LiZeAGXq+tGk7 +2dLuGQ9HwRvWV8xF/RHtT8+xvLw9h4algmC1NluvlGneW4+5ApeHNhE0zqF0wHIg +NH683EDs8Je7jCF94jRNRZjKZnddWxK8Mu7iFj7dDdIRAYcgPy1Z2/b9bSBXtZLY +q0Yhm3nu7A0JYk/bouGOi+mkM5hLO8MVGLMvwd0CggEALAzAKJLp1pdrMwoEWEWI +ChmYCSVWxmlOPeuEeVMHywOfWkh9lYYb1/1g1GS/mqz11Cu5I0TzAu6MAopNMkT1 +Ds5YckyJjFKkhi/dsioPV+84XLJCPa5YUGWm47QqI7tscCOkhuAUdor/IDxY2Uxc +oYNtB+YypYZVfvH8D4XMvxvvM4NlAa7JporaEt0DP2ovXW4j3lPZaF6C57hbXDR9 +kT/RMzL/uXjJYMszo2fgHgp9H+3hu4DNjtoIjCMN/Dut+1c19zwZNElYhFsyoil/ +XlaiaHBRc6OhZJUaEcJrZxLo3Z30kW3qqLdWmcslo+PFjBaD0kJ2TNgyEtwdwOnS +oQKCAQAGRfuTC7qMzQTtlW6NDePBCwRyGSp69N93aDw6e63vVRrKq9GEkZT+6Tpw +lIOKco3oFLRhdtSsHeFdmKqQ9lldkIs0KiBLB1yIdBcxppNCqRDV5xutTcurbmhr +8spVlr8XSgYQED97JaCVB5Df7VDQOIJDeiywq76y1+mNmcUCFcBNNkBFUiq1K5U7 +NNRPmoLBfXcn+fipZWNrH8PVkW7y5jRAFpbxh5sWJ4WPXa4SDnXL4DceAduUoJWV +1HaKcVoKgY0UuE3zaak6s9YA85ZjSGgwexJJCteDMIm6QuGAAqamGryIRRo8Q6ML +jBviIlTtD+bygHVmSs4oZTtJMwn1 +-----END PRIVATE KEY----- diff --git a/tests/testdata/registry/config.yml b/tests/testdata/registry/config.yml new file mode 100644 index 0000000..5d21b7f --- /dev/null +++ b/tests/testdata/registry/config.yml @@ -0,0 +1,28 @@ +version: 0.1 + +log: + fields: + service: registry + +storage: + filesystem: + rootdirectory: /var/lib/registry + delete: + enabled: true + +http: + addr: :5000 + + # TLS configuration + tls: + certificate: /etc/docker/registry/ssl/registry/tls.crt + key: /etc/docker/registry/ssl/registry/tls.key + + # Health endpoint + headers: + X-Content-Type-Options: [nosniff] + +# Using mirror registry to avoid pushing to it in the test +proxy: + remoteurl: https://registry-1.docker.io + ttl: 0 \ No newline at end of file diff --git a/tests/testdata/registry/registries.yaml b/tests/testdata/registry/registries.yaml new file mode 100644 index 0000000..aafdda1 --- /dev/null +++ b/tests/testdata/registry/registries.yaml @@ -0,0 +1,8 @@ +mirrors: + docker.io: + endpoint: + - "https://private-registry:5000" +configs: + "private-registry:5000": + tls: + ca_file: /etc/rancher/k3s/tls/ca.crt \ No newline at end of file diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index 4532ddf..9bfdd3f 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strings" "testing" "time" @@ -22,15 +23,19 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" @@ -49,6 +54,13 @@ const ( networkingTestsLabel = "networking" statusTestsLabel = "status" certificatesTestsLabel = "certificates" + registryTestsLabel = "registry" + + registryImage = "registry:2" + registryCACertSecretName = "private-registry-ca-cert" + registryCertSecretName = "private-registry-cert" + registryConfigSecretName = "private-registry-config" + k3sRegistryConfigSecretName = "k3s-registry-config" ) func TestTests(t *testing.T) { @@ -126,6 +138,10 @@ func buildScheme() *runtime.Scheme { err := v1.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred()) + err = appsv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = networkingv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) err = v1beta1.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred()) @@ -533,3 +549,227 @@ func caCertSecret(name, namespace string, crt, key []byte) *v1.Secret { }, } } + +func privateRegistry(ctx context.Context, namespace string) error { + caCrtMap := map[string]string{ + "tls.crt": filepath.Join("testdata", "registry", "certs", "ca.crt"), + "tls.key": filepath.Join("testdata", "registry", "certs", "ca.key"), + } + + caSecret, err := buildRegistryConfigSecret(caCrtMap, namespace, registryCACertSecretName, true) + if err != nil { + return err + } + + if err := k8sClient.Create(ctx, caSecret); err != nil { + return err + } + + registryCrtMap := map[string]string{ + "tls.crt": filepath.Join("testdata", "registry", "certs", "registry.crt"), + "tls.key": filepath.Join("testdata", "registry", "certs", "registry.key"), + } + + registrySecret, err := buildRegistryConfigSecret(registryCrtMap, namespace, registryCertSecretName, true) + if err != nil { + return err + } + + if err := k8sClient.Create(ctx, registrySecret); err != nil { + return err + } + + configMap := map[string]string{ + "config.yml": filepath.Join("testdata", "registry", "config.yml"), + } + + configSecret, err := buildRegistryConfigSecret(configMap, namespace, registryConfigSecretName, false) + if err != nil { + return err + } + + if err := k8sClient.Create(ctx, configSecret); err != nil { + return err + } + + k3sRegistryConfig := map[string]string{ + "registries.yaml": filepath.Join("testdata", "registry", "registries.yaml"), + } + + k3sRegistrySecret, err := buildRegistryConfigSecret(k3sRegistryConfig, namespace, k3sRegistryConfigSecretName, false) + if err != nil { + return err + } + + if err := k8sClient.Create(ctx, k3sRegistrySecret); err != nil { + return err + } + + registryDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "private-registry", + Namespace: namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "private-registry", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "private-registry", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "private-registry", + Image: registryImage, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config", + MountPath: "/etc/docker/registry/", + }, + { + Name: "ca-cert", + MountPath: "/etc/docker/registry/ssl/ca", + }, + { + Name: "registry-cert", + MountPath: "/etc/docker/registry/ssl/registry", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "config", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "private-registry-config", + }, + }, + }, + { + Name: "ca-cert", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "private-registry-ca-cert", + }, + }, + }, + { + Name: "registry-cert", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "private-registry-cert", + }, + }, + }, + }, + }, + }, + }, + } + + if err := k8sClient.Create(ctx, registryDeployment); err != nil { + return err + } + + registryService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "private-registry", + Namespace: namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "private-registry", + }, + Ports: []v1.ServicePort{ + { + Name: "registry-port", + Port: 5000, + TargetPort: intstr.FromInt(5000), + }, + }, + }, + } + + return k8sClient.Create(ctx, registryService) +} + +func buildRegistryNetPolicy(ctx context.Context, namespace string) error { + np := networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "private-registry-test-netpol", + Namespace: namespace, + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "role": "server", + }, + }, + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeEgress, + }, + Egress: []networkingv1.NetworkPolicyEgressRule{ + { + To: []networkingv1.NetworkPolicyPeer{ + { + IPBlock: &networkingv1.IPBlock{ + CIDR: "10.0.0.0/8", + }, + }, + }, + }, + }, + }, + } + + return k8sClient.Create(ctx, &np) +} + +func buildRegistryConfigSecret(tlsMap map[string]string, namespace, name string, tlsSecret bool) (*v1.Secret, error) { + secretType := v1.SecretTypeOpaque + if tlsSecret { + secretType = v1.SecretTypeTLS + } + + data := make(map[string][]byte) + + for key, path := range tlsMap { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + data[key] = b + } + + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Type: secretType, + Data: data, + } + + return secret, nil +}