github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-02-14 10:00:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

SecretMounts feature and private registries (#570)

Browse Source 
* Add SecretMounts field

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
2026-01-26 21:47:40 +02:00
committed by GitHub
parent ff0b03af02
commit c1b7da4c72
25 changed files with 1956 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
charts/k3k/templates/crds/k3k.io_clusters.yaml
View File

@@ -241,10 +241,9 @@ spec:
properties:
secretName:
description: |-
SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'.
The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`.
type: string
required:
- secretName
@@ -255,10 +254,9 @@ spec:
properties:
secretName:
description: |-
SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'.
The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`.
type: string
required:
- secretName
@@ -269,10 +267,9 @@ spec:
properties:
secretName:
description: |-
SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'.
The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`.
type: string
required:
- secretName
@@ -283,10 +280,9 @@ spec:
properties:
secretName:
description: |-
SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'.
The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`.
type: string
required:
- secretName
@@ -296,10 +292,9 @@ spec:
properties:
secretName:
description: |-
SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'.
The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`.
type: string
required:
- secretName
@@ -310,10 +305,9 @@ spec:
properties:
secretName:
description: |-
SecretName specifies the name of an existing secret to use.
The controller expects specific keys inside based on the credential type:
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
- For ServiceAccountTokenKey: 'tls.key'.
The secret must contain specific keys based on the credential type:
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
- For the ServiceAccountToken signing key: `tls.key`.
type: string
required:
- secretName
@@ -456,6 +450,95 @@ spec:
PriorityClass specifies the priorityClassName for server/agent pods.
In "shared" mode, this also applies to workloads.
type: string
secretMounts:
description: |-
SecretMounts specifies a list of secrets to mount into server and agent pods.
Each entry defines a secret and its mount path within the pods.
items:
description: |-
SecretMount defines a secret to be mounted into server or agent pods,
allowing for custom configurations, certificates, or other sensitive data.
properties:
defaultMode:
description: |-
defaultMode is Optional: mode bits used to set permissions on created files by default.
Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.
YAML accepts both octal and decimal values, JSON requires decimal values
for mode bits. Defaults to 0644.
Directories within the path are not affected by this setting.
This might be in conflict with other options that affect the file
mode, like fsGroup, and the result can be other mode bits set.
format: int32
type: integer
items:
description: |-
items If unspecified, each key-value pair in the Data field of the referenced
Secret will be projected into the volume as a file whose name is the
key and content is the value. If specified, the listed keys will be
projected into the specified paths, and unlisted keys will not be
present. If a key is specified which is not present in the Secret,
the volume setup will error unless it is marked optional. Paths must be
relative and may not contain the '..' path or start with '..'.
items:
description: Maps a string key to a path within a volume.
properties:
key:
description: key is the key to project.
type: string
mode:
description: |-
mode is Optional: mode bits used to set permissions on this file.
Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.
YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.
If not specified, the volume defaultMode will be used.
This might be in conflict with other options that affect the file
mode, like fsGroup, and the result can be other mode bits set.
format: int32
type: integer
path:
description: |-
path is the relative path of the file to map the key to.
May not be an absolute path.
May not contain the path element '..'.
May not start with the string '..'.
type: string
required:
- key
- path
type: object
type: array
x-kubernetes-list-type: atomic
mountPath:
description: |-
MountPath is the path within server and agent pods where the
secret contents will be mounted.
type: string
optional:
description: optional field specify whether the Secret or its
keys must be defined
type: boolean
role:
description: |-
Role is the type of the k3k pod that will be used to mount the secret.
This can be 'server', 'agent', or 'all' (for both).
enum:
- server
- agent
- all
type: string
secretName:
description: |-
secretName is the name of the secret in the pod's namespace to use.
More info: https://kubernetes.io/docs/concepts/storage/volumes#secret
type: string
subPath:
description: |-
SubPath is an optional path within the secret to mount instead of the root.
When specified, only the specified key from the secret will be mounted as a file
at MountPath, keeping the parent directory writable.
type: string
type: object
type: array
serverArgs:
description: |-
ServerArgs specifies ordered key-value pairs for K3s server pods.

54
docs/crds/crds.adoc
View File

@@ -182,6 +182,8 @@ Example: ["--node-name=my-agent-node"] + | |
are mirrored into the virtual cluster. + | |
| *`customCAs`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-customcas[$$CustomCAs$$]__ | CustomCAs specifies the cert/key pairs for custom CA certificates. + | |
| *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. + | { } |
| *`secretMounts`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretmount[$$SecretMount$$] array__ | SecretMounts specifies a list of secrets to mount into server and agent pods. +
Each entry defines a secret and its mount path within the pods. + | |
|===
@@ -226,10 +228,9 @@ _Appears In:_
[cols="25a,55a,10a,10a", options="header"]
|===
| Field | Description | Default | Validation
| *`secretName`* __string__ | SecretName specifies the name of an existing secret to use. +
The controller expects specific keys inside based on the credential type: +
- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'. +
- For ServiceAccountTokenKey: 'tls.key'. + | |
| *`secretName`* __string__ | The secret must contain specific keys based on the credential type: +
- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`. +
- For the ServiceAccountToken signing key: `tls.key`. + | |
|===
@@ -494,6 +495,51 @@ then all resources of the given type will be synced. + | |
|===
[id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretmount"]
=== SecretMount
SecretMount defines a secret to be mounted into server or agent pods,
allowing for custom configurations, certificates, or other sensitive data.
_Appears In:_
* xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clusterspec[$$ClusterSpec$$]
[cols="25a,55a,10a,10a", options="header"]
|===
| Field | Description | Default | Validation
| *`secretName`* __string__ | secretName is the name of the secret in the pod's namespace to use. +
More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + | |
| *`items`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#keytopath-v1-core[$$KeyToPath$$] array__ | items If unspecified, each key-value pair in the Data field of the referenced +
Secret will be projected into the volume as a file whose name is the +
key and content is the value. If specified, the listed keys will be +
projected into the specified paths, and unlisted keys will not be +
present. If a key is specified which is not present in the Secret, +
the volume setup will error unless it is marked optional. Paths must be +
relative and may not contain the '..' path or start with '..'. + | |
| *`defaultMode`* __integer__ | defaultMode is Optional: mode bits used to set permissions on created files by default. +
Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. +
YAML accepts both octal and decimal values, JSON requires decimal values +
for mode bits. Defaults to 0644. +
Directories within the path are not affected by this setting. +
This might be in conflict with other options that affect the file +
mode, like fsGroup, and the result can be other mode bits set. + | |
| *`optional`* __boolean__ | optional field specify whether the Secret or its keys must be defined + | |
| *`mountPath`* __string__ | MountPath is the path within server and agent pods where the +
secret contents will be mounted. + | |
| *`subPath`* __string__ | SubPath is an optional path within the secret to mount instead of the root. +
When specified, only the specified key from the secret will be mounted as a file +
at MountPath, keeping the parent directory writable. + | |
| *`role`* __string__ | Role is the type of the k3k pod that will be used to mount the secret. +
This can be 'server', 'agent', or 'all' (for both). + | | Enum: [server agent all] +
|===
[id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-secretsyncconfig"]
=== SecretSyncConfig

26
docs/crds/crds.md
View File

@@ -135,6 +135,7 @@ _Appears in:_
| `mirrorHostNodes` _boolean_ | MirrorHostNodes controls whether node objects from the host cluster<br />are mirrored into the virtual cluster. | | |
| `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. | | |
| `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | |
| `secretMounts` _[SecretMount](#secretmount) array_ | SecretMounts specifies a list of secrets to mount into server and agent pods.<br />Each entry defines a secret and its mount path within the pods. | | |
@@ -170,7 +171,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `secretName` _string_ | SecretName specifies the name of an existing secret to use.<br />The controller expects specific keys inside based on the credential type:<br />- For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.<br />- For ServiceAccountTokenKey: 'tls.key'. | | |
| `secretName` _string_ | The secret must contain specific keys based on the credential type:<br />- For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.<br />- For the ServiceAccountToken signing key: `tls.key`. | | |
#### CredentialSources
@@ -377,6 +378,29 @@ _Appears in:_
| `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. | | |
#### SecretMount
SecretMount defines a secret to be mounted into server or agent pods,
allowing for custom configurations, certificates, or other sensitive data.
_Appears in:_
- [ClusterSpec](#clusterspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `secretName` _string_ | secretName is the name of the secret in the pod's namespace to use.<br />More info: https://kubernetes.io/docs/concepts/storage/volumes#secret | | |
| `items` _[KeyToPath](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#keytopath-v1-core) array_ | items If unspecified, each key-value pair in the Data field of the referenced<br />Secret will be projected into the volume as a file whose name is the<br />key and content is the value. If specified, the listed keys will be<br />projected into the specified paths, and unlisted keys will not be<br />present. If a key is specified which is not present in the Secret,<br />the volume setup will error unless it is marked optional. Paths must be<br />relative and may not contain the '..' path or start with '..'. | | |
| `defaultMode` _integer_ | defaultMode is Optional: mode bits used to set permissions on created files by default.<br />Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.<br />YAML accepts both octal and decimal values, JSON requires decimal values<br />for mode bits. Defaults to 0644.<br />Directories within the path are not affected by this setting.<br />This might be in conflict with other options that affect the file<br />mode, like fsGroup, and the result can be other mode bits set. | | |
| `optional` _boolean_ | optional field specify whether the Secret or its keys must be defined | | |
| `mountPath` _string_ | MountPath is the path within server and agent pods where the<br />secret contents will be mounted. | | |
| `subPath` _string_ | SubPath is an optional path within the secret to mount instead of the root.<br />When specified, only the specified key from the secret will be mounted as a file<br />at MountPath, keeping the parent directory writable. | | |
| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.<br />This can be 'server', 'agent', or 'all' (for both). | | Enum: [server agent all] <br /> |
#### SecretSyncConfig

37
pkg/apis/k3k.io/v1beta1/types.go
View File

@@ -185,6 +185,36 @@ type ClusterSpec struct {
// +kubebuilder:default={}
// +optional
Sync *SyncConfig `json:"sync,omitempty"`
// SecretMounts specifies a list of secrets to mount into server and agent pods.
// Each entry defines a secret and its mount path within the pods.
//
// +optional
SecretMounts []SecretMount `json:"secretMounts,omitempty"`
}
// SecretMount defines a secret to be mounted into server or agent pods,
// allowing for custom configurations, certificates, or other sensitive data.
type SecretMount struct {
// Embeds SecretName, Items, DefaultMode, and Optional
corev1.SecretVolumeSource `json:",inline"`
// MountPath is the path within server and agent pods where the
// secret contents will be mounted.
//
// +optional
MountPath string `json:"mountPath,omitempty"`
// SubPath is an optional path within the secret to mount instead of the root.
// When specified, only the specified key from the secret will be mounted as a file
// at MountPath, keeping the parent directory writable.
//
// +optional
SubPath string `json:"subPath,omitempty"`
// Role is the type of the k3k pod that will be used to mount the secret.
// This can be 'server', 'agent', or 'all' (for both).
//
// +optional
// +kubebuilder:validation:Enum=server;agent;all
Role string `json:"role,omitempty"`
}
// SyncConfig will contain the resources that should be synced from virtual cluster to host cluster.
@@ -470,10 +500,9 @@ type CredentialSources struct {
// CredentialSource defines where to get a credential from.
// It can represent either a TLS key pair or a single private key.
type CredentialSource struct {
// SecretName specifies the name of an existing secret to use.
// The controller expects specific keys inside based on the credential type:
// - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
// - For ServiceAccountTokenKey: 'tls.key'.
// The secret must contain specific keys based on the credential type:
// - For TLS certificate pairs (e.g., ServerCA): `tls.crt` and `tls.key`.
// - For the ServiceAccountToken signing key: `tls.key`.
SecretName string `json:"secretName"`
}

23
pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go
View File

@@ -173,6 +173,13 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
*out = new(SyncConfig)
(*in).DeepCopyInto(*out)
}
if in.SecretMounts != nil {
in, out := &in.SecretMounts, &out.SecretMounts
*out = make([]SecretMount, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec.
@@ -479,6 +486,22 @@ func (in *PriorityClassSyncConfig) DeepCopy() *PriorityClassSyncConfig {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SecretMount) DeepCopyInto(out *SecretMount) {
*out = *in
in.SecretVolumeSource.DeepCopyInto(&out.SecretVolumeSource)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretMount.
func (in *SecretMount) DeepCopy() *SecretMount {
if in == nil {
return nil
}
out := new(SecretMount)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SecretSyncConfig) DeepCopyInto(out *SecretSyncConfig) {
*out = *in

15
pkg/controller/cluster/agent/virtual.go
View File

@@ -13,6 +13,7 @@ import (
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/mounts"
)
const (
@@ -98,6 +99,15 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
"mode": "virtual",
},
}
podSpec := v.podSpec(image, name)
if len(v.cluster.Spec.SecretMounts) > 0 {
vols, volMounts := mounts.BuildSecretsMountsVolumes(v.cluster.Spec.SecretMounts, "agent")
podSpec.Volumes = append(podSpec.Volumes, vols...)
podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volMounts...)
}
deployment := &apps.Deployment{
TypeMeta: metav1.TypeMeta{
@@ -116,7 +126,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
ObjectMeta: metav1.ObjectMeta{
Labels: selector.MatchLabels,
},
Spec: v.podSpec(image, name, v.cluster.Spec.AgentArgs, &selector),
Spec: podSpec,
},
},
}
@@ -124,9 +134,10 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
return v.ensureObject(ctx, deployment)
}
func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelector *metav1.LabelSelector) v1.PodSpec {
func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec {
var limit v1.ResourceList
args := v.cluster.Spec.AgentArgs
args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
podSpec := v1.PodSpec{

1
pkg/controller/cluster/cluster.go
View File

@@ -43,7 +43,6 @@ import (
)
const (
namePrefix = "k3k"
clusterController = "k3k-cluster-controller"
clusterFinalizerName = "cluster.k3k.io/finalizer"
ClusterInvalidName = "system"

160
pkg/controller/cluster/cluster_test.go
View File

@@ -2,12 +2,14 @@ package cluster_test
import (
"context"
"strings"
"time"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -295,6 +297,164 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
Expect(err).To(HaveOccurred())
})
})
When("adding addons to the cluster", func() {
It("will create a statefulset with the correct addon volumes and volume mounts", func() {
// Create the addon secret first
addonSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-addon",
Namespace: namespace,
},
Data: map[string][]byte{
"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: test-cm\n"),
},
}
Expect(k8sClient.Create(ctx, addonSecret)).To(Succeed())
// Create the cluster with an addon referencing the secret
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace,
},
Spec: v1beta1.ClusterSpec{
Addons: []v1beta1.Addon{
{
SecretRef: "test-addon",
},
},
},
}
Expect(k8sClient.Create(ctx, cluster)).To(Succeed())
// Wait for the statefulset to be created and verify volumes/mounts
var statefulSet appsv1.StatefulSet
statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server")
Eventually(func() error {
return k8sClient.Get(ctx, client.ObjectKey{
Name: statefulSetName,
Namespace: cluster.Namespace,
}, &statefulSet)
}).
WithTimeout(time.Second * 30).
WithPolling(time.Second).
Should(Succeed())
// Verify the addon volume exists
var addonVolume *corev1.Volume
for i := range statefulSet.Spec.Template.Spec.Volumes {
v := &statefulSet.Spec.Template.Spec.Volumes[i]
if v.Name == "addon-test-addon" {
addonVolume = v
break
}
}
Expect(addonVolume).NotTo(BeNil(), "addon volume should exist")
Expect(addonVolume.VolumeSource.Secret).NotTo(BeNil())
Expect(addonVolume.VolumeSource.Secret.SecretName).To(Equal("test-addon"))
// Verify the addon volume mount exists in the first container
containers := statefulSet.Spec.Template.Spec.Containers
Expect(containers).NotTo(BeEmpty())
var addonMount *corev1.VolumeMount
for i := range containers[0].VolumeMounts {
m := &containers[0].VolumeMounts[i]
if m.Name == "addon-test-addon" {
addonMount = m
break
}
}
Expect(addonMount).NotTo(BeNil(), "addon volume mount should exist")
Expect(addonMount.MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/test-addon"))
Expect(addonMount.ReadOnly).To(BeTrue())
})
It("will create volumes for multiple addons in the correct order", func() {
// Create multiple addon secrets
addonSecret1 := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "addon-one",
Namespace: namespace,
},
Data: map[string][]byte{
"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cm-one\n"),
},
}
addonSecret2 := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "addon-two",
Namespace: namespace,
},
Data: map[string][]byte{
"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cm-two\n"),
},
}
Expect(k8sClient.Create(ctx, addonSecret1)).To(Succeed())
Expect(k8sClient.Create(ctx, addonSecret2)).To(Succeed())
// Create the cluster with multiple addons in specific order
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace,
},
Spec: v1beta1.ClusterSpec{
Addons: []v1beta1.Addon{
{SecretRef: "addon-one"},
{SecretRef: "addon-two"},
},
},
}
Expect(k8sClient.Create(ctx, cluster)).To(Succeed())
// Wait for the statefulset to be created
var statefulSet appsv1.StatefulSet
statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server")
Eventually(func() error {
return k8sClient.Get(ctx, client.ObjectKey{
Name: statefulSetName,
Namespace: cluster.Namespace,
}, &statefulSet)
}).
WithTimeout(time.Second * 30).
WithPolling(time.Second).
Should(Succeed())
// Verify both addon volumes exist and are in the correct order
volumes := statefulSet.Spec.Template.Spec.Volumes
// Extract only addon volumes (those starting with "addon-")
var addonVolumes []corev1.Volume
for _, v := range volumes {
if strings.HasPrefix(v.Name, "addon-") {
addonVolumes = append(addonVolumes, v)
}
}
Expect(addonVolumes).To(HaveLen(2))
Expect(addonVolumes[0].Name).To(Equal("addon-addon-one"))
Expect(addonVolumes[1].Name).To(Equal("addon-addon-two"))
// Verify both addon volume mounts exist and are in the correct order
containers := statefulSet.Spec.Template.Spec.Containers
Expect(containers).NotTo(BeEmpty())
// Extract only addon mounts (those starting with "addon-")
var addonMounts []corev1.VolumeMount
for _, m := range containers[0].VolumeMounts {
if strings.HasPrefix(m.Name, "addon-") {
addonMounts = append(addonMounts, m)
}
}
Expect(addonMounts).To(HaveLen(2))
Expect(addonMounts[0].Name).To(Equal("addon-addon-one"))
Expect(addonMounts[0].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-one"))
Expect(addonMounts[1].Name).To(Equal("addon-addon-two"))
Expect(addonMounts[1].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-two"))
})
})
})
})
})

60
pkg/controller/cluster/mounts/mounts.go Normal file
View File

@@ -0,0 +1,60 @@
package mounts
import (
v1 "k8s.io/api/core/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
func BuildSecretsMountsVolumes(secretMounts []v1beta1.SecretMount, role string) ([]v1.Volume, []v1.VolumeMount) {
var (
vols []v1.Volume
volMounts []v1.VolumeMount
)
for _, secretMount := range secretMounts {
if secretMount.SecretName == "" || secretMount.MountPath == "" {
continue
}
if secretMount.Role == role || secretMount.Role == "" || secretMount.Role == "all" {
vol, volMount := buildSecretMountVolume(secretMount)
vols = append(vols, vol)
volMounts = append(volMounts, volMount)
}
}
return vols, volMounts
}
func buildSecretMountVolume(secretMount v1beta1.SecretMount) (v1.Volume, v1.VolumeMount) {
projectedVolSources := []v1.VolumeProjection{
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: secretMount.SecretName,
},
Items: secretMount.Items,
Optional: secretMount.Optional,
},
},
}
vol := v1.Volume{
Name: secretMount.SecretName,
VolumeSource: v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: projectedVolSources,
},
},
}
volMount := v1.VolumeMount{
Name: secretMount.SecretName,
MountPath: secretMount.MountPath,
SubPath: secretMount.SubPath,
}
return vol, volMount
}

523
pkg/controller/cluster/mounts/mounts_test.go Normal file
View File

@@ -0,0 +1,523 @@
package mounts
import (
"testing"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
func Test_BuildSecretMountsVolume(t *testing.T) {
type args struct {
secretMounts []v1beta1.SecretMount
role string
}
type expectedVolumes struct {
vols []v1.Volume
volMounts []v1.VolumeMount
}
tests := []struct {
name string
args args
expectedData expectedVolumes
}{
{
name: "empty secret mounts",
args: args{
secretMounts: []v1beta1.SecretMount{},
role: "server",
},
expectedData: expectedVolumes{
vols: nil,
volMounts: nil,
},
},
{
name: "nil secret mounts",
args: args{
secretMounts: nil,
role: "server",
},
expectedData: expectedVolumes{
vols: nil,
volMounts: nil,
},
},
{
name: "single secret mount with no role specified defaults to all",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
},
MountPath: "/mount-dir-1",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-1", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-1", "/mount-dir-1", ""),
},
},
},
{
name: "multiple secrets mounts with no role specified defaults to all",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
},
MountPath: "/mount-dir-1",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-2",
},
MountPath: "/mount-dir-2",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-1", nil),
expectedVolume("secret-2", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-1", "/mount-dir-1", ""),
expectedVolumeMount("secret-2", "/mount-dir-2", ""),
},
},
},
{
name: "single secret mount with items",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
Items: []v1.KeyToPath{
{
Key: "key-1",
Path: "path-1",
},
},
},
MountPath: "/mount-dir-1",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-1", []v1.KeyToPath{{Key: "key-1", Path: "path-1"}}),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-1", "/mount-dir-1", ""),
},
},
},
{
name: "multiple secret mounts with items",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
Items: []v1.KeyToPath{
{
Key: "key-1",
Path: "path-1",
},
},
},
MountPath: "/mount-dir-1",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-2",
Items: []v1.KeyToPath{
{
Key: "key-2",
Path: "path-2",
},
},
},
MountPath: "/mount-dir-2",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-1", []v1.KeyToPath{{Key: "key-1", Path: "path-1"}}),
expectedVolume("secret-2", []v1.KeyToPath{{Key: "key-2", Path: "path-2"}}),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-1", "/mount-dir-1", ""),
expectedVolumeMount("secret-2", "/mount-dir-2", ""),
},
},
},
{
name: "user will specify the order",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "z-secret",
},
MountPath: "/z",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "a-secret",
},
MountPath: "/a",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "m-secret",
},
MountPath: "/m",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("z-secret", nil),
expectedVolume("a-secret", nil),
expectedVolume("m-secret", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("z-secret", "/z", ""),
expectedVolumeMount("a-secret", "/a", ""),
expectedVolumeMount("m-secret", "/m", ""),
},
},
},
{
name: "skip entries with empty secret name",
args: args{
secretMounts: []v1beta1.SecretMount{
{
MountPath: "/mount-dir-1",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-2",
},
MountPath: "/mount-dir-2",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-2", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-2", "/mount-dir-2", ""),
},
},
},
{
name: "skip entries with empty mount path",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
},
MountPath: "",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-2",
},
MountPath: "/mount-dir-2",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-2", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-2", "/mount-dir-2", ""),
},
},
},
{
name: "secret mount with subPath",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
},
MountPath: "/etc/rancher/k3s/registries.yaml",
SubPath: "registries.yaml",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-1", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-1", "/etc/rancher/k3s/registries.yaml", "registries.yaml"),
},
},
},
// Role-based filtering tests
{
name: "role server includes only server and all roles",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "server-secret",
},
MountPath: "/server",
Role: "server",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "agent-secret",
},
MountPath: "/agent",
Role: "agent",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "all-secret",
},
MountPath: "/all",
Role: "all",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("server-secret", nil),
expectedVolume("all-secret", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("server-secret", "/server", ""),
expectedVolumeMount("all-secret", "/all", ""),
},
},
},
{
name: "role agent includes only agent and all roles",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "server-secret",
},
MountPath: "/server",
Role: "server",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "agent-secret",
},
MountPath: "/agent",
Role: "agent",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "all-secret",
},
MountPath: "/all",
Role: "all",
},
},
role: "agent",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("agent-secret", nil),
expectedVolume("all-secret", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("agent-secret", "/agent", ""),
expectedVolumeMount("all-secret", "/all", ""),
},
},
},
{
name: "empty role in secret mount defaults to all",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "no-role-secret",
},
MountPath: "/no-role",
Role: "",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "server-secret",
},
MountPath: "/server",
Role: "server",
},
},
role: "agent",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("no-role-secret", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("no-role-secret", "/no-role", ""),
},
},
},
{
name: "mixed roles with server filter",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "registry-config",
},
MountPath: "/etc/rancher/k3s/registries.yaml",
SubPath: "registries.yaml",
Role: "all",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "server-config",
},
MountPath: "/etc/server",
Role: "server",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "agent-config",
},
MountPath: "/etc/agent",
Role: "agent",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("registry-config", nil),
expectedVolume("server-config", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("registry-config", "/etc/rancher/k3s/registries.yaml", "registries.yaml"),
expectedVolumeMount("server-config", "/etc/server", ""),
},
},
},
{
name: "all secrets have role all",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-1",
},
MountPath: "/secret-1",
Role: "all",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "secret-2",
},
MountPath: "/secret-2",
Role: "all",
},
},
role: "server",
},
expectedData: expectedVolumes{
vols: []v1.Volume{
expectedVolume("secret-1", nil),
expectedVolume("secret-2", nil),
},
volMounts: []v1.VolumeMount{
expectedVolumeMount("secret-1", "/secret-1", ""),
expectedVolumeMount("secret-2", "/secret-2", ""),
},
},
},
{
name: "no secrets match agent role",
args: args{
secretMounts: []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "server-only",
},
MountPath: "/server-only",
Role: "server",
},
},
role: "agent",
},
expectedData: expectedVolumes{
vols: nil,
volMounts: nil,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
vols, volMounts := BuildSecretsMountsVolumes(tt.args.secretMounts, tt.args.role)
assert.Equal(t, tt.expectedData.vols, vols)
assert.Equal(t, tt.expectedData.volMounts, volMounts)
})
}
}
func expectedVolume(name string, items []v1.KeyToPath) v1.Volume {
return v1.Volume{
Name: name,
VolumeSource: v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: []v1.VolumeProjection{
{Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: name,
},
Items: items,
}},
},
},
},
}
}
func expectedVolumeMount(name, mountPath, subPath string) v1.VolumeMount {
return v1.VolumeMount{
Name: name,
MountPath: mountPath,
SubPath: subPath,
}
}

2
pkg/controller/cluster/server/config.go
View File

@@ -81,7 +81,7 @@ func serverOptions(cluster *v1beta1.Cluster, token string) string {
}
if cluster.Spec.Mode != agent.VirtualNodeMode {
opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage"
opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n"
}
// TODO: Add extra args to the options

153
pkg/controller/cluster/server/server.go
View File

@@ -21,13 +21,13 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"github.com/rancher/k3k/pkg/controller/cluster/mounts"
)
const (
k3kSystemNamespace = "k3k-system"
serverName = "server"
configName = "server-config"
initConfigName = "init-server-config"
serverName = "server"
configName = "server-config"
initConfigName = "init-server-config"
)
// Server
@@ -279,67 +279,31 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
volumeMounts []v1.VolumeMount
)
for _, addon := range s.cluster.Spec.Addons {
namespace := k3kSystemNamespace
if addon.SecretNamespace != "" {
namespace = addon.SecretNamespace
}
nn := types.NamespacedName{
Name: addon.SecretRef,
Namespace: namespace,
}
var addons v1.Secret
if err := s.client.Get(ctx, nn, &addons); err != nil {
return nil, err
}
clusterAddons := v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: addons.Name,
Namespace: s.cluster.Namespace,
},
Data: make(map[string][]byte, len(addons.Data)),
}
for k, v := range addons.Data {
clusterAddons.Data[k] = v
}
if err := s.client.Create(ctx, &clusterAddons); err != nil {
return nil, err
}
name := "varlibrancherk3smanifests" + addon.SecretRef
volume := v1.Volume{
Name: name,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: addon.SecretRef,
},
},
}
volumes = append(volumes, volume)
volumeMount := v1.VolumeMount{
Name: name,
MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
// changes to this part of the filesystem shouldn't be done manually. The secret should be updated instead.
ReadOnly: true,
}
volumeMounts = append(volumeMounts, volumeMount)
}
if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled {
vols, mounts, err := s.loadCACertBundle(ctx)
if len(s.cluster.Spec.Addons) > 0 {
addonsVols, addonsMounts, err := s.buildAddonsVolumes(ctx)
if err != nil {
return nil, err
}
volumes = append(volumes, addonsVols...)
volumeMounts = append(volumeMounts, addonsMounts...)
}
if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled {
vols, mounts, err := s.buildCABundleVolumes(ctx)
if err != nil {
return nil, err
}
volumes = append(volumes, vols...)
volumeMounts = append(volumeMounts, mounts...)
}
if len(s.cluster.Spec.SecretMounts) > 0 {
vols, mounts := mounts.BuildSecretsMountsVolumes(s.cluster.Spec.SecretMounts, "server")
volumes = append(volumes, vols...)
volumeMounts = append(volumeMounts, mounts...)
@@ -441,7 +405,7 @@ func (s *Server) setupStartCommand() (string, error) {
return output.String(), nil
}
func (s *Server) loadCACertBundle(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
func (s *Server) buildCABundleVolumes(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
if s.cluster.Spec.CustomCAs == nil {
return nil, nil, fmt.Errorf("customCAs not found")
}
@@ -533,6 +497,71 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou
return volume, mounts
}
func (s *Server) buildAddonsVolumes(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
var (
volumes []v1.Volume
mounts []v1.VolumeMount
)
for _, addon := range s.cluster.Spec.Addons {
namespace := s.cluster.Namespace
if addon.SecretNamespace != "" {
namespace = addon.SecretNamespace
}
nn := types.NamespacedName{
Name: addon.SecretRef,
Namespace: namespace,
}
var addons v1.Secret
if err := s.client.Get(ctx, nn, &addons); err != nil {
return nil, nil, err
}
// skip creating the addon secret if it already exists and in the same namespace as the cluster
if namespace != s.cluster.Namespace {
clusterAddons := v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: addons.Name,
Namespace: s.cluster.Namespace,
},
Data: addons.Data,
}
if _, err := controllerutil.CreateOrUpdate(ctx, s.client, &clusterAddons, func() error {
return controllerutil.SetOwnerReference(s.cluster, &clusterAddons, s.client.Scheme())
}); err != nil {
return nil, nil, err
}
}
name := "addon-" + addon.SecretRef
volume := v1.Volume{
Name: name,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: addon.SecretRef,
},
},
}
volumes = append(volumes, volume)
volumeMount := v1.VolumeMount{
Name: name,
MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
ReadOnly: true,
}
mounts = append(mounts, volumeMount)
}
return volumes, mounts, nil
}
func sortedKeys(keyMap map[string]string) []string {
keys := make([]string, 0, len(keyMap))

235
tests/cluster_addons_test.go Normal file
View File

@@ -0,0 +1,235 @@
package k3k_test
import (
"context"
"os"
"time"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
const (
addonsTestsLabel = "addons"
addonsSecretName = "k3s-addons"
secretMountManifestMountPath = "/var/lib/rancher/k3s/server/manifests/nginx.yaml"
addonManifestMountPath = "/var/lib/rancher/k3s/server/manifests/k3s-addons/nginx.yaml"
)
var _ = When("a cluster with secretMounts configuration is used to load addons", Label("e2e"), Label(addonsTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
// Create the addon secret
err := createAddonSecret(ctx, namespace.Name)
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
})
cluster := NewCluster(namespace.Name)
cluster.Spec.SecretMounts = []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: addonsSecretName,
},
MountPath: secretMountManifestMountPath,
SubPath: "nginx.yaml",
},
}
CreateCluster(cluster)
virtualClient, restConfig := NewVirtualK8sClientAndConfig(cluster)
virtualCluster = &VirtualCluster{
Cluster: cluster,
RestConfig: restConfig,
Client: virtualClient,
}
})
It("will load the addon manifest in server pod", func() {
ctx := context.Background()
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
addonContent, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, secretMountManifestMountPath)
Expect(err).To(Not(HaveOccurred()))
addonTestFile, err := os.ReadFile("testdata/addons/nginx.yaml")
Expect(err).To(Not(HaveOccurred()))
Expect(addonContent).To(Equal(addonTestFile))
})
It("will deploy the addon pod in the virtual cluster", func() {
ctx := context.Background()
Eventually(func(g Gomega) {
nginxPod, err := virtualCluster.Client.CoreV1().Pods("default").Get(ctx, "nginx-addon", metav1.GetOptions{})
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(nginxPod.Status.Phase).To(Equal(v1.PodRunning))
}).
WithTimeout(time.Minute * 3).
WithPolling(time.Second * 5).
Should(Succeed())
})
})
var _ = When("a cluster with addon configuration is used with addons secret in the same namespace", Label("e2e"), Label(addonsTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
// Create the addon secret
err := createAddonSecret(ctx, namespace.Name)
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
})
cluster := NewCluster(namespace.Name)
cluster.Spec.Addons = []v1beta1.Addon{
{
SecretNamespace: namespace.Name,
SecretRef: addonsSecretName,
},
}
CreateCluster(cluster)
virtualClient, restConfig := NewVirtualK8sClientAndConfig(cluster)
virtualCluster = &VirtualCluster{
Cluster: cluster,
RestConfig: restConfig,
Client: virtualClient,
}
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
addonContent, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, addonManifestMountPath)
Expect(err).To(Not(HaveOccurred()))
addonTestFile, err := os.ReadFile("testdata/addons/nginx.yaml")
Expect(err).To(Not(HaveOccurred()))
Expect(addonContent).To(Equal(addonTestFile))
Eventually(func(g Gomega) {
nginxPod, err := virtualCluster.Client.CoreV1().Pods("default").Get(ctx, "nginx-addon", metav1.GetOptions{})
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(nginxPod.Status.Phase).To(Equal(v1.PodRunning))
}).
WithTimeout(time.Minute * 3).
WithPolling(time.Second * 5).
Should(Succeed())
})
})
var _ = When("a cluster with addon configuration is used with addons secret in the different namespace", Label("e2e"), Label(addonsTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
secretNamespace := NewNamespace()
// Create the addon secret
err := createAddonSecret(ctx, secretNamespace.Name)
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name, secretNamespace.Name)
})
cluster := NewCluster(namespace.Name)
cluster.Spec.Addons = []v1beta1.Addon{
{
SecretNamespace: secretNamespace.Name,
SecretRef: addonsSecretName,
},
}
CreateCluster(cluster)
virtualClient, restConfig := NewVirtualK8sClientAndConfig(cluster)
virtualCluster = &VirtualCluster{
Cluster: cluster,
RestConfig: restConfig,
Client: virtualClient,
}
})
It("will load the addon manifest in server pod and deploys the pod", func() {
ctx := context.Background()
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
addonContent, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, addonManifestMountPath)
Expect(err).To(Not(HaveOccurred()))
addonTestFile, err := os.ReadFile("testdata/addons/nginx.yaml")
Expect(err).To(Not(HaveOccurred()))
Expect(addonContent).To(Equal(addonTestFile))
Eventually(func(g Gomega) {
nginxPod, err := virtualCluster.Client.CoreV1().Pods("default").Get(ctx, "nginx-addon", metav1.GetOptions{})
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(nginxPod.Status.Phase).To(Equal(v1.PodRunning))
}).
WithTimeout(time.Minute * 3).
WithPolling(time.Second * 5).
Should(Succeed())
})
})
func createAddonSecret(ctx context.Context, namespace string) error {
addonContent, err := os.ReadFile("testdata/addons/nginx.yaml")
if err != nil {
return err
}
secret := &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: addonsSecretName,
Namespace: namespace,
},
Data: map[string][]byte{
"nginx.yaml": addonContent,
},
}
return k8sClient.Create(ctx, secret)
}

10
tests/cluster_certs_test.go
View File

@@ -5,8 +5,6 @@ import (
"os"
"strings"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
. "github.com/onsi/ginkgo/v2"
@@ -98,12 +96,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce
It("will load the custom certs in the server pod", func() {
ctx := context.Background()
labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
Expect(err).To(Not(HaveOccurred()))
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods.Items)).To(Equal(1))
serverPod := serverPods.Items[0]
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
// check server-ca.crt
serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt"

15
tests/cluster_persistence_test.go
View File

@@ -59,12 +59,10 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per
_, err := virtualCluster.Client.ServerVersion()
Expect(err).To(Not(HaveOccurred()))
labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
Expect(err).To(Not(HaveOccurred()))
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods.Items)).To(Equal(1))
serverPod := serverPods.Items[0]
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
@@ -75,10 +73,9 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per
// check that the server pods restarted
Eventually(func() any {
serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
Expect(err).To(Not(HaveOccurred()))
Expect(len(serverPods.Items)).To(Equal(1))
return serverPods.Items[0].DeletionTimestamp
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods)).To(Equal(1))
return serverPods[0].DeletionTimestamp
}).
WithTimeout(time.Minute).
WithPolling(time.Second * 5).

160
tests/cluster_registry_test.go Normal file
View File

@@ -0,0 +1,160 @@
package k3k_test
import (
"context"
"os"
"time"
"k8s.io/kubernetes/pkg/api/v1/pod"
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/policy"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = When("a cluster with private registry configuration is used", Label("e2e"), Label(registryTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {
ctx := context.Background()
vcp := &v1beta1.VirtualClusterPolicy{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "policy-",
},
Spec: v1beta1.VirtualClusterPolicySpec{
AllowedMode: v1beta1.VirtualClusterMode,
DisableNetworkPolicy: true,
},
}
Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
namespace := NewNamespace()
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
Expect(err).To(Not(HaveOccurred()))
namespace.Labels = map[string]string{
policy.PolicyNameLabelKey: vcp.Name,
}
Expect(k8sClient.Update(ctx, namespace)).To(Succeed())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
Expect(k8sClient.Delete(ctx, vcp)).To(Succeed())
})
err = privateRegistry(ctx, namespace.Name)
Expect(err).ToNot(HaveOccurred())
cluster := NewCluster(namespace.Name)
// configure the cluster with the private registry secrets using SecretMounts
// Using subPath allows mounting individual files while keeping parent directories writable
cluster.Spec.SecretMounts = []v1beta1.SecretMount{
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "k3s-registry-config",
},
MountPath: "/etc/rancher/k3s/registries.yaml",
SubPath: "registries.yaml",
},
{
SecretVolumeSource: v1.SecretVolumeSource{
SecretName: "private-registry-ca-cert",
},
MountPath: "/etc/rancher/k3s/tls/ca.crt",
SubPath: "tls.crt",
},
}
cluster.Spec.Mode = v1beta1.VirtualClusterMode
// airgap the k3k-server pod
err = buildRegistryNetPolicy(ctx, cluster.Namespace)
Expect(err).ToNot(HaveOccurred())
CreateCluster(cluster)
client, restConfig := NewVirtualK8sClientAndConfig(cluster)
virtualCluster = &VirtualCluster{
Cluster: cluster,
RestConfig: restConfig,
Client: client,
}
})
It("will be load the registries.yaml and crts in server pod", func() {
ctx := context.Background()
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
// check registries.yaml
registriesConfigPath := "/etc/rancher/k3s/registries.yaml"
registriesConfig, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, registriesConfigPath)
Expect(err).To(Not(HaveOccurred()))
registriesConfigTestFile, err := os.ReadFile("testdata/registry/registries.yaml")
Expect(err).To(Not(HaveOccurred()))
Expect(registriesConfig).To(Equal(registriesConfigTestFile))
// check ca.crt
CACrtPath := "/etc/rancher/k3s/tls/ca.crt"
CACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, CACrtPath)
Expect(err).To(Not(HaveOccurred()))
CACrtTestFile, err := os.ReadFile("testdata/registry/certs/ca.crt")
Expect(err).To(Not(HaveOccurred()))
Expect(CACrt).To(Equal(CACrtTestFile))
})
It("will only pull images from mirrored docker.io registry", func() {
ctx := context.Background()
// make sure that any pod using docker.io mirror works
virtualCluster.NewNginxPod("")
// creating a pod with image that uses any registry other than docker.io should fail
// for example public.ecr.aws/docker/library/alpine:latest
alpinePod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "alpine-",
Namespace: "default",
},
Spec: v1.PodSpec{
Containers: []v1.Container{{
Name: "alpine",
Image: "public.ecr.aws/docker/library/alpine:latest",
}},
},
}
By("Creating Alpine Pod and making sure its failing to start")
var err error
alpinePod, err = virtualCluster.Client.CoreV1().Pods(alpinePod.Namespace).Create(ctx, alpinePod, metav1.CreateOptions{})
Expect(err).To(Not(HaveOccurred()))
// check that the alpine Pod is failing to pull the image
Eventually(func(g Gomega) {
alpinePod, err = virtualCluster.Client.CoreV1().Pods(alpinePod.Namespace).Get(ctx, alpinePod.Name, metav1.GetOptions{})
g.Expect(err).To(Not(HaveOccurred()))
status, _ := pod.GetContainerStatus(alpinePod.Status.ContainerStatuses, "alpine")
state := status.State.Waiting
g.Expect(state).NotTo(BeNil())
g.Expect(state.Reason).To(BeEquivalentTo("ImagePullBackOff"))
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(Succeed())
})
})

19
tests/common_test.go
View File

@@ -397,26 +397,25 @@ func (c *VirtualCluster) ExecCmd(pod *v1.Pod, command string) (string, string, e
func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) {
GinkgoHelper()
labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
Expect(err).To(Not(HaveOccurred()))
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods.Items)).To(Equal(1))
serverPod := serverPods.Items[0]
Expect(len(serverPods)).To(Equal(1))
serverPod := serverPods[0]
GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, metav1.DeleteOptions{})
err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, metav1.DeleteOptions{})
Expect(err).To(Not(HaveOccurred()))
By("Deleting server pod")
// check that the server pods restarted
Eventually(func() any {
serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
Expect(err).To(Not(HaveOccurred()))
Expect(len(serverPods.Items)).To(Equal(1))
return serverPods.Items[0].DeletionTimestamp
serverPods := listServerPods(ctx, virtualCluster)
Expect(len(serverPods)).To(Equal(1))
return serverPods[0].DeletionTimestamp
}).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil())
}

9
tests/testdata/addons/nginx.yaml vendored Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Pod
metadata:
name: nginx-addon
namespace: default
spec:
containers:
- name: nginx
image: nginx:latest

33
tests/testdata/registry/certs/ca.crt vendored Normal file
View File

@@ -0,0 +1,33 @@
-----BEGIN CERTIFICATE-----
MIIFuzCCA6OgAwIBAgIUI2MIXZDFl1X+tVkpbeYv78eGB5swDQYJKoZIhvcNAQEL
BQAwbTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBUxvY2FsMQ4wDAYDVQQHDAVMb2Nh
bDESMBAGA1UECgwJUHJpdmF0ZUNBMQwwCgYDVQQLDANEZXYxHDAaBgNVBAMME1By
aXZhdGUtUmVnaXN0cnktQ0EwHhcNMjUxMTI2MTIxMDIyWhcNMzUxMTI0MTIxMDIy
WjBtMQswCQYDVQQGEwJVUzEOMAwGA1UECAwFTG9jYWwxDjAMBgNVBAcMBUxvY2Fs
MRIwEAYDVQQKDAlQcml2YXRlQ0ExDDAKBgNVBAsMA0RldjEcMBoGA1UEAwwTUHJp
dmF0ZS1SZWdpc3RyeS1DQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
AIwwuetZ0bpDmYkdP1HjQpPVjatiQHyedyulKBfPo9/uVE+HVoPe3Ku4QQgSmhKs
hzYE0g7CoM/Kr1EdCYS0RkJhLp+Mfdqwcn2+QXH0PHqdJU3ocutmxRB0xObXyXBv
2Nf+0yKNY36E096wgcJKIl8eNrONmFbPmNl7PjsIvnilQBUSUbfecxUjDcmbZxXq
dSSntMPN/twPi9FzZGMGhZzHD0+Wf+9V0dfF1L1P7rFtmdEtPbBEH6wZFpKl8qzH
O3aXxznLbRSiVZ8PP4zdEKXGcXU8bWZ0NeGPOfgQhsBZvN2auV+esb1McK+7ZzOK
P5dPuzmlCCa8F7+P8wYgHm3W5dCkwsRqvu+ht4Z2sw/Gj1Uot7hunW2jO1PWoegs
W4kGD2EFHqmYU8j3RzxppY0YvZxlSVMiQDHE11ZKHrvu0p7EGsZnAqntAH73jlEg
kt7ZhAS+AMs+evIGAFL5C6dZc8C99uiXiANausrBlVQUk068kR7W2mVy8omet2mQ
DCkmnLJnrxgq5BwPKKvqMDs6dl+idYirxiX33dhildTrkX07DIOwy0nBTdBz8O5s
PIhIA5maVZt8g8w+YpNLEJSMI7NJ1YlgIJ3P9JBOk47hlNp3gw7WlkUxW5XXS1jU
qUWUDHi8grjL9WxSbENXIY4aGIu5ag/xqgQzKGkWcr1nAgMBAAGjUzBRMB0GA1Ud
DgQWBBQYh4H6KFRZXYi9/pzeZUTgxlWQ3jAfBgNVHSMEGDAWgBQYh4H6KFRZXYi9
/pzeZUTgxlWQ3jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBt
IGJhMJsvDFXWO5pRMjA633k4zquyYBfSt1lTmH5S91SWW8sAickNVvnpBvmsG5lg
GAA1kdF9b7+yrcWGK3TldQobZVpU46MBoiqWPY8tHqbM/0x+UwHHPTwbNa5JLjxT
5ECsZa5wdeACiLt0dFv4CJ8GIcGK7k8QaoxvolEPcxbEpas7bJF9cHNZH3TEwhIb
kC6Q+4+Y2r0pxDW/2uqFpL2RSzk4kJDH3K2y3ywnkSsM6QTDXlaBKijU5bXvrq7v
ZIAX75w+dSHtj4oCWFm/jgVyS+KrAhWbjwxwRMLku0/603OQCAv7c1oEbLh58fLb
zfOOPGkpDvNP5rTwacIyW0P0/GJpmwFjaJYRJgmmCwM7S0qhhJfuVp2d7oZvkkRT
vRpNqpR813ge6T3pnXpdBA9NofTiIsxJ18CGDaHBvDBR+MAsPjmCskIsf9T3fGY/
fAzuYd8qgE2jIuyZzBLIwARq+zKSzBwgLbfnYRprbp62Qi0OQRYOqxgfP0grw29S
k++Wtfd1zr8OCX/8CkC1P85ipzUoF3G7W1Cadn4aesfxvHPKQlRlpHnt7RImwJMZ
k5QP/2igzjjGwvlGb97Br25dgjiJmQlY3IjqNpzt8uIrCK65vQSwpioq7coJWlTZ
feU/+JrEL8lsa4jkMMeYnW4IJTRqq4Aqoe+e+vz8bQ==
-----END CERTIFICATE-----

52
tests/testdata/registry/certs/ca.key vendored Normal file
View File

@@ -0,0 +1,52 @@
-----BEGIN PRIVATE KEY-----
MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCMMLnrWdG6Q5mJ
HT9R40KT1Y2rYkB8nncrpSgXz6Pf7lRPh1aD3tyruEEIEpoSrIc2BNIOwqDPyq9R
HQmEtEZCYS6fjH3asHJ9vkFx9Dx6nSVN6HLrZsUQdMTm18lwb9jX/tMijWN+hNPe
sIHCSiJfHjazjZhWz5jZez47CL54pUAVElG33nMVIw3Jm2cV6nUkp7TDzf7cD4vR
c2RjBoWcxw9Pln/vVdHXxdS9T+6xbZnRLT2wRB+sGRaSpfKsxzt2l8c5y20UolWf
Dz+M3RClxnF1PG1mdDXhjzn4EIbAWbzdmrlfnrG9THCvu2czij+XT7s5pQgmvBe/
j/MGIB5t1uXQpMLEar7vobeGdrMPxo9VKLe4bp1toztT1qHoLFuJBg9hBR6pmFPI
90c8aaWNGL2cZUlTIkAxxNdWSh677tKexBrGZwKp7QB+945RIJLe2YQEvgDLPnry
BgBS+QunWXPAvfbol4gDWrrKwZVUFJNOvJEe1tplcvKJnrdpkAwpJpyyZ68YKuQc
Dyir6jA7OnZfonWIq8Yl993YYpXU65F9OwyDsMtJwU3Qc/DubDyISAOZmlWbfIPM
PmKTSxCUjCOzSdWJYCCdz/SQTpOO4ZTad4MO1pZFMVuV10tY1KlFlAx4vIK4y/Vs
UmxDVyGOGhiLuWoP8aoEMyhpFnK9ZwIDAQABAoICACjieQZLTp/84QUc83+FQMBu
kn9+CwKNEII5C2VOWCORlSMQfEm/MCogdU7OZgK2MESvyTcmydFv8gs85a6/CJKJ
VxiO15F0zh8f4mRCb3Tu6Zc8CG/gq+4tr9MG8aeJ5vqvRZIZHAAk6slSPrWT+0w0
Oo3I6LnAl3otuCttVGdJAlRi4FQ4WuW6MGYwnTLGCt3izxQfuokhO4ydE5TRrRvY
7f0vDiaVp7o+5tlDO4ChTy+y+v+yDm6ZbnzcStbaz9u5Tg/r5OcUpNXbk5QYUKeY
JTSkp98uWxxqMeTHpRTp1uvmGNPrKzji1yZZCDL+yabuSNL571OknWRvrdeGfHjr
lK68EnznaXO5aI7/85fVl+eeU+l3lb3iVkKTmGoS9/wbJwaCy92++P9H0hfO96Zj
2ssIRNds1+3/fLrOR9ctkIHQBze/lp3OfPqglVcbx9CH9jfKp0hgC4d6osSd6cqW
bZAyz5AZb/3tiCbEhsRXgXfJ4YhaNCn7Pug9tR6YUAz6KlsfqrwpJODBYnISIO/E
kKze14lwMtYB5F1RV9plb1GeI3uVA0IY9RcENyLF4bDl5GzeiBMPhEQ3ZTCgz2Uy
AtJphdZekqXJivOhS4D8eRFBVSXrOoC3V40QYBM4DYmvlPOxRG2vxpwOrrtlOUY7
ix6tJXUb16YQWCNkMKPZAoIBAQDEsqHsGN1bJDzjci/sRKbrXvdBVvR80qt1yrfF
5CkPkbTZTnTZ8OSPGhy4icoJ0IxoXjhQidMYI/JyDES49sX373ITD9eTLD2KJGL9
kLhoUbeKsC9oFq0hmsprmmjWZ8N6rn9ONE7HEt5PM9HS7sCS8+mzBpaLJoNWgg//
00y1brjj00/g8tbfkTxEfEaeG4Artmeeq7HXm51Na1sft6XHUYTiEJ5DO+/0S45i
orf2GhHxwawoVj/WFZIpjKOA0CmncKJ3VhiVekkGMUvnP5fJaASy//AsssQ86h58
ELUiRO/tecwLI3X4FYklsOO+LqNrh4BbOlkI9fdVbwIIEvHDAoIBAQC2dMg/N5JO
DkzbVQRZIiIsZoCr7Z6mHOywZ5wpPS6E0anOxBaJTpPormX7gqpVcg5lzj3M0uYX
wQywO81dUAQkrCIri0cHnS93/5oWTlcBzVg51DmeTuhDsVlVOmFwroStCVeqZwOB
rDdOteoYkRt6lHYR3jA4WtLqsghFrpl6xagqJozB4qGZUQ3PXz1xOPaByQRwAbcd
ghZ8QngKnFe1T50WhX8r3UoFD3AqbkBcApLzwOf0Twf4QBMb+49oGZMmfZ4tnTCu
q1zPANnB0PRZVhsMr+tfc9M5GPCPVHfzQzAXnCdkgI8oTniANNDl6laj4wa64ne9
H1ZwiI7fR8eNAoIBABQKMw8P1XWUspNlrdY/hFYUndJNXqlc+VUN6z1BKqHIcYl2
Qdd2gILH4Uc32pq3Yaa8erZR5GzgNLJD57iEg9Tn01J32bnH1xk87czxsqgGM1Hw
81OCg+8Ziyf9WlMFzVexcYzxLVmA5Z9iIy1/X6VZLmUr9aiFqvnkVGb3Cyis+C9V
9xxvAU9Tx7UeiD9Rg/RwKAx1Z7AUzaj2mBkaJ8yv1H8HvGgTMjZMgFwyQdXUACIG
XljZuLVCC1sqVfoouyWxBwxrfCO2irwTx6zuwLMnYtst0jVrnSyrmaGAPkQYi+1A
7HXyDfHRl+B8LifRLpsk+gHRZwLPtHxCzA0wiOsCggEBAJvVJmqH5hdws0fpVuth
8doGOgOd0ZCCx8zq0T+Pl7ms8OE+LRlc2Ysz2Lp1oVGVNqLRAYt83TSQl2u1x/LY
spE3y39xV1szbyWIU2yVwE4zuhS6I/QH5Oxb/raCRFLfW0YG4q8RiLcqBZreWHBf
Dx8kyar9ICYhvF7ja5lIRKHNS5GklzfJfsfZqHfjGjEnu7Kho36emG1FfDro8mnt
miOrObnQjwtB10R3KQ+0Vpe/Qw+ZRQMutNncr/WIZ7U7kqifRYgj5z5n8b6DNXkK
JIhguH2fiuJdpJvxpxRjyockbWDc5/A4tQxx6Q1nDrwv54vWDRt07VvD9inrGEuv
nMkCggEBAK3Qud7mRdXT1GyVT2htguwH0EY+4czXy1mWZvIfZmd9rXzcKrYt/R08
q6Ym4liEzAyZmVJACGF5f9GyKriBinjXoCsIU9qLSsggCZhW48Ma2HXNx53g0u1U
xvIMcWgR/aMfjV2uu0keKCucP+ZCheHYq0Tn67KnhdtyhZiv9rPMVJ2IamDKC8ov
Tr/7ibSsW8xx84e/4rl310w/9CmuDtAP76aOWQ3zg5P0fq68mAKNPOhwlmHGNg7P
udczhHjEsAOiLth5A50ewf4SbnEjSTR8NYMKeIqxcgc/a74E20B84mB8+255LdsZ
OyNDZebtWToMbZu8LqzYns+6z16TmKA=
-----END PRIVATE KEY-----

35
tests/testdata/registry/certs/registry.crt vendored Normal file
View File

@@ -0,0 +1,35 @@
-----BEGIN CERTIFICATE-----
MIIGFDCCA/ygAwIBAgIUOuZ9BopS9seAhiWlSMwUSuT5KWswDQYJKoZIhvcNAQEL
BQAwbTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBUxvY2FsMQ4wDAYDVQQHDAVMb2Nh
bDESMBAGA1UECgwJUHJpdmF0ZUNBMQwwCgYDVQQLDANEZXYxHDAaBgNVBAMME1By
aXZhdGUtUmVnaXN0cnktQ0EwHhcNMjUxMTI3MTE0ODI2WhcNMzUxMTI1MTE0ODI2
WjBwMQswCQYDVQQGEwJVUzEOMAwGA1UECAwFTG9jYWwxDjAMBgNVBAcMBUxvY2Fs
MRgwFgYDVQQKDA9Qcml2YXRlUmVnaXN0cnkxDDAKBgNVBAsMA0RldjEZMBcGA1UE
AwwQcHJpdmF0ZS1yZWdpc3RyeTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC
ggIBAKKJWOvCWPuv3uRjKuRdMFigshFtRz1iFeezrs0GIefHTDX4oSV29QpeQD4Y
eURghpZhVAhE6kaLOCGIiJId5u5l4SGGHdjAt4653Px1a2TjXU2n42GK/b9Q4Mzo
KpcyyK7Rzjh5gKT1moEOgSa1hAXnnzfK6qtThFIamMg9b9xFCSEIU3kZ1us3BN1t
JjjkML27aCnEfXYlZPDVSyvV6dGG2xnI4MDiP1P5FkY10p/ryPMTbqCQpJaamwGX
xmmtieEEkXahO/xAug2cgTMz9TFmZHagvmZH7h/625MAAKMlPJvLoYM1E0q219nS
7TZ85H80ymV7G/otISnsThwX6G2gyS/s5ZbJAuKzNBAjfBl2fXOSIU+YVycbUAr4
2Wzn9rhGIh4972LYTuISAG3BjymFGsUAOxXaFcZwlGOtc1ORqLI4d5XK5yBBW+a7
+1bf9RfWq3KCDmhXgMDCaolHSMnVH3pyzkSuNfZB20Ic4cae7+TnHCUMvKBZ+H5o
N/V0Uo4WNC+tHxtTmGfjdKmvWgAzc8CaD+xtDHcJgRo45wQDBd0TaBR8XPSF9iDy
vnYDRv2cKHpfICDfvDUgN4mvHkABBj0ELQopSMq60nc8WvG6ftd2twoEMOuSekOx
PvDWQCuzhG/Rx2eAckdMIZgPAYiSjA/pCDI8SQXx3EVhW/3vAgMBAAGjgagwgaUw
HwYDVR0jBBgwFoAUGIeB+ihUWV2Ivf6c3mVE4MZVkN4wCQYDVR0TBAIwADALBgNV
HQ8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwNgYDVR0RBC8wLYIQcHJpdmF0
ZS1yZWdpc3RyeYIIcmVnaXN0cnmCCWxvY2FsaG9zdIcEfwAAATAdBgNVHQ4EFgQU
NL/k9cTjQNh6X5LfvYKTyxhccFAwDQYJKoZIhvcNAQELBQADggIBAGD8ZKPjCwhT
2UtNOVf4AXp9GRwEet1r1Q+jO/5suOMm3JsL+ohB00pZaWXdHfMpHvP/SMSXCdl5
fZTqXenTGTrxlItllLinJKjYtTYLH2LUyel8dOIhH+cM8KW/rMjwqnP/urWR+GA9
5t4wq6QZxz3cCoQbPSCRXBcuzR/b2/ebEILkyHGoixpUHsU9GjVaKHU+uqWwlUa5
RQYEgM7sCFhcdaxk2YXlBmqajOvUN/TW5Wxv0IkHZlyqg15DLUeZT+toCSrDJRh9
7zk6S3spTJL+R0P3NVDpvUeMXvo3NSJGlpUXDSqflIxs9V0FSD/Jj+koorW0TaDT
m2G0UrWE8FVuppOI4RkwN/iMBu5m5qM6OJhknZnzJEfgnX9UX+e2g89+JjT9OoVu
8s2kh0bdk+motkV7Vyfjh/jtpdesW+qkIO0RFZ2yKNIA13Tl0zu9okx0TI+Wt8FD
XRzO74dOpWwHjwCOQDaBa8Fy1cxxs+WtEMqjC36SPQzfiW9JaGMFRI9PZei7VNiY
crwOTu8wYjalXXUC8RWtXXUZQjcCp5tNPuCyJBTym2oh+uLu2mimgqYJ4KNEwNKB
21bAipI5hyY+b6AtZ/qHPxv+czL8sPhL4W/tZPCe++WWkQgC8sRa4aVrKnc14cIT
986BhJJ9j0EhPPMBOIbVo0cc75HQSQHE
-----END CERTIFICATE-----

52
tests/testdata/registry/certs/registry.key vendored Normal file
View File

@@ -0,0 +1,52 @@
-----BEGIN PRIVATE KEY-----
MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCiiVjrwlj7r97k
YyrkXTBYoLIRbUc9YhXns67NBiHnx0w1+KEldvUKXkA+GHlEYIaWYVQIROpGizgh
iIiSHebuZeEhhh3YwLeOudz8dWtk411Np+Nhiv2/UODM6CqXMsiu0c44eYCk9ZqB
DoEmtYQF5583yuqrU4RSGpjIPW/cRQkhCFN5GdbrNwTdbSY45DC9u2gpxH12JWTw
1Usr1enRhtsZyODA4j9T+RZGNdKf68jzE26gkKSWmpsBl8ZprYnhBJF2oTv8QLoN
nIEzM/UxZmR2oL5mR+4f+tuTAACjJTyby6GDNRNKttfZ0u02fOR/NMplexv6LSEp
7E4cF+htoMkv7OWWyQLiszQQI3wZdn1zkiFPmFcnG1AK+Nls5/a4RiIePe9i2E7i
EgBtwY8phRrFADsV2hXGcJRjrXNTkaiyOHeVyucgQVvmu/tW3/UX1qtygg5oV4DA
wmqJR0jJ1R96cs5ErjX2QdtCHOHGnu/k5xwlDLygWfh+aDf1dFKOFjQvrR8bU5hn
43Spr1oAM3PAmg/sbQx3CYEaOOcEAwXdE2gUfFz0hfYg8r52A0b9nCh6XyAg37w1
IDeJrx5AAQY9BC0KKUjKutJ3PFrxun7XdrcKBDDrknpDsT7w1kArs4Rv0cdngHJH
TCGYDwGIkowP6QgyPEkF8dxFYVv97wIDAQABAoICAEH/66+wN1ncTHIJIr2gaaVT
e3tAGJGAZsyzVePC/bmUYAn6b9U6vL39D7EnVvbBC2W9F9ZTxZ3nol9bhblvkvpz
PDvUrgH6H49BQc7yDy3kdVq3NcnCGs+5E8+g5sqGwJ7caxTbobVaVebZ8O+6/WU4
bJrHNwti2nRMgIWvDOEw10gmjV67c14H9V3EmKS5ZGFm3CE5vIhhHt/8fI3MSynd
zNJnk3w/Yt/CYZ0Y9fIiWHL8DQv+MBdHqHG5I8R9x2Mr67V0O1tvHR2x03TrQEFT
BrB1DVuTEcrCnq7ObXPSBw5sXaVdw/uuy2+UCub5R/+vfBBBMVchRDo1znHx81sL
ByxnSm3FSnXYuI+v/k1BzTZeZCtPodo57s7w1bL9pkz6HZgob3ZNV5C1gqWz+dyv
gznB8xkZkhUu4783nBY4zgUZAm+eptjhSGX5gqDZ+lwLBHVkm9Ir1Yu4Jo153IVq
tJkTpPsq6E0kMKw3k75b39g4hu7iO2pzZ5bu8iqEcjnkUG47fHyRzPf1VopteW3Q
u3qMgnyO7G6IRYbcy0xKeumN4qtDh0w0PANOCkvqYpYk4kD1u3a4HYygJ+423VK/
SFJvNnMJuAsWruElaFF5bkmSqVIkfbDBZTmw/BtHfA7hf5J1o59LmkOJX8XvKouF
yK11dWuXh+wDgAg2AakBAoIBAQDUYOMFn6jUk2UC8RYUCxhVQmM32BM9xySkjsWb
zNMe9Onmc9jJu6eeCORl/IAZIr1Vc+vND0zs52FOVg1zfTX4X5SFpMall+fLcfIl
MDTJag7bcZX9jJnOaDSMK4jn44wrZDZBjf5F/sLcmcFT+qECnLEI6nqLcU2vBeAH
/KtJzxDLmmFsdgH8oSsiU6dIHbHsBPNtRgP3Xgkv7iuFBBJNwQfQWjysvz+jyC48
Q8oVtJkICLeMn8A2I0a9LaROXLAQi1HV5+PtvbZnC8NzOksgKWrb8uaqQNonOCG2
t/Ck7jKLeYn7Eu/Omln1FQDAf2jA1ZjffbSyxAJ9OlGSbENPAoIBAQDD67YUuls8
z+OMzKZ6lgJSWDDVKcg59bEBk6pCL09zQ4C4KE3wL0eq9cf7yctZFZHES+kebDUp
QS9K3kX0vpyIMtteykLDqjBq0dYYbExos/7i+sCIf0dmqPaW9cyJoEZAUYdMFuvx
vofQP4lm0hSRc+ijXjEXzS69ZeD/NzxABCPrRCppCYbKRa3IfitEh2xjIArHniW0
Wpur5iHRnCHxiNMtKE4wezGPQWJlslTrs2cVt5S5hkyKxfHBsPyrVSbZBFlF9vBI
mLFy4vxkfb3xVjQR6Ezt97aeG71VyC5yvZ1K16TznvzDGwELcd/Fycjaj5vNcp+U
89ZJdXk7xnNhAoIBAFdssseD29n1+uTlHXOOxauDMpiwZ+tMaPcclpf2Dwp1QzvM
gHc6ultBydN5x7mRJWNh3rWBEOeMr++xWMQrzOW7YsZI+ET+bTrAYy+P0or/D7Kh
5V6EXGQtXUQ+P5NFhlPuYq9FpmBl6Q0qdfz99P3ARtgmvd9c+t+LiZeAGXq+tGk7
2dLuGQ9HwRvWV8xF/RHtT8+xvLw9h4algmC1NluvlGneW4+5ApeHNhE0zqF0wHIg
NH683EDs8Je7jCF94jRNRZjKZnddWxK8Mu7iFj7dDdIRAYcgPy1Z2/b9bSBXtZLY
q0Yhm3nu7A0JYk/bouGOi+mkM5hLO8MVGLMvwd0CggEALAzAKJLp1pdrMwoEWEWI
ChmYCSVWxmlOPeuEeVMHywOfWkh9lYYb1/1g1GS/mqz11Cu5I0TzAu6MAopNMkT1
Ds5YckyJjFKkhi/dsioPV+84XLJCPa5YUGWm47QqI7tscCOkhuAUdor/IDxY2Uxc
oYNtB+YypYZVfvH8D4XMvxvvM4NlAa7JporaEt0DP2ovXW4j3lPZaF6C57hbXDR9
kT/RMzL/uXjJYMszo2fgHgp9H+3hu4DNjtoIjCMN/Dut+1c19zwZNElYhFsyoil/
XlaiaHBRc6OhZJUaEcJrZxLo3Z30kW3qqLdWmcslo+PFjBaD0kJ2TNgyEtwdwOnS
oQKCAQAGRfuTC7qMzQTtlW6NDePBCwRyGSp69N93aDw6e63vVRrKq9GEkZT+6Tpw
lIOKco3oFLRhdtSsHeFdmKqQ9lldkIs0KiBLB1yIdBcxppNCqRDV5xutTcurbmhr
8spVlr8XSgYQED97JaCVB5Df7VDQOIJDeiywq76y1+mNmcUCFcBNNkBFUiq1K5U7
NNRPmoLBfXcn+fipZWNrH8PVkW7y5jRAFpbxh5sWJ4WPXa4SDnXL4DceAduUoJWV
1HaKcVoKgY0UuE3zaak6s9YA85ZjSGgwexJJCteDMIm6QuGAAqamGryIRRo8Q6ML
jBviIlTtD+bygHVmSs4oZTtJMwn1
-----END PRIVATE KEY-----

28
tests/testdata/registry/config.yml vendored Normal file
View File

@@ -0,0 +1,28 @@
version: 0.1
log:
fields:
service: registry
storage:
filesystem:
rootdirectory: /var/lib/registry
delete:
enabled: true
http:
addr: :5000
# TLS configuration
tls:
certificate: /etc/docker/registry/ssl/registry/tls.crt
key: /etc/docker/registry/ssl/registry/tls.key
# Health endpoint
headers:
X-Content-Type-Options: [nosniff]
# Using mirror registry to avoid pushing to it in the test
proxy:
remoteurl: https://registry-1.docker.io
ttl: 0

8
tests/testdata/registry/registries.yaml vendored Normal file
View File

@@ -0,0 +1,8 @@
mirrors:
docker.io:
endpoint:
- "https://private-registry:5000"
configs:
"private-registry:5000":
tls:
ca_file: /etc/rancher/k3s/tls/ca.crt

240
tests/tests_suite_test.go
View File

@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"testing"
"time"
@@ -22,15 +23,19 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/remotecommand"
"k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
@@ -49,6 +54,13 @@ const (
networkingTestsLabel = "networking"
statusTestsLabel = "status"
certificatesTestsLabel = "certificates"
registryTestsLabel = "registry"
registryImage = "registry:2"
registryCACertSecretName = "private-registry-ca-cert"
registryCertSecretName = "private-registry-cert"
registryConfigSecretName = "private-registry-config"
k3sRegistryConfigSecretName = "k3s-registry-config"
)
func TestTests(t *testing.T) {
@@ -126,6 +138,10 @@ func buildScheme() *runtime.Scheme {
err := v1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = appsv1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = networkingv1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1beta1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
@@ -533,3 +549,227 @@ func caCertSecret(name, namespace string, crt, key []byte) *v1.Secret {
},
}
}
func privateRegistry(ctx context.Context, namespace string) error {
caCrtMap := map[string]string{
"tls.crt": filepath.Join("testdata", "registry", "certs", "ca.crt"),
"tls.key": filepath.Join("testdata", "registry", "certs", "ca.key"),
}
caSecret, err := buildRegistryConfigSecret(caCrtMap, namespace, registryCACertSecretName, true)
if err != nil {
return err
}
if err := k8sClient.Create(ctx, caSecret); err != nil {
return err
}
registryCrtMap := map[string]string{
"tls.crt": filepath.Join("testdata", "registry", "certs", "registry.crt"),
"tls.key": filepath.Join("testdata", "registry", "certs", "registry.key"),
}
registrySecret, err := buildRegistryConfigSecret(registryCrtMap, namespace, registryCertSecretName, true)
if err != nil {
return err
}
if err := k8sClient.Create(ctx, registrySecret); err != nil {
return err
}
configMap := map[string]string{
"config.yml": filepath.Join("testdata", "registry", "config.yml"),
}
configSecret, err := buildRegistryConfigSecret(configMap, namespace, registryConfigSecretName, false)
if err != nil {
return err
}
if err := k8sClient.Create(ctx, configSecret); err != nil {
return err
}
k3sRegistryConfig := map[string]string{
"registries.yaml": filepath.Join("testdata", "registry", "registries.yaml"),
}
k3sRegistrySecret, err := buildRegistryConfigSecret(k3sRegistryConfig, namespace, k3sRegistryConfigSecretName, false)
if err != nil {
return err
}
if err := k8sClient.Create(ctx, k3sRegistrySecret); err != nil {
return err
}
registryDeployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "private-registry",
Namespace: namespace,
},
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
Spec: appsv1.DeploymentSpec{
Replicas: ptr.To(int32(1)),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "private-registry",
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "private-registry",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "private-registry",
Image: registryImage,
VolumeMounts: []v1.VolumeMount{
{
Name: "config",
MountPath: "/etc/docker/registry/",
},
{
Name: "ca-cert",
MountPath: "/etc/docker/registry/ssl/ca",
},
{
Name: "registry-cert",
MountPath: "/etc/docker/registry/ssl/registry",
},
},
},
},
Volumes: []v1.Volume{
{
Name: "config",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: "private-registry-config",
},
},
},
{
Name: "ca-cert",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: "private-registry-ca-cert",
},
},
},
{
Name: "registry-cert",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: "private-registry-cert",
},
},
},
},
},
},
},
}
if err := k8sClient.Create(ctx, registryDeployment); err != nil {
return err
}
registryService := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "private-registry",
Namespace: namespace,
},
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
Spec: v1.ServiceSpec{
Selector: map[string]string{
"app": "private-registry",
},
Ports: []v1.ServicePort{
{
Name: "registry-port",
Port: 5000,
TargetPort: intstr.FromInt(5000),
},
},
},
}
return k8sClient.Create(ctx, registryService)
}
func buildRegistryNetPolicy(ctx context.Context, namespace string) error {
np := networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "private-registry-test-netpol",
Namespace: namespace,
},
Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{
MatchLabels: map[string]string{
"role": "server",
},
},
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeEgress,
},
Egress: []networkingv1.NetworkPolicyEgressRule{
{
To: []networkingv1.NetworkPolicyPeer{
{
IPBlock: &networkingv1.IPBlock{
CIDR: "10.0.0.0/8",
},
},
},
},
},
},
}
return k8sClient.Create(ctx, &np)
}
func buildRegistryConfigSecret(tlsMap map[string]string, namespace, name string, tlsSecret bool) (*v1.Secret, error) {
secretType := v1.SecretTypeOpaque
if tlsSecret {
secretType = v1.SecretTypeTLS
}
data := make(map[string][]byte)
for key, path := range tlsMap {
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
data[key] = b
}
secret := &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Type: secretType,
Data: data,
}
return secret, nil
}