diff --git a/Makefile b/Makefile index 6cec4648..3e3907d5 100644 --- a/Makefile +++ b/Makefile @@ -105,10 +105,10 @@ validate: generate docs fmt ## Validate the project checking for any dependency .PHONY: install install: ## Install K3k with Helm on the targeted Kubernetes cluster helm upgrade --install --namespace k3k-system --create-namespace \ - --set image.repository=$(REPO)/k3k \ - --set image.tag=$(VERSION) \ - --set sharedAgent.image.repository=$(REPO)/k3k-kubelet \ - --set sharedAgent.image.tag=$(VERSION) \ + --set controller.image.repository=$(REPO)/k3k \ + --set controller.image.tag=$(VERSION) \ + --set agent.shared.image.repository=$(REPO)/k3k-kubelet \ + --set agent.shared.image.tag=$(VERSION) \ k3k ./charts/k3k/ .PHONY: help diff --git a/charts/k3k/templates/_helpers.tpl b/charts/k3k/templates/_helpers.tpl index 745ac4f5..ae668e17 100644 --- a/charts/k3k/templates/_helpers.tpl +++ b/charts/k3k/templates/_helpers.tpl @@ -60,3 +60,54 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* + Print the image pull secrets in the expected format (an array of objects with one possible field, "name"). +*/}} +{{- define "image.pullSecrets" }} + {{- $imagePullSecrets := list }} + {{- range . }} + {{- if kindIs "string" . }} + {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} + {{- else }} + {{- $imagePullSecrets = append $imagePullSecrets . }} + {{- end }} + {{- end }} + {{- toYaml $imagePullSecrets }} +{{- end }} + +{{- define "controller.registry" }} +{{- $registry := .Values.global.imageRegistry | default .Values.controller.image.registry -}} +{{- if $registry }} +{{- $registry }}/ +{{- else }} +{{- $registry }} +{{- end }} +{{- end }} + +{{- define "server.registry" }} +{{- $registry := .Values.global.imageRegistry | default .Values.server.image.registry -}} +{{- if $registry }} +{{- $registry }}/ +{{- else }} +{{- $registry }} +{{- end }} +{{- end }} + +{{- define "agent.virtual.registry" }} +{{- $registry := .Values.global.imageRegistry | default .Values.agent.virtual.image.registry -}} +{{- if $registry }} +{{- $registry }}/ +{{- else }} +{{- $registry }} +{{- end }} +{{- end }} + +{{- define "agent.shared.registry" }} +{{- $registry := .Values.global.imageRegistry | default .Values.agent.shared.image.registry -}} +{{- if $registry }} +{{- $registry }}/ +{{- else }} +{{- $registry }} +{{- end }} +{{- end }} diff --git a/charts/k3k/templates/deployment.yaml b/charts/k3k/templates/deployment.yaml index 93eea8b1..f7d4286e 100644 --- a/charts/k3k/templates/deployment.yaml +++ b/charts/k3k/templates/deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "k3k.labels" . | nindent 4 }} namespace: {{ .Release.Namespace }} spec: - replicas: {{ .Values.image.replicaCount }} + replicas: {{ .Values.controller.replicaCount }} selector: matchLabels: {{- include "k3k.selectorLabels" . | nindent 6 }} @@ -15,25 +15,51 @@ spec: labels: {{- include "k3k.selectorLabels" . | nindent 8 }} spec: + imagePullSecrets: {{- include "image.pullSecrets" (concat .Values.controller.imagePullSecrets .Values.global.imagePullSecrets) | nindent 8 }} containers: - - image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} + - image: "{{- include "controller.registry" .}}{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} name: {{ .Chart.Name }} + resources: + requests: + cpu: {{ .Values.controller.resources.requests.cpu }} + memory: {{ .Values.controller.resources.requests.memory }} + limits: + {{ if .Values.controller.resources.limits.cpu }} + cpu: {{ .Values.controller.resources.limits.cpu }} + {{ end }} + {{ if .Values.controller.resources.limits.memory }} + memory: {{ .Values.controller.resources.limits.memory }} + {{ end}} + args: + - k3k + {{- range $key, $value := include "image.pullSecrets" (concat .Values.agent.imagePullSecrets .Values.global.imagePullSecrets) | fromYamlArray }} + - --agent-image-pull-secret + - {{ .name }} + {{- end }} + {{- range $key, $value := include "image.pullSecrets" (concat .Values.server.imagePullSecrets .Values.global.imagePullSecrets) | fromYamlArray }} + - --server-image-pull-secret + - {{ .name }} + {{- end }} env: - name: CLUSTER_CIDR value: {{ .Values.host.clusterCIDR }} - name: SHARED_AGENT_IMAGE - value: "{{ .Values.sharedAgent.image.repository }}:{{ default .Chart.AppVersion .Values.sharedAgent.image.tag }}" - - name: SHARED_AGENT_PULL_POLICY - value: {{ .Values.sharedAgent.image.pullPolicy }} - - name: K3S_IMAGE - value: {{ .Values.k3sServer.image.repository }} - - name: K3S_IMAGE_PULL_POLICY - value: {{ .Values.k3sServer.image.pullPolicy }} + value: "{{- include "agent.shared.registry" .}}{{ .Values.agent.shared.image.repository }}:{{ default .Chart.AppVersion .Values.agent.shared.image.tag }}" + - name: SHARED_AGENT_IMAGE_PULL_POLICY + value: {{ .Values.agent.shared.image.pullPolicy }} + - name: VIRTUAL_AGENT_IMAGE + value: "{{- include "agent.virtual.registry" .}}{{ .Values.agent.virtual.image.repository }}" + - name: VIRTUAL_AGENT_IMAGE_PULL_POLICY + value: {{ .Values.agent.virtual.image.pullPolicy }} + - name: K3S_SERVER_IMAGE + value: "{{- include "server.registry" .}}{{ .Values.server.image.repository }}" + - name: K3S_SERVER_IMAGE_PULL_POLICY + value: {{ .Values.server.image.pullPolicy }} - name: KUBELET_PORT_RANGE - value: {{ .Values.sharedAgent.kubeletPortRange }} + value: {{ .Values.agent.shared.kubeletPortRange }} - name: WEBHOOK_PORT_RANGE - value: {{ .Values.sharedAgent.webhookPortRange }} + value: {{ .Values.agent.shared.webhookPortRange }} - name: CONTROLLER_NAMESPACE valueFrom: fieldRef: diff --git a/charts/k3k/values.yaml b/charts/k3k/values.yaml index 99167297..b769bc7f 100644 --- a/charts/k3k/values.yaml +++ b/charts/k3k/values.yaml @@ -1,31 +1,11 @@ -replicaCount: 1 - -image: - repository: rancher/k3k - tag: "" - pullPolicy: "" - -imagePullSecrets: [] nameOverride: "" fullnameOverride: "" -# extraEnv allows you to specify additional environment variables for the k3k controller deployment. -# This is useful for passing custom configuration or secrets to the controller. -# For example: -# extraEnv: -# - name: MY_CUSTOM_VAR -# value: "my_custom_value" -# - name: ANOTHER_VAR -# valueFrom: -# secretKeyRef: -# name: my-secret -# key: my-key -extraEnv: [] - -host: - # clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy, if not set - # the controller will collect the PodCIDRs of all the nodes on the system. - clusterCIDR: "" +global: + # -- Global override for container image registry + imageRegistry: "" + # -- Global override for container image registry pull secrets + imagePullSecrets: [] serviceAccount: # Specifies whether a service account should be created @@ -34,18 +14,65 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" -# configuration related to the shared agent mode in k3k -sharedAgent: - # Specifies the port range that will be used for k3k-kubelet api if mirrorHostNodes is enabled - kubeletPortRange: "50000-51000" - # Specifies the port range that will be used for webhook if mirrorHostNodes is enabled - webhookPortRange: "51001-52000" +host: + # clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy, if not set + # the controller will collect the PodCIDRs of all the nodes on the system. + clusterCIDR: "" + +controller: + replicaCount: 1 image: - repository: "rancher/k3k-kubelet" + registry: "" + repository: rancher/k3k tag: "" pullPolicy: "" -# image registry configuration related to the k3s server -k3sServer: + imagePullSecrets: [] + # extraEnv allows you to specify additional environment variables for the k3k controller deployment. + # This is useful for passing custom configuration or secrets to the controller. + # For example: + # extraEnv: + # - name: MY_CUSTOM_VAR + # value: "my_custom_value" + # - name: ANOTHER_VAR + # valueFrom: + # secretKeyRef: + # name: my-secret + # key: my-key + extraEnv: [] + # resources limits and requests allows you to set resources limits and requests for CPU and Memory + resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "" + memory: "" + +# configuration related to the agent component in k3k +agent: + imagePullSecrets: [] + # configuration related to agent in shared mode + shared: + # Specifies the port range that will be used for k3k-kubelet api if mirrorHostNodes is enabled + kubeletPortRange: "50000-51000" + # Specifies the port range that will be used for webhook if mirrorHostNodes is enabled + webhookPortRange: "51001-52000" + image: + registry: "" + repository: "rancher/k3k-kubelet" + tag: "" + pullPolicy: "" + # configuration related to agent in virtual mode + virtual: + image: + registry: "" + repository: "rancher/k3s" + pullPolicy: "" + +# configuration related to k3s server component in k3k +server: + imagePullSecrets: [] image: + registry: repository: "rancher/k3s" pullPolicy: "" diff --git a/docs/howtos/airgap.md b/docs/howtos/airgap.md index a8bdb36e..3f8ac657 100644 --- a/docs/howtos/airgap.md +++ b/docs/howtos/airgap.md @@ -32,18 +32,27 @@ Load these images into your internal (air-gapped) registry. Update the `values.yaml` file in the K3k Helm chart with air gap settings: ```yaml -image: - repository: rancher/k3k - tag: "" # Specify the version tag - pullPolicy: "" # Optional: "IfNotPresent", "Always", etc. - -sharedAgent: +controller: + imagePullSecrets: [] # Optional image: - repository: rancher/k3k-kubelet - tag: "" # Specify the version tag - pullPolicy: "" # Optional + repository: rancher/k3k + tag: "" # Specify the version tag + pullPolicy: "" # Optional: "IfNotPresent", "Always", etc. -k3sServer: +agent: + imagePullSecrets: [] + virtual: + image: + repository: rancher/k3s + pullPolicy: "" # Optional + shared: + image: + repository: rancher/k3k-kubelet + tag: "" # Specify the version tag + pullPolicy: "" # Optional + +server: + imagePullSecrets: [] # Optional image: repository: rancher/k3s pullPolicy: "" # Optional diff --git a/main.go b/main.go index ae39d345..f62a71de 100644 --- a/main.go +++ b/main.go @@ -31,18 +31,14 @@ import ( ) var ( - scheme = runtime.NewScheme() - clusterCIDR string - sharedAgentImage string - sharedAgentImagePullPolicy string - kubeconfig string - k3SImage string - k3SImagePullPolicy string - kubeletPortRange string - webhookPortRange string - maxConcurrentReconciles int - debug bool - logger *log.Logger + scheme = runtime.NewScheme() + config cluster.Config + kubeconfig string + kubeletPortRange string + webhookPortRange string + maxConcurrentReconciles int + debug bool + logger *log.Logger ) func init() { @@ -67,13 +63,17 @@ func main() { rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Debug level logging") rootCmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", "", "kubeconfig path") - rootCmd.PersistentFlags().StringVar(&clusterCIDR, "cluster-cidr", "", "Cluster CIDR to be added to the networkpolicy") - rootCmd.PersistentFlags().StringVar(&sharedAgentImage, "shared-agent-image", "", "K3K Virtual Kubelet image") - rootCmd.PersistentFlags().StringVar(&sharedAgentImagePullPolicy, "shared-agent-pull-policy", "", "K3K Virtual Kubelet image pull policy must be one of Always, IfNotPresent or Never") + rootCmd.PersistentFlags().StringVar(&config.ClusterCIDR, "cluster-cidr", "", "Cluster CIDR to be added to the networkpolicy") + rootCmd.PersistentFlags().StringVar(&config.SharedAgentImage, "shared-agent-image", "rancher/k3k-kubelet", "K3K Virtual Kubelet image") + rootCmd.PersistentFlags().StringVar(&config.SharedAgentImagePullPolicy, "shared-agent-image-pull-policy", "", "K3K Virtual Kubelet image pull policy must be one of Always, IfNotPresent or Never") rootCmd.PersistentFlags().StringVar(&kubeletPortRange, "kubelet-port-range", "50000-51000", "Port Range for k3k kubelet in shared mode") + rootCmd.PersistentFlags().StringVar(&config.VirtualAgentImage, "virtual-agent-image", "rancher/k3s", "K3S Virtual Agent image") + rootCmd.PersistentFlags().StringVar(&config.VirtualAgentImagePullPolicy, "virtual-agent-image-pull-policy", "", "K3S Virtual Agent image pull policy must be one of Always, IfNotPresent or Never") rootCmd.PersistentFlags().StringVar(&webhookPortRange, "webhook-port-range", "51001-52000", "Port Range for k3k kubelet webhook in shared mode") - rootCmd.PersistentFlags().StringVar(&k3SImage, "k3s-image", "rancher/k3k", "K3K server image") - rootCmd.PersistentFlags().StringVar(&k3SImagePullPolicy, "k3s-image-pull-policy", "", "K3K server image pull policy") + rootCmd.PersistentFlags().StringVar(&config.K3SServerImage, "k3s-server-image", "rancher/k3s", "K3K server image") + rootCmd.PersistentFlags().StringVar(&config.K3SServerImagePullPolicy, "k3s-server-image-pull-policy", "", "K3K server image pull policy") + rootCmd.PersistentFlags().StringSliceVar(&config.ServerImagePullSecrets, "server-image-pull-secret", nil, "Image pull secret used for for servers") + rootCmd.PersistentFlags().StringSliceVar(&config.AgentImagePullSecrets, "agent-image-pull-secret", nil, "Image pull secret used for for agents") rootCmd.PersistentFlags().IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", 50, "maximum number of concurrent reconciles") if err := rootCmd.Execute(); err != nil { @@ -113,7 +113,7 @@ func run(cmd *cobra.Command, args []string) error { return err } - if err := cluster.Add(ctx, mgr, sharedAgentImage, sharedAgentImagePullPolicy, k3SImage, k3SImagePullPolicy, maxConcurrentReconciles, portAllocator, nil); err != nil { + if err := cluster.Add(ctx, mgr, &config, maxConcurrentReconciles, portAllocator, nil); err != nil { return fmt.Errorf("failed to add the new cluster controller: %v", err) } @@ -125,7 +125,7 @@ func run(cmd *cobra.Command, args []string) error { logger.Info("adding clusterpolicy controller") - if err := policy.Add(mgr, clusterCIDR, maxConcurrentReconciles); err != nil { + if err := policy.Add(mgr, config.ClusterCIDR, maxConcurrentReconciles); err != nil { return fmt.Errorf("failed to add the clusterpolicy controller: %v", err) } @@ -139,10 +139,10 @@ func run(cmd *cobra.Command, args []string) error { } func validate() error { - if sharedAgentImagePullPolicy != "" { - if sharedAgentImagePullPolicy != string(v1.PullAlways) && - sharedAgentImagePullPolicy != string(v1.PullIfNotPresent) && - sharedAgentImagePullPolicy != string(v1.PullNever) { + if config.SharedAgentImagePullPolicy != "" { + if config.SharedAgentImagePullPolicy != string(v1.PullAlways) && + config.SharedAgentImagePullPolicy != string(v1.PullIfNotPresent) && + config.SharedAgentImagePullPolicy != string(v1.PullNever) { return errors.New("invalid value for shared agent image policy") } } diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index 56fd86cc..5f1af754 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -31,23 +31,26 @@ const ( type SharedAgent struct { *Config - serviceIP string - image string - imagePullPolicy string - token string - kubeletPort int - webhookPort int + serviceIP string + image string + imagePullPolicy string + imageRegistry string + token string + kubeletPort int + webhookPort int + imagePullSecrets []string } -func NewSharedAgent(config *Config, serviceIP, image, imagePullPolicy, token string, kubeletPort, webhookPort int) *SharedAgent { +func NewSharedAgent(config *Config, serviceIP, image, imagePullPolicy, token string, kubeletPort, webhookPort int, imagePullSecrets []string) *SharedAgent { return &SharedAgent{ - Config: config, - serviceIP: serviceIP, - image: image, - imagePullPolicy: imagePullPolicy, - token: token, - kubeletPort: kubeletPort, - webhookPort: webhookPort, + Config: config, + serviceIP: serviceIP, + image: image, + imagePullPolicy: imagePullPolicy, + token: token, + kubeletPort: kubeletPort, + webhookPort: webhookPort, + imagePullSecrets: imagePullSecrets, } } @@ -156,7 +159,13 @@ func (s *SharedAgent) podSpec() v1.PodSpec { dnsPolicy = v1.DNSClusterFirstWithHostNet } - return v1.PodSpec{ + image := s.image + + if s.imageRegistry != "" { + image = s.imageRegistry + "/" + s.image + } + + podSpec := v1.PodSpec{ HostNetwork: hostNetwork, DNSPolicy: dnsPolicy, ServiceAccountName: s.Name(), @@ -202,7 +211,7 @@ func (s *SharedAgent) podSpec() v1.PodSpec { Containers: []v1.Container{ { Name: s.Name(), - Image: s.image, + Image: image, ImagePullPolicy: v1.PullPolicy(s.imagePullPolicy), Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{}, @@ -254,6 +263,11 @@ func (s *SharedAgent) podSpec() v1.PodSpec { }, }, } + for _, imagePullSecret := range s.imagePullSecrets { + podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: imagePullSecret}) + } + + return podSpec } func (s *SharedAgent) service(ctx context.Context) error { diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index ba9b475a..90aa1b36 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -22,19 +22,22 @@ const ( type VirtualAgent struct { *Config - serviceIP string - token string - k3SImage string - k3SImagePullPolicy string + serviceIP string + token string + Image string + ImagePullPolicy string + ImageRegistry string + imagePullSecrets []string } -func NewVirtualAgent(config *Config, serviceIP, token string, k3SImage string, k3SImagePullPolicy string) *VirtualAgent { +func NewVirtualAgent(config *Config, serviceIP, token, Image, ImagePullPolicy string, imagePullSecrets []string) *VirtualAgent { return &VirtualAgent{ - Config: config, - serviceIP: serviceIP, - token: token, - k3SImage: k3SImage, - k3SImagePullPolicy: k3SImagePullPolicy, + Config: config, + serviceIP: serviceIP, + token: token, + Image: Image, + ImagePullPolicy: ImagePullPolicy, + imagePullSecrets: imagePullSecrets, } } @@ -84,7 +87,7 @@ with-node-id: true`, serviceIP, token) } func (v *VirtualAgent) deployment(ctx context.Context) error { - image := controller.K3SImage(v.cluster, v.k3SImage) + image := controller.K3SImage(v.cluster, v.Image) const name = "k3k-agent" @@ -183,7 +186,7 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect { Name: name, Image: image, - ImagePullPolicy: v1.PullPolicy(v.k3SImagePullPolicy), + ImagePullPolicy: v1.PullPolicy(v.ImagePullPolicy), SecurityContext: &v1.SecurityContext{ Privileged: ptr.To(true), }, @@ -243,5 +246,9 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect } } + for _, imagePullSecret := range v.imagePullSecrets { + podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: imagePullSecret}) + } + return podSpec } diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index e5ad9273..136ea69a 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -61,26 +61,36 @@ var ( ErrCustomCACertSecretMissing = errors.New("custom CA certificate secret is missing") ) +type Config struct { + ClusterCIDR string + SharedAgentImage string + SharedAgentImagePullPolicy string + VirtualAgentImage string + VirtualAgentImagePullPolicy string + K3SServerImage string + K3SServerImagePullPolicy string + ServerImagePullSecrets []string + AgentImagePullSecrets []string +} + type ClusterReconciler struct { DiscoveryClient *discovery.DiscoveryClient Client client.Client Scheme *runtime.Scheme + PortAllocator *agent.PortAllocator + record.EventRecorder - SharedAgentImage string - SharedAgentImagePullPolicy string - K3SImage string - K3SImagePullPolicy string - PortAllocator *agent.PortAllocator + Config } // Add adds a new controller to the manager -func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgentImagePullPolicy, k3SImage string, k3SImagePullPolicy string, maxConcurrentReconciles int, portAllocator *agent.PortAllocator, eventRecorder record.EventRecorder) error { +func Add(ctx context.Context, mgr manager.Manager, config *Config, maxConcurrentReconciles int, portAllocator *agent.PortAllocator, eventRecorder record.EventRecorder) error { discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) if err != nil { return err } - if sharedAgentImage == "" { + if config.SharedAgentImage == "" { return errors.New("missing shared agent image") } @@ -90,15 +100,21 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgent // initialize a new Reconciler reconciler := ClusterReconciler{ - DiscoveryClient: discoveryClient, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: eventRecorder, - SharedAgentImage: sharedAgentImage, - SharedAgentImagePullPolicy: sharedAgentImagePullPolicy, - K3SImage: k3SImage, - K3SImagePullPolicy: k3SImagePullPolicy, - PortAllocator: portAllocator, + DiscoveryClient: discoveryClient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EventRecorder: eventRecorder, + PortAllocator: portAllocator, + Config: Config{ + SharedAgentImage: config.SharedAgentImage, + SharedAgentImagePullPolicy: config.SharedAgentImagePullPolicy, + VirtualAgentImage: config.VirtualAgentImage, + VirtualAgentImagePullPolicy: config.VirtualAgentImagePullPolicy, + K3SServerImage: config.K3SServerImage, + K3SServerImagePullPolicy: config.K3SServerImagePullPolicy, + ServerImagePullSecrets: config.ServerImagePullSecrets, + AgentImagePullSecrets: config.AgentImagePullSecrets, + }, } return ctrl.NewControllerManagedBy(mgr). @@ -266,7 +282,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1alpha1.Clu return err } - s := server.New(cluster, c.Client, token, c.K3SImage, c.K3SImagePullPolicy) + s := server.New(cluster, c.Client, token, c.K3SServerImage, c.K3SServerImagePullPolicy, c.ServerImagePullSecrets) cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR if cluster.Status.ClusterCIDR == "" { @@ -673,7 +689,7 @@ func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1alpha1.C var agentEnsurer agent.ResourceEnsurer if cluster.Spec.Mode == agent.VirtualNodeMode { - agentEnsurer = agent.NewVirtualAgent(config, serviceIP, token, c.K3SImage, c.K3SImagePullPolicy) + agentEnsurer = agent.NewVirtualAgent(config, serviceIP, token, c.VirtualAgentImage, c.VirtualAgentImagePullPolicy, c.AgentImagePullSecrets) } else { // Assign port from pool if shared agent enabled mirroring of host nodes kubeletPort := 10250 @@ -697,7 +713,7 @@ func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1alpha1.C cluster.Status.WebhookPort = webhookPort } - agentEnsurer = agent.NewSharedAgent(config, serviceIP, c.SharedAgentImage, c.SharedAgentImagePullPolicy, token, kubeletPort, webhookPort) + agentEnsurer = agent.NewSharedAgent(config, serviceIP, c.SharedAgentImage, c.SharedAgentImagePullPolicy, token, kubeletPort, webhookPort, c.AgentImagePullSecrets) } return agentEnsurer.EnsureResources(ctx) diff --git a/pkg/controller/cluster/cluster_suite_test.go b/pkg/controller/cluster/cluster_suite_test.go index 3014477b..5d70a70a 100644 --- a/pkg/controller/cluster/cluster_suite_test.go +++ b/pkg/controller/cluster/cluster_suite_test.go @@ -71,7 +71,12 @@ var _ = BeforeSuite(func() { ctx, cancel = context.WithCancel(context.Background()) - err = cluster.Add(ctx, mgr, "rancher/k3k-kubelet:latest", "", "rancher/k3s", "", 50, portAllocator, &record.FakeRecorder{}) + clusterConfig := &cluster.Config{ + SharedAgentImage: "rancher/k3k-kubelet:latest", + K3SServerImage: "rancher/k3s", + VirtualAgentImage: "rancher/k3s", + } + err = cluster.Add(ctx, mgr, clusterConfig, 50, portAllocator, &record.FakeRecorder{}) Expect(err).NotTo(HaveOccurred()) go func() { diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 30d3d202..0130397a 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -32,22 +32,24 @@ const ( // Server type Server struct { - cluster *v1alpha1.Cluster - client client.Client - mode string - token string - k3SImage string - k3SImagePullPolicy string + cluster *v1alpha1.Cluster + client client.Client + mode string + token string + image string + imagePullPolicy string + imagePullSecrets []string } -func New(cluster *v1alpha1.Cluster, client client.Client, token string, k3SImage string, k3SImagePullPolicy string) *Server { +func New(cluster *v1alpha1.Cluster, client client.Client, token, image, imagePullPolicy string, imagePullSecrets []string) *Server { return &Server{ - cluster: cluster, - client: client, - token: token, - mode: string(cluster.Spec.Mode), - k3SImage: k3SImage, - k3SImagePullPolicy: k3SImagePullPolicy, + cluster: cluster, + client: client, + token: token, + mode: string(cluster.Spec.Mode), + image: image, + imagePullPolicy: imagePullPolicy, + imagePullSecrets: imagePullSecrets, } } @@ -119,7 +121,7 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) { Name: name, Image: image, - ImagePullPolicy: v1.PullPolicy(s.k3SImagePullPolicy), + ImagePullPolicy: v1.PullPolicy(s.imagePullPolicy), Env: []v1.EnvVar{ { Name: "POD_NAME", @@ -242,6 +244,11 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, s.cluster.Spec.ServerEnvs...) + // add image pull secrets + for _, imagePullSecret := range s.imagePullSecrets { + podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: imagePullSecret}) + } + return podSpec } @@ -253,7 +260,7 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) persistent bool ) - image := controller.K3SImage(s.cluster, s.k3SImage) + image := controller.K3SImage(s.cluster, s.image) name := controller.SafeConcatNameWithPrefix(s.cluster.Name, serverName) replicas = *s.cluster.Spec.Servers diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7a7c6d82..deb6286e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -25,19 +25,21 @@ var Backoff = wait.Backoff{ Jitter: 0.1, } -// K3SImage returns the rancher/k3s image tagged with the specified Version. +// Image returns the rancher/k3s image tagged with the specified Version. // If Version is empty it will use with the same k8s version of the host cluster, -// stored in the Status object. It will return the untagged version as last fallback. +// stored in the Status object. It will return the latest version as last fallback. func K3SImage(cluster *v1alpha1.Cluster, k3SImage string) string { + image := k3SImage + + imageVersion := "latest" + if cluster.Spec.Version != "" { - return k3SImage + ":" + cluster.Spec.Version + imageVersion = cluster.Spec.Version + } else if cluster.Status.HostVersion != "" { + imageVersion = cluster.Status.HostVersion } - if cluster.Status.HostVersion != "" { - return k3SImage + ":" + cluster.Status.HostVersion - } - - return k3SImage + return image + ":" + imageVersion } // SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix. diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go new file mode 100644 index 00000000..c37a55fd --- /dev/null +++ b/pkg/controller/controller_test.go @@ -0,0 +1,77 @@ +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" +) + +func Test_K3S_Image(t *testing.T) { + type args struct { + cluster *v1alpha1.Cluster + k3sImage string + } + + tests := []struct { + name string + args args + expectedData string + }{ + { + name: "cluster with assigned version spec", + args: args{ + k3sImage: "rancher/k3s", + cluster: &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "mycluster", + Namespace: "ns-1", + }, + Spec: v1alpha1.ClusterSpec{ + Version: "v1.2.3", + }, + }, + }, + expectedData: "rancher/k3s:v1.2.3", + }, + { + name: "cluster with empty version spec and assigned hostVersion status", + args: args{ + k3sImage: "rancher/k3s", + cluster: &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "mycluster", + Namespace: "ns-1", + }, + Status: v1alpha1.ClusterStatus{ + HostVersion: "v4.5.6", + }, + }, + }, + expectedData: "rancher/k3s:v4.5.6", + }, + { + name: "cluster with empty version spec and empty hostVersion status", + args: args{ + k3sImage: "rancher/k3s", + cluster: &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "mycluster", + Namespace: "ns-1", + }, + }, + }, + expectedData: "rancher/k3s:latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fullImage := K3SImage(tt.args.cluster, tt.args.k3sImage) + assert.Equal(t, tt.expectedData, fullImage) + }) + } +} diff --git a/tests/common_test.go b/tests/common_test.go index 0e8646d8..2900fcf5 100644 --- a/tests/common_test.go +++ b/tests/common_test.go @@ -151,12 +151,9 @@ func CreateCluster(cluster *v1alpha1.Cluster) { for _, pod := range podList.Items { imageName := pod.Spec.Containers[0].Image - imageName = strings.Split(imageName, ":")[0] // remove tag - - switch imageName { - case "rancher/k3s": + if strings.Contains(imageName, "rancher/k3s") { serverRunning = pod.Status.Phase == corev1.PodRunning - case "rancher/k3k-kubelet": + } else if strings.Contains(imageName, "rancher/k3k-kubelet") { kubeletRunning = pod.Status.Phase == corev1.PodRunning } diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index 81034a28..1370e8fe 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -145,14 +145,16 @@ func installK3kChart(ctx context.Context, kubeconfig []byte) { iCli.Timeout = time.Minute iCli.Wait = true - imageMap, _ := k3kChart.Values["image"].(map[string]any) + controllerMap, _ := k3kChart.Values["controller"].(map[string]any) + imageMap, _ := controllerMap["image"].(map[string]any) maps.Copy(imageMap, map[string]any{ "repository": "rancher/k3k", "tag": "dev", "pullPolicy": "IfNotPresent", }) - sharedAgentMap, _ := k3kChart.Values["sharedAgent"].(map[string]any) + agentMap, _ := k3kChart.Values["agent"].(map[string]any) + sharedAgentMap, _ := agentMap["shared"].(map[string]any) sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any) maps.Copy(sharedAgentImageMap, map[string]any{ "repository": "rancher/k3k-kubelet",