chore(helm): adding scale subresource

api/v1alpha1/tenantcontrolplane_status.go

@@ -208,6 +208,8 @@ type KubernetesVersion struct {
 // KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
 type KubernetesDeploymentStatus struct {
 	appsv1.DeploymentStatus `json:",inline"`
+	// Selector is the label selector used to group the Tenant Control Plane Pods used by the scale subresource.
+	Selector string `json:"selector"`
 	// The name of the Deployment for the given cluster.
 	Name string `json:"name"`
 	// The namespace which the Deployment for the given cluster is deployed.

api/v1alpha1/tenantcontrolplane_types.go

@@ -115,12 +115,17 @@ type ServiceSpec struct {
 }
 
 // AddonSpec defines the spec for every addon.
-type AddonSpec struct{}
+type AddonSpec struct {
+	ImageOverrideTrait `json:",inline"`
+}
 
-type KubeProxySpec struct {
-	// Specify the image overried of the kube-proxy to install in the Tenant Cluster.
-	// If not specified, the Kubernetes default one will be used, according to the specified version.
-	ImageOverride string `json:"imageOverride,omitempty"`
+type ImageOverrideTrait struct {
+	// ImageRepository sets the container registry to pull images from.
+	// if not set, the default ImageRepository will be used instead.
+	ImageRepository string `json:"imageRepository,omitempty"`
+	// ImageTag allows to specify a tag for the image.
+	// In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+	ImageTag string `json:"imageTag,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.
@@ -128,13 +133,13 @@ type KonnectivitySpec struct {
 	// Port of Konnectivity proxy server.
 	ProxyPort int32 `json:"proxyPort"`
 	// Version for Konnectivity server and agent.
-	// +kubebuilder:default=v0.0.31
+	// +kubebuilder:default=v0.0.32
 	Version string `json:"version,omitempty"`
 	// ServerImage defines the container image for Konnectivity's server.
-	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	ServerImage string `json:"serverImage,omitempty"`
 	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	AgentImage string `json:"agentImage,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
@@ -142,9 +147,14 @@ type KonnectivitySpec struct {
 
 // AddonsSpec defines the enabled addons and their features.
 type AddonsSpec struct {
-	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	// Enables the DNS addon in the Tenant Cluster.
+	// The registry and the tag are configurable, the image is hard-coded to `coredns`.
+	CoreDNS *AddonSpec `json:"coreDNS,omitempty"`
+	// Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
 	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
-	KubeProxy    *KubeProxySpec    `json:"kubeProxy,omitempty"`
+	// Enables the kube-proxy addon in the Tenant Cluster.
+	// The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
+	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
@@ -164,6 +174,7 @@ type TenantControlPlaneSpec struct {
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
 // +kubebuilder:resource:shortName=tcp
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Kubernetes version"

api/v1alpha1/zz_generated.deepcopy.go

@@ -61,6 +61,7 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
+	out.ImageOverrideTrait = in.ImageOverrideTrait
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
@@ -104,7 +105,7 @@ func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) {
 	}
 	if in.KubeProxy != nil {
 		in, out := &in.KubeProxy, &out.KubeProxy
-		*out = new(KubeProxySpec)
+		*out = new(AddonSpec)
 		**out = **in
 	}
 }
@@ -599,6 +600,21 @@ func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageOverrideTrait.
+func (in *ImageOverrideTrait) DeepCopy() *ImageOverrideTrait {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageOverrideTrait)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -672,21 +688,6 @@ func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *KubeProxySpec) DeepCopyInto(out *KubeProxySpec) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxySpec.
-func (in *KubeProxySpec) DeepCopy() *KubeProxySpec {
-	if in == nil {
-		return nil
-	}
-	out := new(KubeProxySpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) {
 	*out = *in

charts/kamaji/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.6.0
+version: 0.8.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/kamaji/crds/tenantcontrolplane.yaml

@@ -63,13 +63,27 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The
+                      registry and the tag are configurable, the image is hard-coded
+                      to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster,
+                      required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -106,24 +120,31 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
+                    description: Enables the kube-proxy addon in the Tenant Cluster.
+                      The registry and the tag are configurable, the image is hard-coded
+                      to `kube-proxy`.
                     properties:
-                      imageOverride:
-                        description: Specify the image overried of the kube-proxy
-                          to install in the Tenant Cluster. If not specified, the
-                          Kubernetes default one will be used, according to the specified
-                          version.
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
                         type: string
                     type: object
                 type: object
@@ -1206,6 +1227,10 @@ spec:
                           by this deployment (their labels match the selector).
                         format: int32
                         type: integer
+                      selector:
+                        description: Selector is the label selector used to group
+                          the Tenant Control Plane Pods used by the scale subresource.
+                        type: string
                       unavailableReplicas:
                         description: Total number of unavailable pods targeted by
                           this deployment. This is the total number of pods that are
@@ -1222,6 +1247,7 @@ spec:
                     required:
                     - name
                     - namespace
+                    - selector
                     type: object
                   ingress:
                     description: KubernetesIngressStatus defines the status for the
@@ -1526,4 +1552,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.kubernetesResources.deployment.selector
+        specReplicasPath: .spec.deployment.replicas
+        statusReplicasPath: .status.kubernetesResources.deployment.replicas
       status: {}

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -63,13 +63,27 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The
+                      registry and the tag are configurable, the image is hard-coded
+                      to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster,
+                      required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -106,24 +120,31 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
+                    description: Enables the kube-proxy addon in the Tenant Cluster.
+                      The registry and the tag are configurable, the image is hard-coded
+                      to `kube-proxy`.
                     properties:
-                      imageOverride:
-                        description: Specify the image overried of the kube-proxy
-                          to install in the Tenant Cluster. If not specified, the
-                          Kubernetes default one will be used, according to the specified
-                          version.
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
                         type: string
                     type: object
                 type: object
@@ -1206,6 +1227,10 @@ spec:
                           by this deployment (their labels match the selector).
                         format: int32
                         type: integer
+                      selector:
+                        description: Selector is the label selector used to group
+                          the Tenant Control Plane Pods used by the scale subresource.
+                        type: string
                       unavailableReplicas:
                         description: Total number of unavailable pods targeted by
                           this deployment. This is the total number of pods that are
@@ -1222,6 +1247,7 @@ spec:
                     required:
                     - name
                     - namespace
+                    - selector
                     type: object
                   ingress:
                     description: KubernetesIngressStatus defines the status for the
@@ -1526,4 +1552,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.kubernetesResources.deployment.selector
+        specReplicasPath: .spec.deployment.replicas
+        statusReplicasPath: .status.kubernetesResources.deployment.replicas
       status: {}

config/install.yaml

@@ -64,13 +64,20 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's agent.
                         type: string
                       proxyPort:
@@ -100,20 +107,24 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
+                    description: Enables the kube-proxy addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
                     properties:
-                      imageOverride:
-                        description: Specify the image overried of the kube-proxy to install in the Tenant Cluster. If not specified, the Kubernetes default one will be used, according to the specified version.
+                      imageRepository:
+                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
                         type: string
                     type: object
                 type: object
@@ -932,6 +943,9 @@ spec:
                         description: Total number of non-terminated pods targeted by this deployment (their labels match the selector).
                         format: int32
                         type: integer
+                      selector:
+                        description: Selector is the label selector used to group the Tenant Control Plane Pods used by the scale subresource.
+                        type: string
                       unavailableReplicas:
                         description: Total number of unavailable pods targeted by this deployment. This is the total number of pods that are still required for the deployment to have 100% available capacity. They may either be pods that are running but not yet available or pods that still have not been created.
                         format: int32
@@ -943,6 +957,7 @@ spec:
                     required:
                     - name
                     - namespace
+                    - selector
                     type: object
                   ingress:
                     description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
@@ -1161,6 +1176,10 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.kubernetesResources.deployment.selector
+        specReplicasPath: .spec.deployment.replicas
+        statusReplicasPath: .status.kubernetesResources.deployment.replicas
       status: {}
 ---
 apiVersion: v1

docs/reference.md

@@ -101,7 +101,7 @@ addons:
 addons:
   konnectivity:
     proxyPort: 31132 # mandatory
-    version: v0.0.31
+    version: v0.0.32
     resources:
       requests:
         cpu: 100m
@@ -109,6 +109,6 @@ addons:
       limits:
         cpu: 100m
         memory: 128Mi
-    serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
-    agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+    serverImage: registry.k8s.io/kas-network-proxy/proxy-server
+    agentImage: registry.k8s.io/kas-network-proxy/proxy-agent
 ```

internal/kubeadm/addon.go

@@ -34,6 +34,13 @@ const (
 )
 
 func AddCoreDNS(client kubernetes.Interface, config *Configuration) error {
+	// We're passing the values from the parameters here because they wouldn't be hashed by the YAML encoder:
+	// the struct kubeadm.ClusterConfiguration hasn't struct tags, and it wouldn't be hashed properly.
+	if opts := config.Parameters.CoreDNSOptions; opts != nil {
+		config.InitConfiguration.DNS.ImageRepository = opts.Repository
+		config.InitConfiguration.DNS.ImageTag = opts.Tag
+	}
+
 	return dns.EnsureDNSAddon(&config.InitConfiguration.ClusterConfiguration, client)
 }
 
@@ -113,7 +120,9 @@ func AddKubeProxy(client kubernetes.Interface, config *Configuration) error {
 		return err
 	}
 
-	if err := createKubeProxyAddon(client, config.Parameters.KubeProxyImage); err != nil {
+	image := fmt.Sprintf("%s/kube-proxy:%s", config.Parameters.KubeProxyOptions.Repository, config.Parameters.KubeProxyOptions.Tag)
+
+	if err := createKubeProxyAddon(client, image); err != nil {
 		return err
 	}
 

internal/kubeadm/types.go

@@ -46,7 +46,13 @@ type Parameters struct {
 	ETCDs                          []string
 	CertificatesDir                string
 	KubeconfigDir                  string
-	KubeProxyImage                 string
+	KubeProxyOptions               *AddonOptions
+	CoreDNSOptions                 *AddonOptions
+}
+
+type AddonOptions struct {
+	Repository string
+	Tag        string
 }
 
 type KubeletConfiguration struct {

internal/resources/k8s_deployment_resource.go

@@ -104,6 +104,7 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.
 
 	tenantControlPlane.Status.Kubernetes.Deployment = kamajiv1alpha1.KubernetesDeploymentStatus{
 		DeploymentStatus: r.resource.Status,
+		Selector:         metav1.FormatLabelSelector(r.resource.Spec.Selector),
 		Name:             r.resource.GetName(),
 		Namespace:        r.resource.GetNamespace(),
 		LastUpdate:       metav1.Now(),

internal/resources/kubeadm_utils.go

@@ -5,7 +5,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -40,11 +39,35 @@ func KubeadmPhaseCreate(ctx context.Context, r KubeadmPhaseResource, tenantContr
 		TenantControlPlaneCertSANs:     tenantControlPlane.Spec.NetworkProfile.CertSANs,
 		TenantControlPlanePort:         tenantControlPlane.Spec.NetworkProfile.Port,
 		TenantControlPlaneCGroupDriver: tenantControlPlane.Spec.Kubernetes.Kubelet.CGroupFS.String(),
-		KubeProxyImage:                 fmt.Sprintf("k8s.gcr.io/kube-proxy:%s", tenantControlPlane.Spec.Kubernetes.Version),
 	}
 
-	if kubeProxy := tenantControlPlane.Spec.Addons.KubeProxy; kubeProxy != nil && len(kubeProxy.ImageOverride) > 0 {
-		config.Parameters.KubeProxyImage = kubeProxy.ImageOverride
+	// If CoreDNS addon is enabled and with an override, adding these to the kubeadm init configuration
+	if coreDNS := tenantControlPlane.Spec.Addons.CoreDNS; coreDNS != nil {
+		config.Parameters.CoreDNSOptions = &kubeadm.AddonOptions{}
+
+		if len(coreDNS.ImageRepository) > 0 {
+			config.Parameters.CoreDNSOptions.Repository = coreDNS.ImageRepository
+		}
+
+		if len(coreDNS.ImageRepository) > 0 {
+			config.Parameters.CoreDNSOptions.Tag = coreDNS.ImageTag
+		}
+	}
+	// If the kube-proxy addon is enabled and with overrides, adding it to the kubeadm parameters
+	if kubeProxy := tenantControlPlane.Spec.Addons.KubeProxy; kubeProxy != nil {
+		config.Parameters.KubeProxyOptions = &kubeadm.AddonOptions{}
+
+		if len(kubeProxy.ImageRepository) > 0 {
+			config.Parameters.KubeProxyOptions.Repository = kubeProxy.ImageRepository
+		} else {
+			config.Parameters.KubeProxyOptions.Repository = "k8s.gcr.io"
+		}
+
+		if len(kubeProxy.ImageTag) > 0 {
+			config.Parameters.KubeProxyOptions.Tag = kubeProxy.ImageTag
+		} else {
+			config.Parameters.KubeProxyOptions.Tag = tenantControlPlane.Spec.Kubernetes.Version
+		}
 	}
 
 	checksum := config.Checksum()