chore(helm)!: repository and version override for addons

api/v1alpha1/tenantcontrolplane_types.go

@@ -85,6 +85,11 @@ type ControlPlaneComponentsResources struct {
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
 	Replicas int32 `json:"replicas,omitempty"`
+	// TopologySpreadConstraints describes how the Tenant Control Plane pods ought to spread across topology
+	// domains. Scheduler will schedule pods in a way which abides by the constraints.
+	// In case of nil underlying LabelSelector, the Kamaji one for the given Tenant Control Plane will be used.
+	// All topologySpreadConstraints are ANDed.
+	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
 	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
@@ -110,12 +115,17 @@ type ServiceSpec struct {
 }
 
 // AddonSpec defines the spec for every addon.
-type AddonSpec struct{}
+type AddonSpec struct {
+	ImageOverrideTrait `json:",inline"`
+}
 
-type KubeProxySpec struct {
-	// Specify the image overried of the kube-proxy to install in the Tenant Cluster.
-	// If not specified, the Kubernetes default one will be used, according to the specified version.
-	ImageOverride string `json:"imageOverride,omitempty"`
+type ImageOverrideTrait struct {
+	// ImageRepository sets the container registry to pull images from.
+	// if not set, the default ImageRepository will be used instead.
+	ImageRepository string `json:"imageRepository,omitempty"`
+	// ImageTag allows to specify a tag for the image.
+	// In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+	ImageTag string `json:"imageTag,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.
@@ -123,13 +133,13 @@ type KonnectivitySpec struct {
 	// Port of Konnectivity proxy server.
 	ProxyPort int32 `json:"proxyPort"`
 	// Version for Konnectivity server and agent.
-	// +kubebuilder:default=v0.0.31
+	// +kubebuilder:default=v0.0.32
 	Version string `json:"version,omitempty"`
 	// ServerImage defines the container image for Konnectivity's server.
-	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	ServerImage string `json:"serverImage,omitempty"`
 	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	AgentImage string `json:"agentImage,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
@@ -137,9 +147,14 @@ type KonnectivitySpec struct {
 
 // AddonsSpec defines the enabled addons and their features.
 type AddonsSpec struct {
-	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	// Enables the DNS addon in the Tenant Cluster.
+	// The registry and the tag are configurable, the image is hard-coded to `coredns`.
+	CoreDNS *AddonSpec `json:"coreDNS,omitempty"`
+	// Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
 	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
-	KubeProxy    *KubeProxySpec    `json:"kubeProxy,omitempty"`
+	// Enables the kube-proxy addon in the Tenant Cluster.
+	// The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
+	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.

api/v1alpha1/zz_generated.deepcopy.go

@@ -61,6 +61,7 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
+	out.ImageOverrideTrait = in.ImageOverrideTrait
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
@@ -104,7 +105,7 @@ func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) {
 	}
 	if in.KubeProxy != nil {
 		in, out := &in.KubeProxy, &out.KubeProxy
-		*out = new(KubeProxySpec)
+		*out = new(AddonSpec)
 		**out = **in
 	}
 }
@@ -520,6 +521,13 @@ func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	if in.TopologySpreadConstraints != nil {
+		in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints
+		*out = make([]v1.TopologySpreadConstraint, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
 		*out = new(ControlPlaneComponentsResources)
@@ -592,6 +600,21 @@ func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageOverrideTrait.
+func (in *ImageOverrideTrait) DeepCopy() *ImageOverrideTrait {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageOverrideTrait)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -665,21 +688,6 @@ func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *KubeProxySpec) DeepCopyInto(out *KubeProxySpec) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxySpec.
-func (in *KubeProxySpec) DeepCopy() *KubeProxySpec {
-	if in == nil {
-		return nil
-	}
-	out := new(KubeProxySpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) {
 	*out = *in

charts/kamaji/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.0
+version: 0.7.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/kamaji/crds/tenantcontrolplane.yaml

@@ -63,13 +63,27 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The
+                      registry and the tag are configurable, the image is hard-coded
+                      to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster,
+                      required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -106,24 +120,31 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
+                    description: Enables the kube-proxy addon in the Tenant Cluster.
+                      The registry and the tag are configurable, the image is hard-coded
+                      to `kube-proxy`.
                     properties:
-                      imageOverride:
-                        description: Specify the image overried of the kube-proxy
-                          to install in the Tenant Cluster. If not specified, the
-                          Kubernetes default one will be used, according to the specified
-                          version.
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
                         type: string
                     type: object
                 type: object
@@ -268,6 +289,194 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how the Tenant
+                          Control Plane pods ought to spread across topology domains.
+                          Scheduler will schedule pods in a way which abides by the
+                          constraints. In case of nil underlying LabelSelector, the
+                          Kamaji one for the given Tenant Control Plane will be used.
+                          All topologySpreadConstraints are ANDed.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread
+                            matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching
+                                pods. Pods that match this label selector are counted
+                                to determine the number of pods in their corresponding
+                                topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label
+                                    selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the
+                                          selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string
+                                          values. If the operator is In or NotIn,
+                                          the values array must be non-empty. If the
+                                          operator is Exists or DoesNotExist, the
+                                          values array must be empty. This array is
+                                          replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value}
+                                    pairs. A single {key,value} in the matchLabels
+                                    map is equivalent to an element of matchExpressions,
+                                    whose key field is "key", the operator is "In",
+                                    and the values array contains only "value". The
+                                    requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            matchLabelKeys:
+                              description: MatchLabelKeys is a set of pod label keys
+                                to select the pods over which spreading will be calculated.
+                                The keys are used to lookup values from the incoming
+                                pod labels, those key-value labels are ANDed with
+                                labelSelector to select the group of existing pods
+                                over which spreading will be calculated for the incoming
+                                pod. Keys that don't exist in the incoming pod labels
+                                will be ignored. A null or empty list means only match
+                                against labelSelector.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which
+                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
+                                it is the maximum permitted difference between the
+                                number of matching pods in the target topology and
+                                the global minimum. The global minimum is the minimum
+                                number of matching pods in an eligible domain or zero
+                                if the number of eligible domains is less than MinDomains.
+                                For example, in a 3-zone cluster, MaxSkew is set to
+                                1, and pods with the same labelSelector spread as
+                                2/2/1: In this case, the global minimum is 1. | zone1
+                                | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew
+                                is 1, incoming pod can only be scheduled to zone3
+                                to become 2/2/2; scheduling it onto zone1(zone2) would
+                                make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1).
+                                - if MaxSkew is 2, incoming pod can be scheduled onto
+                                any zone. When `whenUnsatisfiable=ScheduleAnyway`,
+                                it is used to give higher precedence to topologies
+                                that satisfy it. It''s a required field. Default value
+                                is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            minDomains:
+                              description: "MinDomains indicates a minimum number
+                                of eligible domains. When the number of eligible domains
+                                with matching topology keys is less than minDomains,
+                                Pod Topology Spread treats \"global minimum\" as 0,
+                                and then the calculation of Skew is performed. And
+                                when the number of eligible domains with matching
+                                topology keys equals or greater than minDomains, this
+                                value has no effect on scheduling. As a result, when
+                                the number of eligible domains is less than minDomains,
+                                scheduler won't schedule more than maxSkew Pods to
+                                those domains. If value is nil, the constraint behaves
+                                as if MinDomains is equal to 1. Valid values are integers
+                                greater than 0. When value is not nil, WhenUnsatisfiable
+                                must be DoNotSchedule. \n For example, in a 3-zone
+                                cluster, MaxSkew is set to 2, MinDomains is set to
+                                5 and pods with the same labelSelector spread as 2/2/2:
+                                | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  |
+                                The number of domains is less than 5(MinDomains),
+                                so \"global minimum\" is treated as 0. In this situation,
+                                new pod with the same labelSelector cannot be scheduled,
+                                because computed skew will be 3(3 - 0) if new Pod
+                                is scheduled to any of the three zones, it will violate
+                                MaxSkew. \n This is a beta field and requires the
+                                MinDomainsInPodTopologySpread feature gate to be enabled
+                                (enabled by default)."
+                              format: int32
+                              type: integer
+                            nodeAffinityPolicy:
+                              description: "NodeAffinityPolicy indicates how we will
+                                treat Pod's nodeAffinity/nodeSelector when calculating
+                                pod topology spread skew. Options are: - Honor: only
+                                nodes matching nodeAffinity/nodeSelector are included
+                                in the calculations. - Ignore: nodeAffinity/nodeSelector
+                                are ignored. All nodes are included in the calculations.
+                                \n If this value is nil, the behavior is equivalent
+                                to the Honor policy. This is a alpha-level feature
+                                enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            nodeTaintsPolicy:
+                              description: "NodeTaintsPolicy indicates how we will
+                                treat node taints when calculating pod topology spread
+                                skew. Options are: - Honor: nodes without taints,
+                                along with tainted nodes for which the incoming pod
+                                has a toleration, are included. - Ignore: node taints
+                                are ignored. All nodes are included. \n If this value
+                                is nil, the behavior is equivalent to the Ignore policy.
+                                This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            topologyKey:
+                              description: TopologyKey is the key of node labels.
+                                Nodes that have a label with this key and identical
+                                values are considered to be in the same topology.
+                                We consider each <key, value> as a "bucket", and try
+                                to put balanced number of pods into each bucket. We
+                                define a domain as a particular instance of a topology.
+                                Also, we define an eligible domain as a domain whose
+                                nodes meet the requirements of nodeAffinityPolicy
+                                and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname",
+                                each Node is a domain of that topology. And, if TopologyKey
+                                is "topology.kubernetes.io/zone", each zone is a domain
+                                of that topology. It's a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal
+                                with a pod if it doesn''t satisfy the spread constraint.
+                                - DoNotSchedule (default) tells the scheduler not
+                                to schedule it. - ScheduleAnyway tells the scheduler
+                                to schedule the pod in any location, but giving higher
+                                precedence to topologies that would help reduce the
+                                skew. A constraint is considered "Unsatisfiable" for
+                                an incoming pod if and only if every possible node
+                                assignment for that pod would violate "MaxSkew" on
+                                some topology. For example, in a 3-zone cluster, MaxSkew
+                                is set to 1, and pods with the same labelSelector
+                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
+                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
+                                incoming pod can only be scheduled to zone2(zone3)
+                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
+                                satisfies MaxSkew(1). In other words, the cluster
+                                can still be imbalanced, but scheduler won''t make
+                                it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -63,13 +63,27 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The
+                      registry and the tag are configurable, the image is hard-coded
+                      to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster,
+                      required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -106,24 +120,31 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
+                    description: Enables the kube-proxy addon in the Tenant Cluster.
+                      The registry and the tag are configurable, the image is hard-coded
+                      to `kube-proxy`.
                     properties:
-                      imageOverride:
-                        description: Specify the image overried of the kube-proxy
-                          to install in the Tenant Cluster. If not specified, the
-                          Kubernetes default one will be used, according to the specified
-                          version.
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
                         type: string
                     type: object
                 type: object
@@ -268,6 +289,194 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how the Tenant
+                          Control Plane pods ought to spread across topology domains.
+                          Scheduler will schedule pods in a way which abides by the
+                          constraints. In case of nil underlying LabelSelector, the
+                          Kamaji one for the given Tenant Control Plane will be used.
+                          All topologySpreadConstraints are ANDed.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread
+                            matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching
+                                pods. Pods that match this label selector are counted
+                                to determine the number of pods in their corresponding
+                                topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label
+                                    selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the
+                                          selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string
+                                          values. If the operator is In or NotIn,
+                                          the values array must be non-empty. If the
+                                          operator is Exists or DoesNotExist, the
+                                          values array must be empty. This array is
+                                          replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value}
+                                    pairs. A single {key,value} in the matchLabels
+                                    map is equivalent to an element of matchExpressions,
+                                    whose key field is "key", the operator is "In",
+                                    and the values array contains only "value". The
+                                    requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            matchLabelKeys:
+                              description: MatchLabelKeys is a set of pod label keys
+                                to select the pods over which spreading will be calculated.
+                                The keys are used to lookup values from the incoming
+                                pod labels, those key-value labels are ANDed with
+                                labelSelector to select the group of existing pods
+                                over which spreading will be calculated for the incoming
+                                pod. Keys that don't exist in the incoming pod labels
+                                will be ignored. A null or empty list means only match
+                                against labelSelector.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which
+                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
+                                it is the maximum permitted difference between the
+                                number of matching pods in the target topology and
+                                the global minimum. The global minimum is the minimum
+                                number of matching pods in an eligible domain or zero
+                                if the number of eligible domains is less than MinDomains.
+                                For example, in a 3-zone cluster, MaxSkew is set to
+                                1, and pods with the same labelSelector spread as
+                                2/2/1: In this case, the global minimum is 1. | zone1
+                                | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew
+                                is 1, incoming pod can only be scheduled to zone3
+                                to become 2/2/2; scheduling it onto zone1(zone2) would
+                                make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1).
+                                - if MaxSkew is 2, incoming pod can be scheduled onto
+                                any zone. When `whenUnsatisfiable=ScheduleAnyway`,
+                                it is used to give higher precedence to topologies
+                                that satisfy it. It''s a required field. Default value
+                                is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            minDomains:
+                              description: "MinDomains indicates a minimum number
+                                of eligible domains. When the number of eligible domains
+                                with matching topology keys is less than minDomains,
+                                Pod Topology Spread treats \"global minimum\" as 0,
+                                and then the calculation of Skew is performed. And
+                                when the number of eligible domains with matching
+                                topology keys equals or greater than minDomains, this
+                                value has no effect on scheduling. As a result, when
+                                the number of eligible domains is less than minDomains,
+                                scheduler won't schedule more than maxSkew Pods to
+                                those domains. If value is nil, the constraint behaves
+                                as if MinDomains is equal to 1. Valid values are integers
+                                greater than 0. When value is not nil, WhenUnsatisfiable
+                                must be DoNotSchedule. \n For example, in a 3-zone
+                                cluster, MaxSkew is set to 2, MinDomains is set to
+                                5 and pods with the same labelSelector spread as 2/2/2:
+                                | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  |
+                                The number of domains is less than 5(MinDomains),
+                                so \"global minimum\" is treated as 0. In this situation,
+                                new pod with the same labelSelector cannot be scheduled,
+                                because computed skew will be 3(3 - 0) if new Pod
+                                is scheduled to any of the three zones, it will violate
+                                MaxSkew. \n This is a beta field and requires the
+                                MinDomainsInPodTopologySpread feature gate to be enabled
+                                (enabled by default)."
+                              format: int32
+                              type: integer
+                            nodeAffinityPolicy:
+                              description: "NodeAffinityPolicy indicates how we will
+                                treat Pod's nodeAffinity/nodeSelector when calculating
+                                pod topology spread skew. Options are: - Honor: only
+                                nodes matching nodeAffinity/nodeSelector are included
+                                in the calculations. - Ignore: nodeAffinity/nodeSelector
+                                are ignored. All nodes are included in the calculations.
+                                \n If this value is nil, the behavior is equivalent
+                                to the Honor policy. This is a alpha-level feature
+                                enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            nodeTaintsPolicy:
+                              description: "NodeTaintsPolicy indicates how we will
+                                treat node taints when calculating pod topology spread
+                                skew. Options are: - Honor: nodes without taints,
+                                along with tainted nodes for which the incoming pod
+                                has a toleration, are included. - Ignore: node taints
+                                are ignored. All nodes are included. \n If this value
+                                is nil, the behavior is equivalent to the Ignore policy.
+                                This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            topologyKey:
+                              description: TopologyKey is the key of node labels.
+                                Nodes that have a label with this key and identical
+                                values are considered to be in the same topology.
+                                We consider each <key, value> as a "bucket", and try
+                                to put balanced number of pods into each bucket. We
+                                define a domain as a particular instance of a topology.
+                                Also, we define an eligible domain as a domain whose
+                                nodes meet the requirements of nodeAffinityPolicy
+                                and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname",
+                                each Node is a domain of that topology. And, if TopologyKey
+                                is "topology.kubernetes.io/zone", each zone is a domain
+                                of that topology. It's a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal
+                                with a pod if it doesn''t satisfy the spread constraint.
+                                - DoNotSchedule (default) tells the scheduler not
+                                to schedule it. - ScheduleAnyway tells the scheduler
+                                to schedule the pod in any location, but giving higher
+                                precedence to topologies that would help reduce the
+                                skew. A constraint is considered "Unsatisfiable" for
+                                an incoming pod if and only if every possible node
+                                assignment for that pod would violate "MaxSkew" on
+                                some topology. For example, in a 3-zone cluster, MaxSkew
+                                is set to 1, and pods with the same labelSelector
+                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
+                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
+                                incoming pod can only be scheduled to zone2(zone3)
+                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
+                                satisfies MaxSkew(1). In other words, the cluster
+                                can still be imbalanced, but scheduler won''t make
+                                it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which

config/install.yaml

@@ -64,13 +64,20 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's agent.
                         type: string
                       proxyPort:
@@ -100,20 +107,24 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
+                    description: Enables the kube-proxy addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
                     properties:
-                      imageOverride:
-                        description: Specify the image overried of the kube-proxy to install in the Tenant Cluster. If not specified, the Kubernetes default one will be used, according to the specified version.
+                      imageRepository:
+                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
                         type: string
                     type: object
                 type: object
@@ -230,6 +241,74 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how the Tenant Control Plane pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. In case of nil underlying LabelSelector, the Kamaji one for the given Tenant Control Plane will be used. All topologySpreadConstraints are ANDed.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            matchLabelKeys:
+                              description: MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It''s a required field. Default value is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            minDomains:
+                              description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. \n This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default)."
+                              format: int32
+                              type: integer
+                            nodeAffinityPolicy:
+                              description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. \n If this value is nil, the behavior is equivalent to the Honor policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+                              type: string
+                            nodeTaintsPolicy:
+                              description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. \n If this value is nil, the behavior is equivalent to the Ignore policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+                              type: string
+                            topologyKey:
+                              description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each <key, value> as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. It's a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane

docs/reference.md

@@ -101,7 +101,7 @@ addons:
 addons:
   konnectivity:
     proxyPort: 31132 # mandatory
-    version: v0.0.31
+    version: v0.0.32
     resources:
       requests:
         cpu: 100m
@@ -109,6 +109,6 @@ addons:
       limits:
         cpu: 100m
         memory: 128Mi
-    serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
-    agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+    serverImage: registry.k8s.io/kas-network-proxy/proxy-server
+    agentImage: registry.k8s.io/kas-network-proxy/proxy-agent
 ```

internal/builders/controlplane/deployment.go

@@ -772,6 +772,18 @@ func (d *Deployment) SetAnnotations(resource *appsv1.Deployment, annotations map
 	resource.SetAnnotations(annotations)
 }
 
+func (d *Deployment) SetTopologySpreadConstraints(spec *appsv1.DeploymentSpec, topologies []corev1.TopologySpreadConstraint) {
+	defaultSelector := spec.Selector
+
+	for index, topology := range topologies {
+		if topology.LabelSelector == nil {
+			topologies[index].LabelSelector = defaultSelector
+		}
+	}
+
+	spec.Template.Spec.TopologySpreadConstraints = topologies
+}
+
 // ResetKubeAPIServerFlags ensures that upon a change of the kube-apiserver extra flags the desired ones are properly
 // applied, also considering that the container could be lately patched by the konnectivity addon resources.
 func (d *Deployment) ResetKubeAPIServerFlags(resource *appsv1.Deployment, tcp *kamajiv1alpha1.TenantControlPlane) {

internal/kubeadm/addon.go

@@ -34,6 +34,13 @@ const (
 )
 
 func AddCoreDNS(client kubernetes.Interface, config *Configuration) error {
+	// We're passing the values from the parameters here because they wouldn't be hashed by the YAML encoder:
+	// the struct kubeadm.ClusterConfiguration hasn't struct tags, and it wouldn't be hashed properly.
+	if opts := config.Parameters.CoreDNSOptions; opts != nil {
+		config.InitConfiguration.DNS.ImageRepository = opts.Repository
+		config.InitConfiguration.DNS.ImageTag = opts.Tag
+	}
+
 	return dns.EnsureDNSAddon(&config.InitConfiguration.ClusterConfiguration, client)
 }
 
@@ -113,7 +120,9 @@ func AddKubeProxy(client kubernetes.Interface, config *Configuration) error {
 		return err
 	}
 
-	if err := createKubeProxyAddon(client, config.Parameters.KubeProxyImage); err != nil {
+	image := fmt.Sprintf("%s/kube-proxy:%s", config.Parameters.KubeProxyOptions.Repository, config.Parameters.KubeProxyOptions.Tag)
+
+	if err := createKubeProxyAddon(client, image); err != nil {
 		return err
 	}
 

internal/kubeadm/types.go

@@ -46,7 +46,13 @@ type Parameters struct {
 	ETCDs                          []string
 	CertificatesDir                string
 	KubeconfigDir                  string
-	KubeProxyImage                 string
+	KubeProxyOptions               *AddonOptions
+	CoreDNSOptions                 *AddonOptions
+}
+
+type AddonOptions struct {
+	Repository string
+	Tag        string
 }
 
 type KubeletConfiguration struct {

internal/resources/k8s_deployment_resource.go

@@ -71,6 +71,7 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl
 		d.SetTemplateLabels(&r.resource.Spec.Template, r.deploymentTemplateLabels(ctx, tenantControlPlane))
 		d.SetStrategy(&r.resource.Spec)
 		d.SetSelector(&r.resource.Spec, tenantControlPlane)
+		d.SetTopologySpreadConstraints(&r.resource.Spec, tenantControlPlane.Spec.ControlPlane.Deployment.TopologySpreadConstraints)
 		d.SetReplicas(&r.resource.Spec, tenantControlPlane)
 		d.ResetKubeAPIServerFlags(r.resource, tenantControlPlane)
 		d.SetContainers(&r.resource.Spec.Template.Spec, tenantControlPlane, address)

internal/resources/kubeadm_utils.go

@@ -5,7 +5,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -40,11 +39,35 @@ func KubeadmPhaseCreate(ctx context.Context, r KubeadmPhaseResource, tenantContr
 		TenantControlPlaneCertSANs:     tenantControlPlane.Spec.NetworkProfile.CertSANs,
 		TenantControlPlanePort:         tenantControlPlane.Spec.NetworkProfile.Port,
 		TenantControlPlaneCGroupDriver: tenantControlPlane.Spec.Kubernetes.Kubelet.CGroupFS.String(),
-		KubeProxyImage:                 fmt.Sprintf("k8s.gcr.io/kube-proxy:%s", tenantControlPlane.Spec.Kubernetes.Version),
 	}
 
-	if kubeProxy := tenantControlPlane.Spec.Addons.KubeProxy; kubeProxy != nil && len(kubeProxy.ImageOverride) > 0 {
-		config.Parameters.KubeProxyImage = kubeProxy.ImageOverride
+	// If CoreDNS addon is enabled and with an override, adding these to the kubeadm init configuration
+	if coreDNS := tenantControlPlane.Spec.Addons.CoreDNS; coreDNS != nil {
+		config.Parameters.CoreDNSOptions = &kubeadm.AddonOptions{}
+
+		if len(coreDNS.ImageRepository) > 0 {
+			config.Parameters.CoreDNSOptions.Repository = coreDNS.ImageRepository
+		}
+
+		if len(coreDNS.ImageRepository) > 0 {
+			config.Parameters.CoreDNSOptions.Tag = coreDNS.ImageTag
+		}
+	}
+	// If the kube-proxy addon is enabled and with overrides, adding it to the kubeadm parameters
+	if kubeProxy := tenantControlPlane.Spec.Addons.KubeProxy; kubeProxy != nil {
+		config.Parameters.KubeProxyOptions = &kubeadm.AddonOptions{}
+
+		if len(kubeProxy.ImageRepository) > 0 {
+			config.Parameters.KubeProxyOptions.Repository = kubeProxy.ImageRepository
+		} else {
+			config.Parameters.KubeProxyOptions.Repository = "k8s.gcr.io"
+		}
+
+		if len(kubeProxy.ImageTag) > 0 {
+			config.Parameters.KubeProxyOptions.Tag = kubeProxy.ImageTag
+		} else {
+			config.Parameters.KubeProxyOptions.Tag = tenantControlPlane.Spec.Kubernetes.Version
+		}
 	}
 
 	checksum := config.Checksum()