Compare commits

..

2 Commits

Author SHA1 Message Date
Jonathan Crowther
25e910ccaf Add initial affinity to podspecs (#696)
* Add initial affinity to podspecs

* Fix go generate

* Add field to the policy and prioritize it over the cluster spec

* Fix linter issue

* Add docs

* Address comments

* Fix the tests and improve the field descriptions

* Fix formatter issues

* Change logs to info level

* run validation

* undo pandoc changes
2026-03-23 16:16:30 -04:00
Enrico Candino
3ec7434ce3 Add status field in CRDs docs (#720)
* add status field in docs

* add status field in docs
2026-03-23 20:43:03 +01:00
12 changed files with 5585 additions and 11 deletions

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,6 @@
processor:
# RE2 regular expressions describing type fields that should be excluded from the generated documentation.
ignoreFields:
- "status$"
- "TypeMeta$"
render:

View File

@@ -61,6 +61,10 @@ _Appears In:_
| *`priorityClass`* __string__ | priorityClass is the priority class enforced by the active VirtualClusterPolicy. + | |
| *`nodeSelector`* __object (keys:string, values:string)__ | nodeSelector is a node selector enforced by the active VirtualClusterPolicy. + | |
| *`serverAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | serverAffinity is the affinity rules for server pods enforced by the active VirtualClusterPolicy. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`agentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`sync`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-syncconfig[$$SyncConfig$$]__ | sync is the SyncConfig enforced by the active VirtualClusterPolicy. + | |
|===
@@ -88,6 +92,7 @@ _Appears In:_
| *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
| |
| *`spec`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clusterspec[$$ClusterSpec$$]__ | Spec defines the desired state of the Cluster. + | { } |
| *`status`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clusterstatus[$$ClusterStatus$$]__ | Status reflects the observed state of the Cluster. + | { } |
|===
@@ -202,6 +207,10 @@ Example: ["--node-name=my-agent-node"] + | |
| *`addons`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-addon[$$Addon$$] array__ | Addons specifies secrets containing raw YAML to deploy on cluster startup. + | |
| *`serverLimit`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core[$$ResourceList$$]__ | ServerLimit specifies resource limits for server nodes. + | |
| *`workerLimit`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core[$$ResourceList$$]__ | WorkerLimit specifies resource limits for agent nodes. + | |
| *`serverAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | ServerAffinity specifies the affinity rules for server pods. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`agentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | AgentAffinity specifies the affinity rules for agent pods. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`mirrorHostNodes`* __boolean__ | MirrorHostNodes controls whether node objects from the host cluster +
are mirrored into the virtual cluster. + | |
| *`customCAs`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-customcas[$$CustomCAs$$]__ | CustomCAs specifies the cert/key pairs for custom CA certificates. + | |
@@ -211,6 +220,36 @@ Each entry defines a secret and its mount path within the pods. + | |
|===
[id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clusterstatus"]
=== ClusterStatus
ClusterStatus reflects the observed state of a Cluster.
_Appears In:_
* xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-cluster[$$Cluster$$]
[cols="25a,55a,10a,10a", options="header"]
|===
| Field | Description | Default | Validation
| *`hostVersion`* __string__ | HostVersion is the Kubernetes version of the host node. + | |
| *`clusterCIDR`* __string__ | ClusterCIDR is the CIDR range for pod IPs. + | |
| *`serviceCIDR`* __string__ | ServiceCIDR is the CIDR range for service IPs. + | |
| *`clusterDNS`* __string__ | ClusterDNS is the IP address for the CoreDNS service. + | |
| *`tlsSANs`* __string array__ | TLSSANs specifies subject alternative names for the K3s server certificate. + | |
| *`policyName`* __string__ | PolicyName specifies the virtual cluster policy name bound to the virtual cluster. + | |
| *`policy`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-appliedpolicy[$$AppliedPolicy$$]__ | policy represents the status of the policy applied to this cluster. +
This field is set by the VirtualClusterPolicy controller. + | |
| *`kubeletPort`* __integer__ | KubeletPort specefies the port used by k3k-kubelet in shared mode. + | |
| *`webhookPort`* __integer__ | WebhookPort specefies the port used by webhook in k3k-kubelet in shared mode. + | |
| *`conditions`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta[$$Condition$$] array__ | Conditions are the individual conditions for the cluster set. + | |
| *`phase`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clusterphase[$$ClusterPhase$$]__ | Phase is a high-level summary of the cluster's current lifecycle state. + | Unknown | Enum: [Pending Provisioning Ready Failed Terminating Unknown] +
|===
[id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-configmapsyncconfig"]
@@ -680,6 +719,7 @@ _Appears In:_
| *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
| |
| *`spec`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-virtualclusterpolicyspec[$$VirtualClusterPolicySpec$$]__ | Spec defines the desired state of the VirtualClusterPolicy. + | { } |
| *`status`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-virtualclusterpolicystatus[$$VirtualClusterPolicyStatus$$]__ | Status reflects the observed state of the VirtualClusterPolicy. + | |
|===
@@ -726,6 +766,10 @@ _Appears In:_
to set defaults and constraints (min/max) + | |
| *`defaultNodeSelector`* __object (keys:string, values:string)__ | DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace. + | |
| *`defaultPriorityClass`* __string__ | DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. + | |
| *`defaultServerAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | DefaultServerAffinity specifies the affinity rules applied to server pods of all clusters in the target Namespace. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`defaultAgentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + | shared | Enum: [shared virtual] +
| *`disableNetworkPolicy`* __boolean__ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + | |
@@ -735,5 +779,26 @@ to set defaults and constraints (min/max) + | |
|===
[id="{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-virtualclusterpolicystatus"]
=== VirtualClusterPolicyStatus
VirtualClusterPolicyStatus reflects the observed state of a VirtualClusterPolicy.
_Appears In:_
* xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-virtualclusterpolicy[$$VirtualClusterPolicy$$]
[cols="25a,55a,10a,10a", options="header"]
|===
| Field | Description | Default | Validation
| *`observedGeneration`* __integer__ | ObservedGeneration was the generation at the time the status was updated. + | |
| *`lastUpdateTime`* __string__ | LastUpdate is the timestamp when the status was last updated. + | |
| *`summary`* __string__ | Summary is a summary of the status. + | |
| *`conditions`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta[$$Condition$$] array__ | Conditions are the individual conditions for the cluster set. + | |
|===

View File

@@ -48,6 +48,8 @@ _Appears in:_
| `name` _string_ | name is the name of the VirtualClusterPolicy currently applied to this cluster. | | MinLength: 1 <br /> |
| `priorityClass` _string_ | priorityClass is the priority class enforced by the active VirtualClusterPolicy. | | |
| `nodeSelector` _object (keys:string, values:string)_ | nodeSelector is a node selector enforced by the active VirtualClusterPolicy. | | |
| `serverAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | serverAffinity is the affinity rules for server pods enforced by the active VirtualClusterPolicy.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `agentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `sync` _[SyncConfig](#syncconfig)_ | sync is the SyncConfig enforced by the active VirtualClusterPolicy. | | |
@@ -70,6 +72,7 @@ _Appears in:_
| `kind` _string_ | `Cluster` | | |
| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | |
| `spec` _[ClusterSpec](#clusterspec)_ | Spec defines the desired state of the Cluster. | \{ \} | |
| `status` _[ClusterStatus](#clusterstatus)_ | Status reflects the observed state of the Cluster. | \{ \} | |
#### ClusterList
@@ -151,12 +154,38 @@ _Appears in:_
| `addons` _[Addon](#addon) array_ | Addons specifies secrets containing raw YAML to deploy on cluster startup. | | |
| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | |
| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | |
| `serverAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | ServerAffinity specifies the affinity rules for server pods.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `agentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | AgentAffinity specifies the affinity rules for agent pods.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `mirrorHostNodes` _boolean_ | MirrorHostNodes controls whether node objects from the host cluster<br />are mirrored into the virtual cluster. | | |
| `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. | | |
| `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | |
| `secretMounts` _[SecretMount](#secretmount) array_ | SecretMounts specifies a list of secrets to mount into server and agent pods.<br />Each entry defines a secret and its mount path within the pods. | | |
#### ClusterStatus
ClusterStatus reflects the observed state of a Cluster.
_Appears in:_
- [Cluster](#cluster)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `hostVersion` _string_ | HostVersion is the Kubernetes version of the host node. | | |
| `clusterCIDR` _string_ | ClusterCIDR is the CIDR range for pod IPs. | | |
| `serviceCIDR` _string_ | ServiceCIDR is the CIDR range for service IPs. | | |
| `clusterDNS` _string_ | ClusterDNS is the IP address for the CoreDNS service. | | |
| `tlsSANs` _string array_ | TLSSANs specifies subject alternative names for the K3s server certificate. | | |
| `policyName` _string_ | PolicyName specifies the virtual cluster policy name bound to the virtual cluster. | | |
| `policy` _[AppliedPolicy](#appliedpolicy)_ | policy represents the status of the policy applied to this cluster.<br />This field is set by the VirtualClusterPolicy controller. | | |
| `kubeletPort` _integer_ | KubeletPort specefies the port used by k3k-kubelet in shared mode. | | |
| `webhookPort` _integer_ | WebhookPort specefies the port used by webhook in k3k-kubelet in shared mode. | | |
| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta) array_ | Conditions are the individual conditions for the cluster set. | | |
| `phase` _[ClusterPhase](#clusterphase)_ | Phase is a high-level summary of the cluster's current lifecycle state. | Unknown | Enum: [Pending Provisioning Ready Failed Terminating Unknown] <br /> |
#### ConfigMapSyncConfig
@@ -513,6 +542,7 @@ _Appears in:_
| `kind` _string_ | `VirtualClusterPolicy` | | |
| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | |
| `spec` _[VirtualClusterPolicySpec](#virtualclusterpolicyspec)_ | Spec defines the desired state of the VirtualClusterPolicy. | \{ \} | |
| `status` _[VirtualClusterPolicyStatus](#virtualclusterpolicystatus)_ | Status reflects the observed state of the VirtualClusterPolicy. | | |
#### VirtualClusterPolicyList
@@ -550,11 +580,30 @@ _Appears in:_
| `limit` _[LimitRangeSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#limitrangespec-v1-core)_ | Limit specifies the LimitRange that will be applied to all pods within the VirtualClusterPolicy<br />to set defaults and constraints (min/max) | | |
| `defaultNodeSelector` _object (keys:string, values:string)_ | DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace. | | |
| `defaultPriorityClass` _string_ | DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. | | |
| `defaultServerAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultServerAffinity specifies the affinity rules applied to server pods of all clusters in the target Namespace.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `defaultAgentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". | shared | Enum: [shared virtual] <br /> |
| `disableNetworkPolicy` _boolean_ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. | | |
| `podSecurityAdmissionLevel` _[PodSecurityAdmissionLevel](#podsecurityadmissionlevel)_ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. | | Enum: [privileged baseline restricted] <br /> |
| `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | |
#### VirtualClusterPolicyStatus
VirtualClusterPolicyStatus reflects the observed state of a VirtualClusterPolicy.
_Appears in:_
- [VirtualClusterPolicy](#virtualclusterpolicy)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `observedGeneration` _integer_ | ObservedGeneration was the generation at the time the status was updated. | | |
| `lastUpdateTime` _string_ | LastUpdate is the timestamp when the status was last updated. | | |
| `summary` _string_ | Summary is a summary of the status. | | |
| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta) array_ | Conditions are the individual conditions for the cluster set. | | |

View File

@@ -169,6 +169,18 @@ type ClusterSpec struct {
// +optional
WorkerLimit corev1.ResourceList `json:"workerLimit,omitempty"`
// ServerAffinity specifies the affinity rules for server pods.
// This includes both node affinity and pod affinity/anti-affinity rules.
//
// +optional
ServerAffinity *corev1.Affinity `json:"serverAffinity,omitempty"`
// AgentAffinity specifies the affinity rules for agent pods.
// This includes both node affinity and pod affinity/anti-affinity rules.
//
// +optional
AgentAffinity *corev1.Affinity `json:"agentAffinity,omitempty"`
// MirrorHostNodes controls whether node objects from the host cluster
// are mirrored into the virtual cluster.
//
@@ -605,6 +617,18 @@ type AppliedPolicy struct {
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// serverAffinity is the affinity rules for server pods enforced by the active VirtualClusterPolicy.
// This includes both node affinity and pod affinity/anti-affinity rules.
//
// +optional
ServerAffinity *corev1.Affinity `json:"serverAffinity,omitempty"`
// agentAffinity is the affinity rules for agent pods enforced by the active VirtualClusterPolicy.
// This includes both node affinity and pod affinity/anti-affinity rules.
//
// +optional
AgentAffinity *corev1.Affinity `json:"agentAffinity,omitempty"`
// sync is the SyncConfig enforced by the active VirtualClusterPolicy.
//
// +optional
@@ -682,6 +706,18 @@ type VirtualClusterPolicySpec struct {
// +optional
DefaultPriorityClass string `json:"defaultPriorityClass,omitempty"`
// DefaultServerAffinity specifies the affinity rules applied to server pods of all clusters in the target Namespace.
// This includes both node affinity and pod affinity/anti-affinity rules.
//
// +optional
DefaultServerAffinity *corev1.Affinity `json:"defaultServerAffinity,omitempty"`
// DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace.
// This includes both node affinity and pod affinity/anti-affinity rules.
//
// +optional
DefaultAgentAffinity *corev1.Affinity `json:"defaultAgentAffinity,omitempty"`
// AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared".
//
// +kubebuilder:default=shared

View File

@@ -40,6 +40,16 @@ func (in *AppliedPolicy) DeepCopyInto(out *AppliedPolicy) {
(*out)[key] = val
}
}
if in.ServerAffinity != nil {
in, out := &in.ServerAffinity, &out.ServerAffinity
*out = new(v1.Affinity)
(*in).DeepCopyInto(*out)
}
if in.AgentAffinity != nil {
in, out := &in.AgentAffinity, &out.AgentAffinity
*out = new(v1.Affinity)
(*in).DeepCopyInto(*out)
}
if in.Sync != nil {
in, out := &in.Sync, &out.Sync
*out = new(SyncConfig)
@@ -195,6 +205,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
(*out)[key] = val.DeepCopy()
}
}
if in.ServerAffinity != nil {
in, out := &in.ServerAffinity, &out.ServerAffinity
*out = new(v1.Affinity)
(*in).DeepCopyInto(*out)
}
if in.AgentAffinity != nil {
in, out := &in.AgentAffinity, &out.AgentAffinity
*out = new(v1.Affinity)
(*in).DeepCopyInto(*out)
}
if in.CustomCAs != nil {
in, out := &in.CustomCAs, &out.CustomCAs
*out = new(CustomCAs)
@@ -706,6 +726,16 @@ func (in *VirtualClusterPolicySpec) DeepCopyInto(out *VirtualClusterPolicySpec)
(*out)[key] = val
}
}
if in.DefaultServerAffinity != nil {
in, out := &in.DefaultServerAffinity, &out.DefaultServerAffinity
*out = new(v1.Affinity)
(*in).DeepCopyInto(*out)
}
if in.DefaultAgentAffinity != nil {
in, out := &in.DefaultAgentAffinity, &out.DefaultAgentAffinity
*out = new(v1.Affinity)
(*in).DeepCopyInto(*out)
}
if in.PodSecurityAdmissionLevel != nil {
in, out := &in.PodSecurityAdmissionLevel, &out.PodSecurityAdmissionLevel
*out = new(PodSecurityAdmissionLevel)

View File

@@ -16,6 +16,7 @@ import (
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/k3k-kubelet/translate"
@@ -142,7 +143,7 @@ func (s *SharedAgent) daemonset(ctx context.Context) error {
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: s.podSpec(),
Spec: s.podSpec(ctx),
},
},
}
@@ -150,7 +151,9 @@ func (s *SharedAgent) daemonset(ctx context.Context) error {
return s.ensureObject(ctx, deploy)
}
func (s *SharedAgent) podSpec() v1.PodSpec {
func (s *SharedAgent) podSpec(ctx context.Context) v1.PodSpec {
log := ctrl.LoggerFrom(ctx)
hostNetwork := false
dnsPolicy := v1.DNSClusterFirst
@@ -165,7 +168,15 @@ func (s *SharedAgent) podSpec() v1.PodSpec {
image = s.imageRegistry + "/" + s.image
}
// Use the agent affinity from the policy status if it exists, otherwise fall back to the spec.
agentAffinity := s.cluster.Spec.AgentAffinity
if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.AgentAffinity != nil {
log.V(1).Info("Using agent affinity from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)
agentAffinity = s.cluster.Status.Policy.AgentAffinity
}
podSpec := v1.PodSpec{
Affinity: agentAffinity,
HostNetwork: hostNetwork,
DNSPolicy: dnsPolicy,
ServiceAccountName: s.Name(),

View File

@@ -10,6 +10,7 @@ import (
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/pkg/controller"
@@ -99,7 +100,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
"mode": "virtual",
},
}
podSpec := v.podSpec(image, name)
podSpec := v.podSpec(ctx, image, name)
if len(v.cluster.Spec.SecretMounts) > 0 {
vols, volMounts := mounts.BuildSecretsMountsVolumes(v.cluster.Spec.SecretMounts, "agent")
@@ -134,13 +135,22 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
return v.ensureObject(ctx, deployment)
}
func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec {
func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) v1.PodSpec {
log := ctrl.LoggerFrom(ctx)
var limit v1.ResourceList
args := v.cluster.Spec.AgentArgs
args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
// Use the agent affinity from the policy status if it exists, otherwise fall back to the spec.
agentAffinity := v.cluster.Spec.AgentAffinity
if v.cluster.Status.Policy != nil && v.cluster.Status.Policy.AgentAffinity != nil {
log.V(1).Info("Using agent affinity from policy", "policyName", v.cluster.Status.PolicyName, "clusterName", v.cluster.Name)
agentAffinity = v.cluster.Status.Policy.AgentAffinity
}
podSpec := v1.PodSpec{
Affinity: agentAffinity,
NodeSelector: v.cluster.Spec.NodeSelector,
Volumes: []v1.Volume{
{

View File

@@ -17,6 +17,7 @@ import (
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
@@ -53,8 +54,18 @@ func New(cluster *v1beta1.Cluster, client client.Client, token, image, imagePull
}
}
func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec {
func (s *Server) podSpec(ctx context.Context, image, name string, persistent bool, startupCmd string) v1.PodSpec {
log := ctrl.LoggerFrom(ctx)
// Use the server affinity from the policy status if it exists, otherwise fall back to the spec.
serverAffinity := s.cluster.Spec.ServerAffinity
if s.cluster.Status.Policy != nil && s.cluster.Status.Policy.ServerAffinity != nil {
log.V(1).Info("Using server affinity from policy", "policyName", s.cluster.Status.PolicyName, "clusterName", s.cluster.Name)
serverAffinity = s.cluster.Status.Policy.ServerAffinity
}
podSpec := v1.PodSpec{
Affinity: serverAffinity,
NodeSelector: s.cluster.Spec.NodeSelector,
PriorityClassName: s.cluster.Spec.PriorityClass,
Volumes: []v1.Volume{
@@ -321,7 +332,7 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
return nil, err
}
podSpec := s.podSpec(image, name, persistent, startupCommand)
podSpec := s.podSpec(ctx, image, name, persistent, startupCommand)
podSpec.Volumes = append(podSpec.Volumes, volumes...)
podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)

View File

@@ -473,10 +473,12 @@ func (c *VirtualClusterPolicyReconciler) reconcileClusters(ctx context.Context,
origStatus := cluster.Status.DeepCopy()
cluster.Status.Policy = &v1beta1.AppliedPolicy{
Name: policy.Name,
PriorityClass: &policy.Spec.DefaultPriorityClass,
NodeSelector: policy.Spec.DefaultNodeSelector,
Sync: policy.Spec.Sync,
Name: policy.Name,
PriorityClass: &policy.Spec.DefaultPriorityClass,
NodeSelector: policy.Spec.DefaultNodeSelector,
Sync: policy.Spec.Sync,
ServerAffinity: policy.Spec.DefaultServerAffinity,
AgentAffinity: policy.Spec.DefaultAgentAffinity,
}
if !reflect.DeepEqual(origStatus, &cluster.Status) {

View File

@@ -469,6 +469,117 @@ var _ = Describe("VirtualClusterPolicy Controller", Label("controller"), Label("
Should(Succeed())
})
It("updates the cluster policy status with the DefaultServerAffinity and DefaultAgentAffinity", func() {
serverAffinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
{Weight: 10},
},
},
}
agentAffinity := serverAffinity.DeepCopy()
agentAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 20
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
DefaultServerAffinity: serverAffinity,
DefaultAgentAffinity: agentAffinity,
})
bindPolicyToNamespace(namespace, policy)
err := k8sClient.Update(ctx, policy)
Expect(err).To(Not(HaveOccurred()))
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace.Name,
},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
Servers: ptr.To[int32](1),
Agents: ptr.To[int32](0),
},
}
err = k8sClient.Create(ctx, cluster)
Expect(err).To(Not(HaveOccurred()))
// wait a bit
Eventually(func(g Gomega) {
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
err = k8sClient.Get(ctx, key, cluster)
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(cluster.Spec.AgentAffinity).To(BeNil())
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
g.Expect(cluster.Status.Policy.AgentAffinity).To(Not(BeNil()))
g.Expect(cluster.Status.Policy.AgentAffinity).To(Equal(agentAffinity))
g.Expect(cluster.Spec.ServerAffinity).To(BeNil())
g.Expect(cluster.Status.Policy.ServerAffinity).To(Not(BeNil()))
g.Expect(cluster.Status.Policy.ServerAffinity).To(Equal(serverAffinity))
}).
WithTimeout(time.Second * 10).
WithPolling(time.Second).
Should(Succeed())
})
It("overrides the cluster ServerAffinity and AgentAffinity with the DefaultServerAffinity and DefaultAgentAffinity from the policy", func() {
serverAffinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
{Weight: 10},
},
},
}
agentAffinity := serverAffinity.DeepCopy()
agentAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 20
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
DefaultServerAffinity: serverAffinity,
DefaultAgentAffinity: agentAffinity,
})
bindPolicyToNamespace(namespace, policy)
err := k8sClient.Update(ctx, policy)
Expect(err).To(Not(HaveOccurred()))
// Cluster values that will get overwritten by the policy in the cluster status
clusterAgentAffinity := agentAffinity.DeepCopy()
clusterAgentAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 30
clusterServerAffinity := serverAffinity.DeepCopy()
clusterServerAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 40
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace.Name,
},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
Servers: ptr.To[int32](1),
Agents: ptr.To[int32](0),
AgentAffinity: clusterAgentAffinity,
ServerAffinity: clusterServerAffinity,
},
}
err = k8sClient.Create(ctx, cluster)
Expect(err).To(Not(HaveOccurred()))
// wait a bit
Eventually(func(g Gomega) {
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
err = k8sClient.Get(ctx, key, cluster)
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(cluster.Spec.AgentAffinity).To(Equal(clusterAgentAffinity))
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
g.Expect(cluster.Status.Policy.AgentAffinity).To(Not(BeNil()))
g.Expect(cluster.Status.Policy.AgentAffinity).To(Equal(agentAffinity))
g.Expect(cluster.Spec.ServerAffinity).To(Equal(clusterServerAffinity))
g.Expect(cluster.Status.Policy.ServerAffinity).To(Not(BeNil()))
g.Expect(cluster.Status.Policy.ServerAffinity).To(Equal(serverAffinity))
}).
WithTimeout(time.Second * 10).
WithPolling(time.Second).
Should(Succeed())
})
It("should create a ResourceQuota if Quota is enabled", func() {
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
Quota: &v1.ResourceQuotaSpec{