mirror of
https://github.com/rancher/k3k.git
synced 2026-05-19 15:56:44 +00:00
737dcf998e
* added policy finalizer * check with a small timeout * comments * removed requeue, renamed log
888 lines
30 KiB
Go
888 lines
30 KiB
Go
package policy_test
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
networkingv1 "k8s.io/api/networking/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
|
k3kcontroller "github.com/rancher/k3k/pkg/controller"
|
|
"github.com/rancher/k3k/pkg/controller/policy"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("VirtualClusterPolicy Controller", Label("controller"), Label("VirtualClusterPolicy"), func() {
|
|
Context("creating a VirtualClusterPolicy", func() {
|
|
It("should have the 'shared' allowedMode", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{})
|
|
Expect(policy.Spec.AllowedMode).To(Equal(v1beta1.SharedClusterMode))
|
|
})
|
|
|
|
It("should have the 'virtual' mode if specified", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
AllowedMode: v1beta1.VirtualClusterMode,
|
|
})
|
|
|
|
Expect(policy.Spec.AllowedMode).To(Equal(v1beta1.VirtualClusterMode))
|
|
})
|
|
|
|
It("should fail for a non-existing mode", func() {
|
|
policy := &v1beta1.VirtualClusterPolicy{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "policy-",
|
|
},
|
|
Spec: v1beta1.VirtualClusterPolicySpec{
|
|
AllowedMode: v1beta1.ClusterMode("non-existing"),
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, policy)
|
|
Expect(err).To(HaveOccurred())
|
|
})
|
|
|
|
When("bound to a namespace", func() {
|
|
var namespace *corev1.Namespace
|
|
|
|
BeforeEach(func() {
|
|
namespace = &corev1.Namespace{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "ns-",
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, namespace)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
})
|
|
|
|
It("should create a NetworkPolicy", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{})
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
// look for network policies etc
|
|
networkPolicy := &networkingv1.NetworkPolicy{}
|
|
|
|
Eventually(func() error {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
|
|
return k8sClient.Get(ctx, key, networkPolicy)
|
|
}).
|
|
WithTimeout(time.Minute).
|
|
WithPolling(time.Second).
|
|
Should(BeNil())
|
|
|
|
spec := networkPolicy.Spec
|
|
Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress))
|
|
Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress))
|
|
|
|
// ingress should allow everything
|
|
Expect(spec.Ingress).To(ConsistOf(networkingv1.NetworkPolicyIngressRule{}))
|
|
|
|
// egress should contains some rules
|
|
Expect(spec.Egress).To(HaveLen(1))
|
|
|
|
// allow networking to all external IPs
|
|
ipBlockRule := networkingv1.NetworkPolicyPeer{
|
|
IPBlock: &networkingv1.IPBlock{CIDR: "0.0.0.0/0"},
|
|
}
|
|
|
|
// allow networking in the same namespace
|
|
namespaceRule := networkingv1.NetworkPolicyPeer{
|
|
NamespaceSelector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace.Name},
|
|
},
|
|
}
|
|
|
|
// allow networking to the "kube-dns" pod in the "kube-system" namespace
|
|
kubeDNSRule := networkingv1.NetworkPolicyPeer{
|
|
PodSelector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"k8s-app": "kube-dns"},
|
|
},
|
|
NamespaceSelector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"},
|
|
},
|
|
}
|
|
|
|
Expect(spec.Egress[0].To).To(ContainElements(
|
|
ipBlockRule, namespaceRule, kubeDNSRule,
|
|
))
|
|
})
|
|
|
|
It("should recreate the NetworkPolicy if deleted", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{})
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
// look for network policy
|
|
networkPolicy := &networkingv1.NetworkPolicy{}
|
|
|
|
Eventually(func() error {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
|
|
return k8sClient.Get(context.Background(), key, networkPolicy)
|
|
}).
|
|
WithTimeout(time.Minute).
|
|
WithPolling(time.Second).
|
|
Should(BeNil())
|
|
|
|
err := k8sClient.Delete(ctx, networkPolicy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
err = k8sClient.Get(ctx, key, networkPolicy)
|
|
Expect(apierrors.IsNotFound(err)).Should(BeTrue())
|
|
|
|
// wait a bit for the network policy to being recreated
|
|
Eventually(func() error {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
|
|
return k8sClient.Get(ctx, key, networkPolicy)
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeNil())
|
|
})
|
|
|
|
It("should add and update the proper pod-security labels to the namespace", func() {
|
|
var (
|
|
privileged = v1beta1.PrivilegedPodSecurityAdmissionLevel
|
|
baseline = v1beta1.BaselinePodSecurityAdmissionLevel
|
|
restricted = v1beta1.RestrictedPodSecurityAdmissionLevel
|
|
)
|
|
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
PodSecurityAdmissionLevel: &privileged,
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
var ns corev1.Namespace
|
|
|
|
// Check privileged
|
|
|
|
// wait a bit for the namespace to be updated
|
|
Eventually(func() string {
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
return ns.Labels["pod-security.kubernetes.io/enforce"]
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Equal("privileged"))
|
|
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "privileged"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn-version")))
|
|
|
|
// Check baseline
|
|
|
|
// get policy again
|
|
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(policy), policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
policy.Spec.PodSecurityAdmissionLevel = &baseline
|
|
err = k8sClient.Update(ctx, policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit for the namespace to be updated
|
|
Eventually(func() string {
|
|
err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
return ns.Labels["pod-security.kubernetes.io/enforce"]
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Equal("baseline"))
|
|
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "baseline"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn", "baseline"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn-version", "latest"))
|
|
|
|
// Check restricted
|
|
|
|
policy.Spec.PodSecurityAdmissionLevel = &restricted
|
|
err = k8sClient.Update(ctx, policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit for the namespace to be updated
|
|
Eventually(func() string {
|
|
err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
return ns.Labels["pod-security.kubernetes.io/enforce"]
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Equal("restricted"))
|
|
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "restricted"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn", "restricted"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn-version", "latest"))
|
|
|
|
// check cleanup
|
|
|
|
policy.Spec.PodSecurityAdmissionLevel = nil
|
|
err = k8sClient.Update(ctx, policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit for the namespace to be updated
|
|
Eventually(func() bool {
|
|
err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
_, found := ns.Labels["pod-security.kubernetes.io/enforce"]
|
|
|
|
return found
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeFalse())
|
|
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/enforce")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/enforce-version")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn-version")))
|
|
})
|
|
|
|
It("should restore the labels if Namespace is updated", func() {
|
|
privileged := v1beta1.PrivilegedPodSecurityAdmissionLevel
|
|
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
PodSecurityAdmissionLevel: &privileged,
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
var ns corev1.Namespace
|
|
|
|
// wait a bit for the namespace to be updated
|
|
Eventually(func() bool {
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
|
|
|
|
return enforceValue == "privileged"
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeTrue())
|
|
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "privileged"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
|
|
|
|
ns.Labels["pod-security.kubernetes.io/enforce"] = "baseline"
|
|
err := k8sClient.Update(ctx, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit for the namespace to be restored
|
|
Eventually(func() bool {
|
|
err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
|
|
|
|
return enforceValue == "privileged"
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeTrue())
|
|
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "privileged"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
|
|
})
|
|
|
|
It("updates the Cluster's policy status with the DefaultPriorityClass", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
DefaultPriorityClass: "foobar",
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
cluster := &v1beta1.Cluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "cluster-",
|
|
Namespace: namespace.Name,
|
|
},
|
|
Spec: v1beta1.ClusterSpec{
|
|
Mode: v1beta1.SharedClusterMode,
|
|
Servers: ptr.To[int32](1),
|
|
Agents: ptr.To[int32](0),
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Spec.PriorityClass).To(BeEmpty())
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.PriorityClass).To(Not(BeNil()))
|
|
g.Expect(*cluster.Status.Policy.PriorityClass).To(Equal(policy.Spec.DefaultPriorityClass))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
})
|
|
|
|
It("updates the Cluster's policy status with the DefaultNodeSelector", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
DefaultNodeSelector: map[string]string{"label-1": "value-1"},
|
|
})
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
cluster := &v1beta1.Cluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "cluster-",
|
|
Namespace: namespace.Name,
|
|
},
|
|
Spec: v1beta1.ClusterSpec{
|
|
Mode: v1beta1.SharedClusterMode,
|
|
Servers: ptr.To[int32](1),
|
|
Agents: ptr.To[int32](0),
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Spec.NodeSelector).To(BeEmpty())
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.NodeSelector).To(Equal(map[string]string{"label-1": "value-1"}))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
})
|
|
|
|
It("updates the Cluster's policy status when the VCP nodeSelector changes", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
DefaultNodeSelector: map[string]string{"label-1": "value-1"},
|
|
})
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
cluster := &v1beta1.Cluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "cluster-",
|
|
Namespace: namespace.Name,
|
|
},
|
|
Spec: v1beta1.ClusterSpec{
|
|
Mode: v1beta1.SharedClusterMode,
|
|
Servers: ptr.To[int32](1),
|
|
Agents: ptr.To[int32](0),
|
|
NodeSelector: map[string]string{"label-1": "value-1"},
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// Cluster Spec should not change, VCP NodeSelector should be present in the Status
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Spec.NodeSelector).To(Equal(map[string]string{"label-1": "value-1"}))
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.NodeSelector).To(Equal(map[string]string{"label-1": "value-1"}))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// update the VirtualClusterPolicy
|
|
err = k8sClient.Get(ctx, client.ObjectKeyFromObject(policy), policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
policy.Spec.DefaultNodeSelector["label-2"] = "value-2"
|
|
err = k8sClient.Update(ctx, policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Spec.NodeSelector).To(Equal(map[string]string{"label-1": "value-1"}))
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.NodeSelector).To(Equal(map[string]string{"label-1": "value-1", "label-2": "value-2"}))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Update the Cluster
|
|
err = k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
cluster.Spec.NodeSelector["label-3"] = "value-3"
|
|
err = k8sClient.Update(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit and check it's restored
|
|
Consistently(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
g.Expect(cluster.Spec.NodeSelector).To(Equal(map[string]string{"label-1": "value-1", "label-3": "value-3"}))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
})
|
|
|
|
It("updates the cluster policy status with the DefaultServerAffinity and DefaultAgentAffinity", func() {
|
|
serverAffinity := &corev1.Affinity{
|
|
NodeAffinity: &corev1.NodeAffinity{
|
|
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
|
|
{Weight: 10},
|
|
},
|
|
},
|
|
}
|
|
agentAffinity := serverAffinity.DeepCopy()
|
|
agentAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 20
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
DefaultServerAffinity: serverAffinity,
|
|
DefaultAgentAffinity: agentAffinity,
|
|
})
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
cluster := &v1beta1.Cluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "cluster-",
|
|
Namespace: namespace.Name,
|
|
},
|
|
Spec: v1beta1.ClusterSpec{
|
|
Mode: v1beta1.SharedClusterMode,
|
|
Servers: ptr.To[int32](1),
|
|
Agents: ptr.To[int32](0),
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Spec.AgentAffinity).To(BeNil())
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.AgentAffinity).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.AgentAffinity).To(Equal(agentAffinity))
|
|
|
|
g.Expect(cluster.Spec.ServerAffinity).To(BeNil())
|
|
g.Expect(cluster.Status.Policy.ServerAffinity).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.ServerAffinity).To(Equal(serverAffinity))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
})
|
|
|
|
It("overrides the cluster ServerAffinity and AgentAffinity with the DefaultServerAffinity and DefaultAgentAffinity from the policy", func() {
|
|
serverAffinity := &corev1.Affinity{
|
|
NodeAffinity: &corev1.NodeAffinity{
|
|
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
|
|
{Weight: 10},
|
|
},
|
|
},
|
|
}
|
|
agentAffinity := serverAffinity.DeepCopy()
|
|
agentAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 20
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
DefaultServerAffinity: serverAffinity,
|
|
DefaultAgentAffinity: agentAffinity,
|
|
})
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
// Cluster values that will get overwritten by the policy in the cluster status
|
|
clusterAgentAffinity := agentAffinity.DeepCopy()
|
|
clusterAgentAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 30
|
|
clusterServerAffinity := serverAffinity.DeepCopy()
|
|
clusterServerAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 40
|
|
cluster := &v1beta1.Cluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "cluster-",
|
|
Namespace: namespace.Name,
|
|
},
|
|
Spec: v1beta1.ClusterSpec{
|
|
Mode: v1beta1.SharedClusterMode,
|
|
Servers: ptr.To[int32](1),
|
|
Agents: ptr.To[int32](0),
|
|
AgentAffinity: clusterAgentAffinity,
|
|
ServerAffinity: clusterServerAffinity,
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait a bit
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Spec.AgentAffinity).To(Equal(clusterAgentAffinity))
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.AgentAffinity).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.AgentAffinity).To(Equal(agentAffinity))
|
|
|
|
g.Expect(cluster.Spec.ServerAffinity).To(Equal(clusterServerAffinity))
|
|
g.Expect(cluster.Status.Policy.ServerAffinity).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.ServerAffinity).To(Equal(serverAffinity))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
})
|
|
|
|
It("should create a ResourceQuota if Quota is enabled", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
Quota: &corev1.ResourceQuotaSpec{
|
|
Hard: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("800m"),
|
|
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
|
},
|
|
},
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
var resourceQuota corev1.ResourceQuota
|
|
|
|
Eventually(func() error {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
|
|
return k8sClient.Get(ctx, key, &resourceQuota)
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeNil())
|
|
Expect(resourceQuota.Spec.Hard.Cpu().String()).To(BeEquivalentTo("800m"))
|
|
Expect(resourceQuota.Spec.Hard.Memory().String()).To(BeEquivalentTo("1Gi"))
|
|
})
|
|
|
|
It("should delete the ResourceQuota if Quota is deleted", func() {
|
|
policy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
Quota: &corev1.ResourceQuotaSpec{
|
|
Hard: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("800m"),
|
|
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
|
},
|
|
},
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, policy)
|
|
|
|
var resourceQuota corev1.ResourceQuota
|
|
|
|
Eventually(func() error {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
|
|
return k8sClient.Get(ctx, key, &resourceQuota)
|
|
}).
|
|
WithTimeout(time.Minute).
|
|
WithPolling(time.Second).
|
|
Should(BeNil())
|
|
|
|
// get policy again
|
|
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(policy), policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
policy.Spec.Quota = nil
|
|
err = k8sClient.Update(ctx, policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait for a bit for the resourceQuota to be deleted
|
|
Eventually(func() bool {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(policy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
err := k8sClient.Get(ctx, key, &resourceQuota)
|
|
|
|
return apierrors.IsNotFound(err)
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeTrue())
|
|
})
|
|
|
|
It("should delete the ResourceQuota if unbound", func() {
|
|
clusterPolicy := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
Quota: &corev1.ResourceQuotaSpec{
|
|
Hard: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("800m"),
|
|
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
|
},
|
|
},
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, clusterPolicy)
|
|
|
|
var resourceQuota corev1.ResourceQuota
|
|
|
|
Eventually(func() error {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterPolicy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
|
|
return k8sClient.Get(ctx, key, &resourceQuota)
|
|
}).
|
|
WithTimeout(time.Minute).
|
|
WithPolling(time.Second).
|
|
Should(BeNil())
|
|
|
|
delete(namespace.Labels, policy.PolicyNameLabelKey)
|
|
err := k8sClient.Update(ctx, namespace)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// wait for a bit for the resourceQuota to be deleted
|
|
Eventually(func() bool {
|
|
key := types.NamespacedName{
|
|
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterPolicy.Name),
|
|
Namespace: namespace.Name,
|
|
}
|
|
err := k8sClient.Get(ctx, key, &resourceQuota)
|
|
|
|
return apierrors.IsNotFound(err)
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(BeTrue())
|
|
})
|
|
|
|
It("should clean up namespace labels and cluster status when policy is deleted", func() {
|
|
baseline := v1beta1.BaselinePodSecurityAdmissionLevel
|
|
|
|
vcp := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
PodSecurityAdmissionLevel: &baseline,
|
|
DefaultPriorityClass: "test-priority",
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, vcp)
|
|
|
|
// Create a cluster in the namespace
|
|
cluster := &v1beta1.Cluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "cluster-",
|
|
Namespace: namespace.Name,
|
|
},
|
|
Spec: v1beta1.ClusterSpec{
|
|
Mode: v1beta1.SharedClusterMode,
|
|
Servers: ptr.To[int32](1),
|
|
Agents: ptr.To[int32](0),
|
|
},
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, cluster)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
var ns corev1.Namespace
|
|
|
|
// Verify namespace has policy label and PSA labels
|
|
Eventually(func(g Gomega) {
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(ns.Labels).Should(HaveKeyWithValue(policy.PolicyNameLabelKey, vcp.Name))
|
|
g.Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "baseline"))
|
|
g.Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Verify cluster has policy status set
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Status.PolicyName).To(Equal(vcp.Name))
|
|
|
|
g.Expect(cluster.Status.Policy).To(Not(BeNil()))
|
|
g.Expect(cluster.Status.Policy.PriorityClass).To(Not(BeNil()))
|
|
g.Expect(*cluster.Status.Policy.PriorityClass).To(Equal("test-priority"))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Delete the policy
|
|
err = k8sClient.Delete(ctx, vcp)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// Verify policy label is removed from namespace
|
|
Eventually(func(g Gomega) {
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
g.Expect(ns.Labels).Should(Not(HaveKey(policy.PolicyNameLabelKey)))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Verify PSA labels are removed from namespace (since policy set them)
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/enforce")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/enforce-version")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn")))
|
|
Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn-version")))
|
|
|
|
// Verify Policy cleared from cluster status
|
|
Eventually(func(g Gomega) {
|
|
key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
|
|
err = k8sClient.Get(ctx, key, cluster)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
|
|
g.Expect(cluster.Status.PolicyName).To(BeEmpty())
|
|
g.Expect(cluster.Status.Policy).To(BeNil())
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Verify policy is actually deleted
|
|
Eventually(func() bool {
|
|
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(vcp), vcp)
|
|
return apierrors.IsNotFound(err)
|
|
}).
|
|
WithTimeout(time.Minute).
|
|
WithPolling(time.Second).
|
|
Should(BeTrue())
|
|
})
|
|
|
|
It("should NOT remove PSA labels if policy did not set them", func() {
|
|
// Create policy without PSA labels
|
|
vcp := newPolicy(v1beta1.VirtualClusterPolicySpec{
|
|
DefaultPriorityClass: "test-priority",
|
|
})
|
|
|
|
bindPolicyToNamespace(namespace, vcp)
|
|
|
|
var ns corev1.Namespace
|
|
|
|
// Manually set PSA labels on namespace
|
|
Eventually(func(g Gomega) {
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
g.Expect(ns.Labels).Should(HaveKeyWithValue(policy.PolicyNameLabelKey, vcp.Name))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Add PSA labels manually
|
|
ns.Labels["pod-security.kubernetes.io/enforce"] = "restricted"
|
|
ns.Labels["pod-security.kubernetes.io/enforce-version"] = "v1.28"
|
|
|
|
err := k8sClient.Update(ctx, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// Verify PSA labels are present
|
|
err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "restricted"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "v1.28"))
|
|
|
|
// Delete the policy
|
|
err = k8sClient.Delete(ctx, vcp)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
// Verify policy label is removed
|
|
Eventually(func(g Gomega) {
|
|
err := k8sClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, &ns)
|
|
g.Expect(err).To(Not(HaveOccurred()))
|
|
g.Expect(ns.Labels).Should(Not(HaveKey(policy.PolicyNameLabelKey)))
|
|
}).
|
|
WithTimeout(time.Second * 10).
|
|
WithPolling(time.Second).
|
|
Should(Succeed())
|
|
|
|
// Verify PSA labels are NOT removed (since policy didn't set them)
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "restricted"))
|
|
Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "v1.28"))
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
func newPolicy(spec v1beta1.VirtualClusterPolicySpec) *v1beta1.VirtualClusterPolicy {
|
|
GinkgoHelper()
|
|
|
|
policy := &v1beta1.VirtualClusterPolicy{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "policy-",
|
|
},
|
|
Spec: spec,
|
|
}
|
|
|
|
err := k8sClient.Create(ctx, policy)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
|
|
return policy
|
|
}
|
|
|
|
func bindPolicyToNamespace(namespace *corev1.Namespace, pol *v1beta1.VirtualClusterPolicy) {
|
|
GinkgoHelper()
|
|
|
|
if len(namespace.Labels) == 0 {
|
|
namespace.Labels = map[string]string{}
|
|
}
|
|
|
|
namespace.Labels[policy.PolicyNameLabelKey] = pol.Name
|
|
|
|
err := k8sClient.Update(ctx, namespace)
|
|
Expect(err).To(Not(HaveOccurred()))
|
|
}
|