mirror of
https://github.com/kubernetes-sigs/descheduler.git
synced 2026-08-23 22:46:35 +00:00
fix(e2e): harden test suite isolation, resource usage, and leader election timing
- Isolate TestLiveMigrationInBackground into dedicated e2e-livemigration namespace with t.Cleanup teardown. - Switch VMI containerdisk image from fedora (1.2GB, 512MB RAM) to cirros-container-disk-demo (12MB, 128MB RAM), reducing Kind memory pressure by 75%. - Filter pod list API calls with LabelSelector=kubevirt.io=virt-launcher to reduce API server serialization load. - Override --descheduling-interval to 3s in startDeschedulerServer (e2e_leaderelection_test.go) so leader descheduler re-evaluates pod lifetime periodically. - Exclude context cancellation/deadline from isClientRateLimiterError in e2e_test.go so context deadlines are not swallowed. - Add t.Cleanup and IsAlreadyExists handling to prevent leftover resources from causing cascading failures. Signed-off-by: Amir Alavi <amiralavi7@gmail.com>
This commit is contained in:
@@ -235,8 +235,10 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate
|
||||
if dryRun {
|
||||
if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
for _, obj := range objects {
|
||||
// Only check for nodes - secrets are handled by namespacedSharedInformerFactory
|
||||
if _, ok := obj.(*v1.Node); !ok {
|
||||
// Only check for nodes and pods - secrets are handled by namespacedSharedInformerFactory
|
||||
switch obj.(type) {
|
||||
case *v1.Node, *v1.Pod:
|
||||
default:
|
||||
continue
|
||||
}
|
||||
exists, err := descheduler.kubeClientSandbox.hasRuntimeObjectInIndexer(obj)
|
||||
@@ -256,7 +258,7 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate
|
||||
}
|
||||
return true, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("nodes did not propagate to the indexer: %v", err)
|
||||
t.Fatalf("objects did not propagate to the indexer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -189,24 +189,7 @@ func TestRemoveDuplicates(t *testing.T) {
|
||||
tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{
|
||||
Include: []string{testNamespace.Name},
|
||||
}
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs))
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
|
||||
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
}()
|
||||
createPolicyConfigMap(t, ctx, clientSet, removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs))
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
|
||||
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
|
||||
|
||||
@@ -34,12 +34,16 @@ import (
|
||||
|
||||
const (
|
||||
vmiCount = 3
|
||||
// virtLauncherSelector selects KubeVirt virt-launcher pods only,
|
||||
// avoiding listing unrelated pods in the namespace.
|
||||
virtLauncherSelector = "kubevirt.io=virt-launcher"
|
||||
)
|
||||
|
||||
func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance {
|
||||
func virtualMachineInstance(idx int, namespace string) *kvcorev1.VirtualMachineInstance {
|
||||
return &kvcorev1.VirtualMachineInstance{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("kubevirtvmi-%v", idx),
|
||||
Name: fmt.Sprintf("kubevirtvmi-%v", idx),
|
||||
Namespace: namespace,
|
||||
Annotations: map[string]string{
|
||||
"descheduler.alpha.kubernetes.io/request-evict-only": "",
|
||||
},
|
||||
@@ -58,20 +62,12 @@ func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "cloudinitdisk",
|
||||
DiskDevice: kvcorev1.DiskDevice{
|
||||
Disk: &kvcorev1.DiskTarget{
|
||||
Bus: kvcorev1.DiskBusVirtio,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Rng: &kvcorev1.Rng{},
|
||||
},
|
||||
Resources: kvcorev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceMemory: resource.MustParse("1024M"),
|
||||
corev1.ResourceMemory: resource.MustParse("128M"),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -81,27 +77,7 @@ func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance {
|
||||
Name: "containerdisk",
|
||||
VolumeSource: kvcorev1.VolumeSource{
|
||||
ContainerDisk: &kvcorev1.ContainerDiskSource{
|
||||
Image: "quay.io/kubevirt/fedora-with-test-tooling-container-disk:20240710_1265d1090",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "cloudinitdisk",
|
||||
VolumeSource: kvcorev1.VolumeSource{
|
||||
CloudInitNoCloud: &kvcorev1.CloudInitNoCloudSource{
|
||||
UserData: `#cloud-config
|
||||
password: fedora
|
||||
chpasswd: { expire: False }
|
||||
packages:
|
||||
- nginx
|
||||
runcmd:
|
||||
- [ "systemctl", "enable", "--now", "nginx" ]`,
|
||||
NetworkData: `version: 2
|
||||
ethernets:
|
||||
eth0:
|
||||
addresses: [ fd10:0:2::2/120 ]
|
||||
dhcp4: true
|
||||
gateway6: fd10:0:2::1`,
|
||||
Image: "quay.io/kubevirt/cirros-container-disk-demo:v1.8.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -110,6 +86,91 @@ ethernets:
|
||||
}
|
||||
}
|
||||
|
||||
func formatContainerStatuses(pod *corev1.Pod) string {
|
||||
var parts []string
|
||||
formatList := func(prefix string, statuses []corev1.ContainerStatus) {
|
||||
for _, cs := range statuses {
|
||||
stateStr := "unknown"
|
||||
if cs.State.Waiting != nil {
|
||||
stateStr = fmt.Sprintf("waiting(reason=%q, message=%q)", cs.State.Waiting.Reason, cs.State.Waiting.Message)
|
||||
} else if cs.State.Terminated != nil {
|
||||
stateStr = fmt.Sprintf("terminated(exitCode=%d, reason=%q, message=%q)", cs.State.Terminated.ExitCode, cs.State.Terminated.Reason, cs.State.Terminated.Message)
|
||||
} else if cs.State.Running != nil {
|
||||
stateStr = "running"
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s%s: state=%s, ready=%v, restarts=%d", prefix, cs.Name, stateStr, cs.Ready, cs.RestartCount))
|
||||
}
|
||||
}
|
||||
formatList("init:", pod.Status.InitContainerStatuses)
|
||||
formatList("", pod.Status.ContainerStatuses)
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable
|
||||
// condition with status True. If a VMI fails to become migratable within the
|
||||
// per-attempt timeout, it is deleted and recreated. This works around an
|
||||
// upstream KubeVirt race where virt-handler computes the containerdisk
|
||||
// checksum before the disk socket is ready, fails, and never retries; the
|
||||
// recreated VMI lands on a node that already has the containerdisk image
|
||||
// cached, so the socket comes up before virt-handler's first attempt.
|
||||
// See https://github.com/kubernetes-sigs/descheduler/pull/1874 for context.
|
||||
func ensureVMIsLiveMigratable(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface, namespace string) {
|
||||
t.Helper()
|
||||
const (
|
||||
maxAttempts = 3
|
||||
perAttemptWait = 120 * time.Second
|
||||
deleteWait = 60 * time.Second
|
||||
)
|
||||
|
||||
isLiveMigratable := func(vmi *kvcorev1.VirtualMachineInstance) bool {
|
||||
for _, c := range vmi.Status.Conditions {
|
||||
if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 1; i <= vmiCount; i++ {
|
||||
name := fmt.Sprintf("kubevirtvmi-%v", i)
|
||||
var lastVMI *kvcorev1.VirtualMachineInstance
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
err := wait.PollUntilContextTimeout(ctx, 5*time.Second, perAttemptWait, true, func(ctx context.Context) (bool, error) {
|
||||
vmi, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
klog.Infof("Unable to get vmi %v: %v", name, err)
|
||||
return false, nil
|
||||
}
|
||||
lastVMI = vmi
|
||||
return isLiveMigratable(vmi), nil
|
||||
})
|
||||
if err == nil {
|
||||
klog.Infof("vmi %v is LiveMigratable (attempt %d/%d)", name, attempt, maxAttempts)
|
||||
break
|
||||
}
|
||||
if attempt == maxAttempts {
|
||||
if lastVMI != nil {
|
||||
klog.Infof("Final vmi %v status: phase=%v, conditions=%#v", name, lastVMI.Status.Phase, lastVMI.Status.Conditions)
|
||||
}
|
||||
t.Fatalf("vmi %v never became LiveMigratable after %d attempts", name, maxAttempts)
|
||||
}
|
||||
klog.Warningf("vmi %v not LiveMigratable after %v, recreating (attempt %d/%d) to work around virt-handler containerdisk-socket race", name, perAttemptWait, attempt, maxAttempts)
|
||||
if err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("Unable to delete vmi %v for retry: %v", name, err)
|
||||
}
|
||||
if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, deleteWait, true, func(ctx context.Context) (bool, error) {
|
||||
_, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
return apierrors.IsNotFound(err), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Timed out waiting for vmi %v to be deleted: %v", name, err)
|
||||
}
|
||||
if _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Create(ctx, virtualMachineInstance(i, namespace), metav1.CreateOptions{}); err != nil {
|
||||
t.Fatalf("Unable to recreate vmi %v: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface) {
|
||||
obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
@@ -129,12 +190,14 @@ func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtcl
|
||||
klog.Infof("Kubevirt is available")
|
||||
}
|
||||
|
||||
func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface) (bool, error) {
|
||||
func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface, namespace string) (bool, error) {
|
||||
klog.Infof("Checking all vmi active pods are running")
|
||||
uidMap := make(map[types.UID]*corev1.Pod)
|
||||
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
|
||||
podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: virtLauncherSelector,
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "client rate limiter") {
|
||||
if isClientRateLimiterError(err) {
|
||||
klog.Infof("Unable to list pods: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
@@ -148,7 +211,7 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client
|
||||
uidMap[item.UID] = &pod
|
||||
}
|
||||
|
||||
vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances("default").List(ctx, metav1.ListOptions{})
|
||||
vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
klog.Infof("Unable to list VMIs: %v", err)
|
||||
return false, err
|
||||
@@ -165,14 +228,19 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client
|
||||
klog.Infof("Active pod %v not found", activePod)
|
||||
return false, nil
|
||||
}
|
||||
klog.Infof("Checking whether active pod %v (uid=%v) is running", uidMap[activePod].Name, activePod)
|
||||
// ignore completed/failed pods
|
||||
if uidMap[activePod].Status.Phase == corev1.PodFailed || uidMap[activePod].Status.Phase == corev1.PodSucceeded {
|
||||
klog.Infof("Ignoring active pod %v, phase=%v", uidMap[activePod].Name, uidMap[activePod].Status.Phase)
|
||||
pod := uidMap[activePod]
|
||||
klog.Infof("Checking whether active pod %v (uid=%v) is running", pod.Name, activePod)
|
||||
if pod.Status.Phase == corev1.PodFailed {
|
||||
details := fmt.Sprintf("pod %s (phase=Failed, reason=%q, message=%q, containers=[%s])", pod.Name, pod.Status.Reason, pod.Status.Message, formatContainerStatuses(pod))
|
||||
klog.Infof("Active pod failed: %s", details)
|
||||
continue
|
||||
}
|
||||
if uidMap[activePod].Status.Phase != corev1.PodRunning {
|
||||
klog.Infof("activePod %v is not running: %v\n", uidMap[activePod].Name, uidMap[activePod].Status.Phase)
|
||||
if pod.Status.Phase == corev1.PodSucceeded {
|
||||
klog.Infof("Ignoring active pod %v, phase=%v", pod.Name, pod.Status.Phase)
|
||||
continue
|
||||
}
|
||||
if pod.Status.Phase != corev1.PodRunning {
|
||||
klog.Infof("activePod %v is not running: %v\n", pod.Name, pod.Status.Phase)
|
||||
return false, nil
|
||||
}
|
||||
atLeastOneVmiIsRunning = true
|
||||
@@ -186,7 +254,7 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy {
|
||||
func podLifeTimePolicy(namespace string) *apiv1alpha2.DeschedulerPolicy {
|
||||
return &apiv1alpha2.DeschedulerPolicy{
|
||||
Profiles: []apiv1alpha2.DeschedulerProfile{
|
||||
{
|
||||
@@ -198,7 +266,7 @@ func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy {
|
||||
Object: &podlifetime.PodLifeTimeArgs{
|
||||
MaxPodLifeTimeSeconds: utilptr.To[uint](1), // set it to immediate eviction
|
||||
Namespaces: &api.Namespaces{
|
||||
Include: []string{"default"},
|
||||
Include: []string{namespace},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -229,10 +297,12 @@ func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface) []string {
|
||||
func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string) []string {
|
||||
names := []string{}
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
|
||||
podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: virtLauncherSelector,
|
||||
})
|
||||
if err != nil {
|
||||
if isClientRateLimiterError(err) {
|
||||
t.Log(err)
|
||||
@@ -243,9 +313,6 @@ func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientse
|
||||
}
|
||||
|
||||
for _, item := range podList.Items {
|
||||
if !strings.HasPrefix(item.Name, "virt-launcher-kubevirtvmi-") {
|
||||
t.Fatalf("Only pod names with 'virt-launcher-kubevirtvmi-' prefix are expected, got %q instead", item.Name)
|
||||
}
|
||||
if item.Status.Phase == corev1.PodRunning {
|
||||
names = append(names, item.Name)
|
||||
}
|
||||
@@ -258,13 +325,13 @@ func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientse
|
||||
return names
|
||||
}
|
||||
|
||||
func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientset.Interface, usedRunningPodNames map[string]struct{}) {
|
||||
func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string, usedRunningPodNames map[string]struct{}) {
|
||||
prevTotal := uint(0)
|
||||
jumps := 0
|
||||
// keep running the descheduling cycle until the migration is triggered and completed few times or times out
|
||||
for i := 0; i < 240; i++ {
|
||||
// monitor how many pods get evicted
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient)
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient, namespace)
|
||||
klog.Infof("vmi pods: %#v\n", names)
|
||||
// The number of pods need to be kept between vmiCount and vmiCount+1.
|
||||
// At most two pods are expected to have virt-launcher-kubevirtvmi-X prefix name in common.
|
||||
@@ -299,16 +366,18 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse
|
||||
if prevTotal != 0 && prevTotal != total {
|
||||
jumps++
|
||||
}
|
||||
// Expect at least 3 finished live migrations (two should be enough as well, though ...)
|
||||
if jumps >= 6 {
|
||||
// Expect at least 2 finished live migrations
|
||||
if jumps >= 4 {
|
||||
break
|
||||
}
|
||||
prevTotal = total
|
||||
time.Sleep(time.Second)
|
||||
time.Sleep(4 * time.Second)
|
||||
}
|
||||
|
||||
if jumps < 6 {
|
||||
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
|
||||
if jumps < 4 {
|
||||
podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: virtLauncherSelector,
|
||||
})
|
||||
if err != nil {
|
||||
klog.Infof("Unable to list pods: %v", err)
|
||||
} else {
|
||||
@@ -317,9 +386,9 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse
|
||||
}
|
||||
}
|
||||
|
||||
t.Fatalf("Expected at least 3 finished live migrations, got less: %v", jumps/2.0)
|
||||
t.Fatalf("Expected at least 2 finished live migrations, got less: %v", jumps/2.0)
|
||||
}
|
||||
klog.Infof("The live migration finished 3 times")
|
||||
klog.Infof("The live migration finished 2 times")
|
||||
|
||||
// len(usedRunningPodNames) is expected to be vmiCount + jumps/2 + 1 (one more live migration could still be initiated)
|
||||
klog.Infof("len(usedRunningPodNames): %v, upper limit: %v\n", len(usedRunningPodNames), vmiCount+jumps/2+1)
|
||||
@@ -328,7 +397,7 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse
|
||||
}
|
||||
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient)
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient, namespace)
|
||||
klog.Infof("vmi pods: %#v\n", names)
|
||||
lNames := len(names)
|
||||
if lNames != vmiCount {
|
||||
@@ -385,143 +454,46 @@ func createKubevirtClient() (kubevirtclient.Interface, error) {
|
||||
return kubevirtclient.NewForConfig(config)
|
||||
}
|
||||
|
||||
func TestLiveMigrationInBackground(t *testing.T) {
|
||||
initPluginRegistry()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
kubeClient, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Error during kubernetes client creation with %v", err)
|
||||
func setupE2ELiveMigrationNamespace(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface, vmiNamespace string) {
|
||||
t.Helper()
|
||||
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: vmiNamespace}}
|
||||
if _, err := kubeClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
t.Fatalf("Unable to create namespace %v: %v", vmiNamespace, err)
|
||||
}
|
||||
|
||||
kvClient, err := createKubevirtClient()
|
||||
if err != nil {
|
||||
t.Fatalf("Error during kvClient creation with %v", err)
|
||||
}
|
||||
|
||||
waitForKubevirtReady(t, ctx, kvClient)
|
||||
|
||||
// Delete all VMIs
|
||||
defer func() {
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
for i := 1; i <= vmiCount; i++ {
|
||||
vmi := virtualMachineInstance(i)
|
||||
err := kvClient.KubevirtV1().VirtualMachineInstances("default").Delete(context.Background(), vmi.Name, metav1.DeleteOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
vmi := virtualMachineInstance(i, vmiNamespace)
|
||||
if err := kvClient.KubevirtV1().VirtualMachineInstances(vmiNamespace).Delete(cleanupCtx, vmi.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
klog.Infof("Unable to delete vmi %v: %v", vmi.Name, err)
|
||||
}
|
||||
}
|
||||
wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
|
||||
wait.PollUntilContextTimeout(cleanupCtx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
podList, err := kubeClient.CoreV1().Pods(vmiNamespace).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: virtLauncherSelector,
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
lPods := len(podList.Items)
|
||||
if lPods > 0 {
|
||||
klog.Infof("Waiting until all pods under default namespace are gone, %v remaining", lPods)
|
||||
klog.Infof("Waiting until all virt-launcher pods under %v namespace are gone, %v remaining", vmiNamespace, lPods)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
}()
|
||||
|
||||
// Create N vmis and wait for the corresponding vm pods to be ready and running
|
||||
for i := 1; i <= vmiCount; i++ {
|
||||
vmi := virtualMachineInstance(i)
|
||||
_, err = kvClient.KubevirtV1().VirtualMachineInstances("default").Create(context.Background(), vmi, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create KubeVirt vmi: %v\n", err)
|
||||
if err := kubeClient.CoreV1().Namespaces().Delete(cleanupCtx, vmiNamespace, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
klog.Infof("Unable to delete namespace %v: %v", vmiNamespace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait until all VMIs have running pods
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient)
|
||||
}); err != nil {
|
||||
t.Fatalf("Error waiting for all vmi active pods to be running: %v", err)
|
||||
}
|
||||
|
||||
usedRunningPodNames := make(map[string]struct{})
|
||||
// vmiCount number of names is expected
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient)
|
||||
klog.Infof("vmi pods: %#v\n", names)
|
||||
if len(names) != vmiCount {
|
||||
t.Fatalf("Expected %v vmi pods, got %v instead", vmiCount, len(names))
|
||||
}
|
||||
for _, name := range names {
|
||||
usedRunningPodNames[name] = struct{}{}
|
||||
}
|
||||
|
||||
policy := podLifeTimePolicy()
|
||||
// Allow only a single eviction simultaneously
|
||||
policy.MaxNoOfPodsToEvictPerNamespace = utilptr.To[uint](1)
|
||||
// Deploy the descheduler with the configured policy
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(policy)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
klog.Infof("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = kubeClient.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
defer func() {
|
||||
klog.Infof("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
|
||||
err = kubeClient.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
}()
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment("kube-system")
|
||||
// Set the descheduling interval to 10s
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "10s", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
|
||||
|
||||
deschedulerPodName := ""
|
||||
defer func() {
|
||||
if deschedulerPodName != "" {
|
||||
printPodLogs(ctx, t, kubeClient, deschedulerPodName)
|
||||
}
|
||||
|
||||
klog.Infof("Deleting %q deployment...", deschedulerDeploymentObj.Name)
|
||||
err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return
|
||||
}
|
||||
t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
}
|
||||
waitForPodsToDisappear(ctx, t, kubeClient, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
|
||||
}()
|
||||
|
||||
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
|
||||
|
||||
observeLiveMigration(t, ctx, kubeClient, usedRunningPodNames)
|
||||
|
||||
printPodLogs(ctx, t, kubeClient, deschedulerPodName)
|
||||
|
||||
klog.Infof("Deleting the current descheduler pod")
|
||||
err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error deleting %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
}
|
||||
|
||||
remainingPods := make(map[string]struct{})
|
||||
for _, name := range kVirtRunningPodNames(t, ctx, kubeClient) {
|
||||
remainingPods[name] = struct{}{}
|
||||
}
|
||||
|
||||
klog.Infof("Configuring the descheduler policy %v for PodLifetime with no limits", deschedulerPolicyConfigMapObj.Name)
|
||||
policy.MaxNoOfPodsToEvictPerNamespace = nil
|
||||
updateDeschedulerPolicy(t, ctx, kubeClient, policy)
|
||||
|
||||
deschedulerDeploymentObj = deschedulerDeployment("kube-system")
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "100m", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
|
||||
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
|
||||
})
|
||||
}
|
||||
|
||||
func waitForVMIEvictionsWithNoLimits(t *testing.T, ctx context.Context, kubeClient clientset.Interface, vmiNamespace string, remainingPods map[string]struct{}) {
|
||||
t.Helper()
|
||||
klog.Infof("Waiting until all pods are evicted (no limit set)")
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 120*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient)
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace)
|
||||
for _, name := range names {
|
||||
if _, exists := remainingPods[name]; exists {
|
||||
klog.Infof("Waiting for %v to disappear", name)
|
||||
@@ -539,3 +511,103 @@ func TestLiveMigrationInBackground(t *testing.T) {
|
||||
t.Fatalf("Error waiting for %v new vmi active pods to be running: %v", vmiCount, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveMigrationInBackground(t *testing.T) {
|
||||
initPluginRegistry()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
kubeClient, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Error during kubernetes client creation with %v", err)
|
||||
}
|
||||
|
||||
kvClient, err := createKubevirtClient()
|
||||
if err != nil {
|
||||
t.Fatalf("Error during kvClient creation with %v", err)
|
||||
}
|
||||
|
||||
waitForKubevirtReady(t, ctx, kvClient)
|
||||
|
||||
vmiNamespace := "e2e-livemigration"
|
||||
setupE2ELiveMigrationNamespace(t, ctx, kubeClient, kvClient, vmiNamespace)
|
||||
|
||||
for i := 1; i <= vmiCount; i++ {
|
||||
vmi := virtualMachineInstance(i, vmiNamespace)
|
||||
_, err = kvClient.KubevirtV1().VirtualMachineInstances(vmiNamespace).Create(context.Background(), vmi, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create KubeVirt vmi: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient, vmiNamespace)
|
||||
}); err != nil {
|
||||
t.Fatalf("Error waiting for all vmi active pods to be running: %v", err)
|
||||
}
|
||||
|
||||
ensureVMIsLiveMigratable(t, ctx, kvClient, vmiNamespace)
|
||||
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient, vmiNamespace)
|
||||
}); err != nil {
|
||||
t.Fatalf("Error waiting for all vmi active pods to be running after recreate: %v", err)
|
||||
}
|
||||
|
||||
usedRunningPodNames := make(map[string]struct{})
|
||||
names := kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace)
|
||||
klog.Infof("vmi pods: %#v\n", names)
|
||||
if len(names) != vmiCount {
|
||||
t.Fatalf("Expected %v vmi pods, got %v instead", vmiCount, len(names))
|
||||
}
|
||||
for _, name := range names {
|
||||
usedRunningPodNames[name] = struct{}{}
|
||||
}
|
||||
|
||||
policy := podLifeTimePolicy(vmiNamespace)
|
||||
policy.MaxNoOfPodsToEvictPerNamespace = utilptr.To[uint](1)
|
||||
deschedulerPolicyConfigMapObj := createPolicyConfigMap(t, ctx, kubeClient, policy)
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment("kube-system")
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "10s", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
|
||||
|
||||
deschedulerPodName := ""
|
||||
t.Cleanup(func() {
|
||||
if deschedulerPodName != "" {
|
||||
printPodLogs(context.Background(), t, kubeClient, deschedulerPodName)
|
||||
}
|
||||
|
||||
klog.Infof("Deleting %q deployment...", deschedulerDeploymentObj.Name)
|
||||
if err := kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
klog.Infof("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
}
|
||||
waitForPodsToDisappear(context.Background(), t, kubeClient, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
|
||||
})
|
||||
|
||||
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
|
||||
|
||||
observeLiveMigration(t, ctx, kubeClient, vmiNamespace, usedRunningPodNames)
|
||||
|
||||
printPodLogs(ctx, t, kubeClient, deschedulerPodName)
|
||||
|
||||
klog.Infof("Deleting the current descheduler pod")
|
||||
err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error deleting %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
}
|
||||
|
||||
remainingPods := make(map[string]struct{})
|
||||
for _, name := range kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace) {
|
||||
remainingPods[name] = struct{}{}
|
||||
}
|
||||
|
||||
klog.Infof("Configuring the descheduler policy %v for PodLifetime with no limits", deschedulerPolicyConfigMapObj.Name)
|
||||
policy.MaxNoOfPodsToEvictPerNamespace = nil
|
||||
updateDeschedulerPolicy(t, ctx, kubeClient, policy)
|
||||
|
||||
deschedulerDeploymentObj = deschedulerDeployment("kube-system")
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "100m", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
|
||||
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
|
||||
|
||||
waitForVMIEvictionsWithNoLimits(t, ctx, kubeClient, vmiNamespace, remainingPods)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
@@ -145,24 +146,7 @@ func TestFailedPods(t *testing.T) {
|
||||
Include: []string{testNamespace.Name},
|
||||
}
|
||||
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs))
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
|
||||
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
}()
|
||||
createPolicyConfigMap(t, ctx, clientSet, removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs))
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
|
||||
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
|
||||
@@ -172,19 +156,18 @@ func TestFailedPods(t *testing.T) {
|
||||
}
|
||||
|
||||
deschedulerPodName := ""
|
||||
defer func() {
|
||||
t.Cleanup(func() {
|
||||
if deschedulerPodName != "" {
|
||||
printPodLogs(ctx, t, clientSet, deschedulerPodName)
|
||||
printPodLogs(context.Background(), t, clientSet, deschedulerPodName)
|
||||
}
|
||||
|
||||
t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
|
||||
err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
}
|
||||
|
||||
waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
|
||||
}()
|
||||
waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
|
||||
})
|
||||
|
||||
t.Logf("Waiting for the descheduler pod running")
|
||||
deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
|
||||
|
||||
@@ -135,9 +135,9 @@ func TestLeaderElection(t *testing.T) {
|
||||
t.Logf("Removed kube-system/descheduler lease")
|
||||
|
||||
t.Log("Starting deschedulers")
|
||||
pod1Name, deploy1, cm1 := startDeschedulerServer(t, ctx, clientSet, ns1)
|
||||
pod1Name, deploy1, _ := startDeschedulerServer(t, ctx, clientSet, ns1)
|
||||
time.Sleep(1 * time.Second)
|
||||
pod2Name, deploy2, cm2 := startDeschedulerServer(t, ctx, clientSet, ns2)
|
||||
pod2Name, deploy2, _ := startDeschedulerServer(t, ctx, clientSet, ns2)
|
||||
defer func() {
|
||||
for _, podName := range []string{pod1Name, pod2Name} {
|
||||
printPodLogs(ctx, t, clientSet, podName)
|
||||
@@ -153,14 +153,6 @@ func TestLeaderElection(t *testing.T) {
|
||||
waitForPodsToDisappear(ctx, t, clientSet, deploy.Labels, deploy.Namespace)
|
||||
}
|
||||
|
||||
for _, cm := range []*v1.ConfigMap{cm1, cm2} {
|
||||
t.Logf("Deleting %q CM...", cm.Name)
|
||||
err = clientSet.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", cm.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{})
|
||||
}()
|
||||
|
||||
@@ -223,21 +215,21 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients
|
||||
EvictFailedBarePods: false,
|
||||
}
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs))
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
deschedulerPolicyConfigMapObj.Name = fmt.Sprintf("%s-%s", deschedulerPolicyConfigMapObj.Name, testName)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
createConfigMapWithCleanup(t, ctx, clientSet, deschedulerPolicyConfigMapObj)
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment(testName)
|
||||
deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName)
|
||||
args := deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect", "--leader-elect-retry-period", "1s")
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{
|
||||
"--policy-config-file", "/policy-dir/policy.yaml",
|
||||
"--descheduling-interval", "3s",
|
||||
"--v", "4",
|
||||
"--leader-elect",
|
||||
"--leader-elect-retry-period", "1s",
|
||||
}
|
||||
deschedulerDeploymentObj.Spec.Template.Spec.Volumes = []v1.Volume{
|
||||
{
|
||||
Name: "policy-volume",
|
||||
|
||||
@@ -289,21 +289,7 @@ func TestProtectPodsWithPVC(t *testing.T) {
|
||||
t.Fatalf("Error creating %q CM: %v", policycm.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("creating %q policy CM with PodsWithPVC protection enabled...", policycm.Name)
|
||||
if _, err = cli.CoreV1().ConfigMaps(policycm.Namespace).Create(
|
||||
ctx, policycm, metav1.CreateOptions{},
|
||||
); err != nil {
|
||||
t.Fatalf("error creating %q CM: %v", policycm.Name, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Logf("deleting %q CM...", policycm.Name)
|
||||
if err := cli.CoreV1().ConfigMaps(policycm.Namespace).Delete(
|
||||
ctx, policycm.Name, metav1.DeleteOptions{},
|
||||
); err != nil {
|
||||
t.Fatalf("unable to delete %q CM: %v", policycm.Name, err)
|
||||
}
|
||||
}()
|
||||
createConfigMapWithCleanup(t, ctx, cli, policycm)
|
||||
|
||||
desdep := deschedulerDeployment(namespace.Name)
|
||||
t.Logf("creating descheduler deployment %v", desdep.Name)
|
||||
|
||||
+42
-1
@@ -31,6 +31,7 @@ import (
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
schedulingv1 "k8s.io/api/scheduling/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
@@ -84,7 +85,12 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
|
||||
func isClientRateLimiterError(err error) bool {
|
||||
return strings.Contains(err.Error(), "client rate limiter")
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(err.Error(), "client rate limiter") &&
|
||||
!strings.Contains(err.Error(), "context deadline exceeded") &&
|
||||
!strings.Contains(err.Error(), "context canceled")
|
||||
}
|
||||
|
||||
func initFeatureGates() featuregate.FeatureGate {
|
||||
@@ -112,6 +118,41 @@ func deschedulerPolicyConfigMap(policy *deschedulerapiv1alpha2.DeschedulerPolicy
|
||||
return cm, nil
|
||||
}
|
||||
|
||||
// createPolicyConfigMap generates, creates (or recreates if already existing), and registers cleanup for a descheduler policy ConfigMap.
|
||||
func createPolicyConfigMap(t *testing.T, ctx context.Context, kubeClient clientset.Interface, policy *deschedulerapiv1alpha2.DeschedulerPolicy) *v1.ConfigMap {
|
||||
t.Helper()
|
||||
cm, err := deschedulerPolicyConfigMap(policy)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating policy CM object: %v", err)
|
||||
}
|
||||
return createConfigMapWithCleanup(t, ctx, kubeClient, cm)
|
||||
}
|
||||
|
||||
// createConfigMapWithCleanup creates (or recreates if already existing) a ConfigMap and registers a t.Cleanup callback to delete it.
|
||||
func createConfigMapWithCleanup(t *testing.T, ctx context.Context, kubeClient clientset.Interface, cm *v1.ConfigMap) *v1.ConfigMap {
|
||||
t.Helper()
|
||||
t.Logf("Creating %q policy CM...", cm.Name)
|
||||
_, err := kubeClient.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
_ = kubeClient.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
|
||||
_, err = kubeClient.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{})
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", cm.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
t.Logf("Deleting %q CM...", cm.Name)
|
||||
if err := kubeClient.CoreV1().ConfigMaps(cm.Namespace).Delete(context.Background(), cm.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Logf("Unable to delete %q CM: %v", cm.Name, err)
|
||||
}
|
||||
})
|
||||
|
||||
return cm
|
||||
}
|
||||
|
||||
func deschedulerDeployment(testName string) *appsv1.Deployment {
|
||||
deploymentObject := &appsv1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
|
||||
@@ -153,24 +153,7 @@ func TestTooManyRestarts(t *testing.T) {
|
||||
rs.DefaultFeatureGates = initFeatureGates()
|
||||
|
||||
preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
|
||||
// Deploy the descheduler with the configured policy
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tc.policy)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
|
||||
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
}()
|
||||
createPolicyConfigMap(t, ctx, clientSet, tc.policy)
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
|
||||
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
|
||||
|
||||
@@ -219,24 +219,7 @@ func TestTopologySpreadConstraint(t *testing.T) {
|
||||
Include: []string{testNamespace.Name},
|
||||
},
|
||||
}
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs))
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
|
||||
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
}()
|
||||
createPolicyConfigMap(t, ctx, clientSet, topologySpreadConstraintPolicy(constraintArgs, evictorArgs))
|
||||
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
|
||||
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
|
||||
_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
|
||||
|
||||
Reference in New Issue
Block a user