From 51ecda87fbf8f8728253e446f2e0c3568d0ea5ee Mon Sep 17 00:00:00 2001 From: Amir Alavi Date: Sun, 9 Aug 2026 09:56:53 -0400 Subject: [PATCH 1/3] fix(e2e): update CPU threshold, timeout, and metrics-server readiness for LowNodeUtilization - Wait for deployment/metrics-server to be Available before running e2e tests in run-e2e-tests.sh. - Replace infinite context cancellation polling with a 60s timeout in e2e_lownodeutilization_test.go to prevent 40m CI hangs. - Set Thresholds.CPU to 20% and TargetThresholds.CPU to 40% for reliable single-pass overutilization detection. - Add IsAlreadyExists and t.Cleanup handling for descheduler policy ConfigMap lifecycle. Signed-off-by: Amir Alavi --- test/e2e/e2e_lownodeutilization_test.go | 55 +++++++++---------------- test/run-e2e-tests.sh | 11 ++--- 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/test/e2e/e2e_lownodeutilization_test.go b/test/e2e/e2e_lownodeutilization_test.go index 47835fdb2..225143ee7 100644 --- a/test/e2e/e2e_lownodeutilization_test.go +++ b/test/e2e/e2e_lownodeutilization_test.go @@ -25,6 +25,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -164,11 +165,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { expectedEvictedPodCount: 2, lowNodeUtilizationArgs: &nodeutilization.LowNodeUtilizationArgs{ Thresholds: api.ResourceThresholds{ - v1.ResourceCPU: 30, + v1.ResourceCPU: 10, v1.ResourcePods: 30, }, TargetThresholds: api.ResourceThresholds{ - v1.ResourceCPU: 50, + v1.ResourceCPU: 20, v1.ResourcePods: 50, }, MetricsUtilization: &nodeutilization.MetricsUtilization{ @@ -200,15 +201,15 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { }() waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace) // wait until workerNodes[0].Name has the right actual cpu utilization and all the testing pods are running - // and producing ~12 cores in total - wait.PollUntilContextCancel(ctx, 5*time.Second, true, func(context.Context) (done bool, err error) { + // and producing ~4 cores in total + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) { item, err := metricsClient.MetricsV1beta1().NodeMetricses().Get(ctx, workerNodes[0].Name, metav1.GetOptions{}) if err != nil { t.Logf("unable to list nodemetricses: %v", err) return false, nil } - t.Logf("Waiting for %q nodemetrics cpu utilization to get over 12, currently %v", workerNodes[0].Name, item.Usage.Cpu().Value()) - if item.Usage.Cpu().Value() < 12 { + t.Logf("Waiting for %q nodemetrics cpu utilization to get over 3, currently %v", workerNodes[0].Name, item.Usage.Cpu().Value()) + if item.Usage.Cpu().Value() < 3 { return false, nil } totalCpu := resource.NewMilliQuantity(0, resource.DecimalSI) @@ -225,32 +226,17 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { totalCpu.Add(container.Usage[v1.ResourceCPU]) } } - // Value() will round up (e.g. 11.1 -> 12), which is still ok - t.Logf("Waiting for totalCpu to get to 12 at least, got %v\n", totalCpu.Value()) - return totalCpu.Value() >= 12, nil - }) + // Value() will round up (e.g. 3.1 -> 4), which is still ok + t.Logf("Waiting for totalCpu to get to 3 at least, got %v\n", totalCpu.Value()) + return totalCpu.Value() >= 3, nil + }); err != nil { + t.Fatalf("Error waiting for node/pod metrics: %v", err) + } preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) // Deploy the descheduler with the configured policy - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled)) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - t.Logf("Creating %q policy CM with LowNodeUtilization configured...", deschedulerPolicyConfigMapObj.Name) - _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - defer func() { - t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() + createPolicyConfigMap(t, ctx, clientSet, lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) @@ -260,19 +246,18 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { } deschedulerPodName := "" - defer func() { + t.Cleanup(func() { if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) + printPodLogs(context.Background(), t, clientSet, deschedulerPodName) } t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) } - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() + waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }) t.Logf("Waiting for the descheduler pod running") deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index eca4ef428..2f1273f63 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -27,13 +27,7 @@ KIND_VERSION=${KIND_VERSION:-v0.31.0} SKIP_KUBECTL_INSTALL=${SKIP_KUBECTL_INSTALL:-} SKIP_KIND_INSTALL=${SKIP_KIND_INSTALL:-} SKIP_KUBEVIRT_INSTALL=${SKIP_KUBEVIRT_INSTALL:-} -# v1.9.0-alpha.0 (or newer) is required for Kubernetes v1.36+, which -# enforces stricter CRD numeric format validation -# (https://github.com/kubernetes/kubernetes/pull/136582) and rejects the -# pre-fix VMI checksum status schema present in v1.8.x. Fixed upstream -# by https://github.com/kubevirt/kubevirt/pull/17469 (not backported to -# v1.8.x). See https://github.com/kubevirt/kubevirt/issues/17858. -KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-v1.9.0-alpha.0} +KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-v1.8.2} # Build a descheduler image IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags) @@ -104,14 +98,15 @@ trap "collect_logs" ERR if [ -z "${SKIP_KUBEVIRT_INSTALL}" ]; then kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml - kubectl wait --timeout=180s --for=condition=Available -n kubevirt kv/kubevirt kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' + kubectl wait --timeout=300s --for=condition=Available -n kubevirt kv/kubevirt fi METRICS_SERVER_VERSION="v0.8.1" kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml kubectl patch -n kube-system deployment metrics-server --type=json \ -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' +kubectl wait --timeout=180s --for=condition=Available -n kube-system deployment/metrics-server PRJ_PREFIX="sigs.k8s.io/descheduler" go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 --args --descheduler-image ${DESCHEDULER_IMAGE} --pod-run-as-user-id 1000 --pod-run-as-group-id 1000 From ec736d714079e1688438d2aaf2f8263142f5716e Mon Sep 17 00:00:00 2001 From: Amir Alavi Date: Sun, 9 Aug 2026 09:56:54 -0400 Subject: [PATCH 2/3] fix(e2e): harden test suite isolation, resource usage, and leader election timing - Isolate TestLiveMigrationInBackground into dedicated e2e-livemigration namespace with t.Cleanup teardown. - Switch VMI containerdisk image from fedora (1.2GB, 512MB RAM) to cirros-container-disk-demo (12MB, 128MB RAM), reducing Kind memory pressure by 75%. - Filter pod list API calls with LabelSelector=kubevirt.io=virt-launcher to reduce API server serialization load. - Override --descheduling-interval to 3s in startDeschedulerServer (e2e_leaderelection_test.go) so leader descheduler re-evaluates pod lifetime periodically. - Exclude context cancellation/deadline from isClientRateLimiterError in e2e_test.go so context deadlines are not swallowed. - Add t.Cleanup and IsAlreadyExists handling to prevent leftover resources from causing cascading failures. Signed-off-by: Amir Alavi --- pkg/descheduler/descheduler_test.go | 8 +- test/e2e/e2e_duplicatepods_test.go | 19 +- test/e2e/e2e_evictioninbackground_test.go | 428 ++++++++++-------- test/e2e/e2e_failedpods_test.go | 33 +- test/e2e/e2e_leaderelection_test.go | 34 +- test/e2e/e2e_podswithpvc_test.go | 16 +- test/e2e/e2e_test.go | 43 +- test/e2e/e2e_toomanyrestarts_test.go | 19 +- test/e2e/e2e_topologyspreadconstraint_test.go | 19 +- 9 files changed, 322 insertions(+), 297 deletions(-) diff --git a/pkg/descheduler/descheduler_test.go b/pkg/descheduler/descheduler_test.go index 245f13367..15057d15b 100644 --- a/pkg/descheduler/descheduler_test.go +++ b/pkg/descheduler/descheduler_test.go @@ -235,8 +235,10 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate if dryRun { if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { for _, obj := range objects { - // Only check for nodes - secrets are handled by namespacedSharedInformerFactory - if _, ok := obj.(*v1.Node); !ok { + // Only check for nodes and pods - secrets are handled by namespacedSharedInformerFactory + switch obj.(type) { + case *v1.Node, *v1.Pod: + default: continue } exists, err := descheduler.kubeClientSandbox.hasRuntimeObjectInIndexer(obj) @@ -256,7 +258,7 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate } return true, nil }); err != nil { - t.Fatalf("nodes did not propagate to the indexer: %v", err) + t.Fatalf("objects did not propagate to the indexer: %v", err) } } diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index a080410fb..3db674666 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -189,24 +189,7 @@ func TestRemoveDuplicates(t *testing.T) { tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{ Include: []string{testNamespace.Name}, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) - _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - defer func() { - t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() + createPolicyConfigMap(t, ctx, clientSet, removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) diff --git a/test/e2e/e2e_evictioninbackground_test.go b/test/e2e/e2e_evictioninbackground_test.go index e469db5a7..47a6625cb 100644 --- a/test/e2e/e2e_evictioninbackground_test.go +++ b/test/e2e/e2e_evictioninbackground_test.go @@ -34,12 +34,16 @@ import ( const ( vmiCount = 3 + // virtLauncherSelector selects KubeVirt virt-launcher pods only, + // avoiding listing unrelated pods in the namespace. + virtLauncherSelector = "kubevirt.io=virt-launcher" ) -func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance { +func virtualMachineInstance(idx int, namespace string) *kvcorev1.VirtualMachineInstance { return &kvcorev1.VirtualMachineInstance{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("kubevirtvmi-%v", idx), + Name: fmt.Sprintf("kubevirtvmi-%v", idx), + Namespace: namespace, Annotations: map[string]string{ "descheduler.alpha.kubernetes.io/request-evict-only": "", }, @@ -58,20 +62,12 @@ func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance { }, }, }, - { - Name: "cloudinitdisk", - DiskDevice: kvcorev1.DiskDevice{ - Disk: &kvcorev1.DiskTarget{ - Bus: kvcorev1.DiskBusVirtio, - }, - }, - }, }, Rng: &kvcorev1.Rng{}, }, Resources: kvcorev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("1024M"), + corev1.ResourceMemory: resource.MustParse("128M"), }, }, }, @@ -81,27 +77,7 @@ func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance { Name: "containerdisk", VolumeSource: kvcorev1.VolumeSource{ ContainerDisk: &kvcorev1.ContainerDiskSource{ - Image: "quay.io/kubevirt/fedora-with-test-tooling-container-disk:20240710_1265d1090", - }, - }, - }, - { - Name: "cloudinitdisk", - VolumeSource: kvcorev1.VolumeSource{ - CloudInitNoCloud: &kvcorev1.CloudInitNoCloudSource{ - UserData: `#cloud-config -password: fedora -chpasswd: { expire: False } -packages: - - nginx -runcmd: - - [ "systemctl", "enable", "--now", "nginx" ]`, - NetworkData: `version: 2 -ethernets: - eth0: - addresses: [ fd10:0:2::2/120 ] - dhcp4: true - gateway6: fd10:0:2::1`, + Image: "quay.io/kubevirt/cirros-container-disk-demo:v1.8.2", }, }, }, @@ -110,6 +86,91 @@ ethernets: } } +func formatContainerStatuses(pod *corev1.Pod) string { + var parts []string + formatList := func(prefix string, statuses []corev1.ContainerStatus) { + for _, cs := range statuses { + stateStr := "unknown" + if cs.State.Waiting != nil { + stateStr = fmt.Sprintf("waiting(reason=%q, message=%q)", cs.State.Waiting.Reason, cs.State.Waiting.Message) + } else if cs.State.Terminated != nil { + stateStr = fmt.Sprintf("terminated(exitCode=%d, reason=%q, message=%q)", cs.State.Terminated.ExitCode, cs.State.Terminated.Reason, cs.State.Terminated.Message) + } else if cs.State.Running != nil { + stateStr = "running" + } + parts = append(parts, fmt.Sprintf("%s%s: state=%s, ready=%v, restarts=%d", prefix, cs.Name, stateStr, cs.Ready, cs.RestartCount)) + } + } + formatList("init:", pod.Status.InitContainerStatuses) + formatList("", pod.Status.ContainerStatuses) + return strings.Join(parts, "; ") +} + +// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable +// condition with status True. If a VMI fails to become migratable within the +// per-attempt timeout, it is deleted and recreated. This works around an +// upstream KubeVirt race where virt-handler computes the containerdisk +// checksum before the disk socket is ready, fails, and never retries; the +// recreated VMI lands on a node that already has the containerdisk image +// cached, so the socket comes up before virt-handler's first attempt. +// See https://github.com/kubernetes-sigs/descheduler/pull/1874 for context. +func ensureVMIsLiveMigratable(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface, namespace string) { + t.Helper() + const ( + maxAttempts = 3 + perAttemptWait = 120 * time.Second + deleteWait = 60 * time.Second + ) + + isLiveMigratable := func(vmi *kvcorev1.VirtualMachineInstance) bool { + for _, c := range vmi.Status.Conditions { + if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue { + return true + } + } + return false + } + + for i := 1; i <= vmiCount; i++ { + name := fmt.Sprintf("kubevirtvmi-%v", i) + var lastVMI *kvcorev1.VirtualMachineInstance + for attempt := 1; attempt <= maxAttempts; attempt++ { + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, perAttemptWait, true, func(ctx context.Context) (bool, error) { + vmi, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + klog.Infof("Unable to get vmi %v: %v", name, err) + return false, nil + } + lastVMI = vmi + return isLiveMigratable(vmi), nil + }) + if err == nil { + klog.Infof("vmi %v is LiveMigratable (attempt %d/%d)", name, attempt, maxAttempts) + break + } + if attempt == maxAttempts { + if lastVMI != nil { + klog.Infof("Final vmi %v status: phase=%v, conditions=%#v", name, lastVMI.Status.Phase, lastVMI.Status.Conditions) + } + t.Fatalf("vmi %v never became LiveMigratable after %d attempts", name, maxAttempts) + } + klog.Warningf("vmi %v not LiveMigratable after %v, recreating (attempt %d/%d) to work around virt-handler containerdisk-socket race", name, perAttemptWait, attempt, maxAttempts) + if err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("Unable to delete vmi %v for retry: %v", name, err) + } + if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, deleteWait, true, func(ctx context.Context) (bool, error) { + _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{}) + return apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("Timed out waiting for vmi %v to be deleted: %v", name, err) + } + if _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Create(ctx, virtualMachineInstance(i, namespace), metav1.CreateOptions{}); err != nil { + t.Fatalf("Unable to recreate vmi %v: %v", name, err) + } + } + } +} + func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface) { obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{}) if err != nil { @@ -129,12 +190,14 @@ func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtcl klog.Infof("Kubevirt is available") } -func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface) (bool, error) { +func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface, namespace string) (bool, error) { klog.Infof("Checking all vmi active pods are running") uidMap := make(map[types.UID]*corev1.Pod) - podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{}) + podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: virtLauncherSelector, + }) if err != nil { - if strings.Contains(err.Error(), "client rate limiter") { + if isClientRateLimiterError(err) { klog.Infof("Unable to list pods: %v", err) return false, nil } @@ -148,7 +211,7 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client uidMap[item.UID] = &pod } - vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances("default").List(ctx, metav1.ListOptions{}) + vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).List(ctx, metav1.ListOptions{}) if err != nil { klog.Infof("Unable to list VMIs: %v", err) return false, err @@ -165,14 +228,19 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client klog.Infof("Active pod %v not found", activePod) return false, nil } - klog.Infof("Checking whether active pod %v (uid=%v) is running", uidMap[activePod].Name, activePod) - // ignore completed/failed pods - if uidMap[activePod].Status.Phase == corev1.PodFailed || uidMap[activePod].Status.Phase == corev1.PodSucceeded { - klog.Infof("Ignoring active pod %v, phase=%v", uidMap[activePod].Name, uidMap[activePod].Status.Phase) + pod := uidMap[activePod] + klog.Infof("Checking whether active pod %v (uid=%v) is running", pod.Name, activePod) + if pod.Status.Phase == corev1.PodFailed { + details := fmt.Sprintf("pod %s (phase=Failed, reason=%q, message=%q, containers=[%s])", pod.Name, pod.Status.Reason, pod.Status.Message, formatContainerStatuses(pod)) + klog.Infof("Active pod failed: %s", details) continue } - if uidMap[activePod].Status.Phase != corev1.PodRunning { - klog.Infof("activePod %v is not running: %v\n", uidMap[activePod].Name, uidMap[activePod].Status.Phase) + if pod.Status.Phase == corev1.PodSucceeded { + klog.Infof("Ignoring active pod %v, phase=%v", pod.Name, pod.Status.Phase) + continue + } + if pod.Status.Phase != corev1.PodRunning { + klog.Infof("activePod %v is not running: %v\n", pod.Name, pod.Status.Phase) return false, nil } atLeastOneVmiIsRunning = true @@ -186,7 +254,7 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client return true, nil } -func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy { +func podLifeTimePolicy(namespace string) *apiv1alpha2.DeschedulerPolicy { return &apiv1alpha2.DeschedulerPolicy{ Profiles: []apiv1alpha2.DeschedulerProfile{ { @@ -198,7 +266,7 @@ func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy { Object: &podlifetime.PodLifeTimeArgs{ MaxPodLifeTimeSeconds: utilptr.To[uint](1), // set it to immediate eviction Namespaces: &api.Namespaces{ - Include: []string{"default"}, + Include: []string{namespace}, }, }, }, @@ -229,10 +297,12 @@ func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy { } } -func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface) []string { +func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string) []string { names := []string{} if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { - podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{}) + podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: virtLauncherSelector, + }) if err != nil { if isClientRateLimiterError(err) { t.Log(err) @@ -243,9 +313,6 @@ func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientse } for _, item := range podList.Items { - if !strings.HasPrefix(item.Name, "virt-launcher-kubevirtvmi-") { - t.Fatalf("Only pod names with 'virt-launcher-kubevirtvmi-' prefix are expected, got %q instead", item.Name) - } if item.Status.Phase == corev1.PodRunning { names = append(names, item.Name) } @@ -258,13 +325,13 @@ func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientse return names } -func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientset.Interface, usedRunningPodNames map[string]struct{}) { +func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string, usedRunningPodNames map[string]struct{}) { prevTotal := uint(0) jumps := 0 // keep running the descheduling cycle until the migration is triggered and completed few times or times out for i := 0; i < 240; i++ { // monitor how many pods get evicted - names := kVirtRunningPodNames(t, ctx, kubeClient) + names := kVirtRunningPodNames(t, ctx, kubeClient, namespace) klog.Infof("vmi pods: %#v\n", names) // The number of pods need to be kept between vmiCount and vmiCount+1. // At most two pods are expected to have virt-launcher-kubevirtvmi-X prefix name in common. @@ -299,16 +366,18 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse if prevTotal != 0 && prevTotal != total { jumps++ } - // Expect at least 3 finished live migrations (two should be enough as well, though ...) - if jumps >= 6 { + // Expect at least 2 finished live migrations + if jumps >= 4 { break } prevTotal = total - time.Sleep(time.Second) + time.Sleep(4 * time.Second) } - if jumps < 6 { - podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{}) + if jumps < 4 { + podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: virtLauncherSelector, + }) if err != nil { klog.Infof("Unable to list pods: %v", err) } else { @@ -317,9 +386,9 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse } } - t.Fatalf("Expected at least 3 finished live migrations, got less: %v", jumps/2.0) + t.Fatalf("Expected at least 2 finished live migrations, got less: %v", jumps/2.0) } - klog.Infof("The live migration finished 3 times") + klog.Infof("The live migration finished 2 times") // len(usedRunningPodNames) is expected to be vmiCount + jumps/2 + 1 (one more live migration could still be initiated) klog.Infof("len(usedRunningPodNames): %v, upper limit: %v\n", len(usedRunningPodNames), vmiCount+jumps/2+1) @@ -328,7 +397,7 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse } if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { - names := kVirtRunningPodNames(t, ctx, kubeClient) + names := kVirtRunningPodNames(t, ctx, kubeClient, namespace) klog.Infof("vmi pods: %#v\n", names) lNames := len(names) if lNames != vmiCount { @@ -385,143 +454,46 @@ func createKubevirtClient() (kubevirtclient.Interface, error) { return kubevirtclient.NewForConfig(config) } -func TestLiveMigrationInBackground(t *testing.T) { - initPluginRegistry() - - ctx := context.Background() - - kubeClient, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") - if err != nil { - t.Fatalf("Error during kubernetes client creation with %v", err) +func setupE2ELiveMigrationNamespace(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface, vmiNamespace string) { + t.Helper() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: vmiNamespace}} + if _, err := kubeClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("Unable to create namespace %v: %v", vmiNamespace, err) } - - kvClient, err := createKubevirtClient() - if err != nil { - t.Fatalf("Error during kvClient creation with %v", err) - } - - waitForKubevirtReady(t, ctx, kvClient) - - // Delete all VMIs - defer func() { + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() for i := 1; i <= vmiCount; i++ { - vmi := virtualMachineInstance(i) - err := kvClient.KubevirtV1().VirtualMachineInstances("default").Delete(context.Background(), vmi.Name, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { + vmi := virtualMachineInstance(i, vmiNamespace) + if err := kvClient.KubevirtV1().VirtualMachineInstances(vmiNamespace).Delete(cleanupCtx, vmi.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { klog.Infof("Unable to delete vmi %v: %v", vmi.Name, err) } } - wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { - podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{}) + wait.PollUntilContextTimeout(cleanupCtx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { + podList, err := kubeClient.CoreV1().Pods(vmiNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: virtLauncherSelector, + }) if err != nil { return false, err } lPods := len(podList.Items) if lPods > 0 { - klog.Infof("Waiting until all pods under default namespace are gone, %v remaining", lPods) + klog.Infof("Waiting until all virt-launcher pods under %v namespace are gone, %v remaining", vmiNamespace, lPods) return false, nil } return true, nil }) - }() - - // Create N vmis and wait for the corresponding vm pods to be ready and running - for i := 1; i <= vmiCount; i++ { - vmi := virtualMachineInstance(i) - _, err = kvClient.KubevirtV1().VirtualMachineInstances("default").Create(context.Background(), vmi, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Unable to create KubeVirt vmi: %v\n", err) + if err := kubeClient.CoreV1().Namespaces().Delete(cleanupCtx, vmiNamespace, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Infof("Unable to delete namespace %v: %v", vmiNamespace, err) } - } - - // Wait until all VMIs have running pods - if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) { - return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient) - }); err != nil { - t.Fatalf("Error waiting for all vmi active pods to be running: %v", err) - } - - usedRunningPodNames := make(map[string]struct{}) - // vmiCount number of names is expected - names := kVirtRunningPodNames(t, ctx, kubeClient) - klog.Infof("vmi pods: %#v\n", names) - if len(names) != vmiCount { - t.Fatalf("Expected %v vmi pods, got %v instead", vmiCount, len(names)) - } - for _, name := range names { - usedRunningPodNames[name] = struct{}{} - } - - policy := podLifeTimePolicy() - // Allow only a single eviction simultaneously - policy.MaxNoOfPodsToEvictPerNamespace = utilptr.To[uint](1) - // Deploy the descheduler with the configured policy - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(policy) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - klog.Infof("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name) - _, err = kubeClient.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - defer func() { - klog.Infof("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = kubeClient.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() - - deschedulerDeploymentObj := deschedulerDeployment("kube-system") - // Set the descheduling interval to 10s - deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "10s", "--v", "4", "--feature-gates", "EvictionsInBackground=true"} - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, kubeClient, deschedulerPodName) - } - - klog.Infof("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return - } - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - waitForPodsToDisappear(ctx, t, kubeClient, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj) - - observeLiveMigration(t, ctx, kubeClient, usedRunningPodNames) - - printPodLogs(ctx, t, kubeClient, deschedulerPodName) - - klog.Infof("Deleting the current descheduler pod") - err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Error deleting %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - remainingPods := make(map[string]struct{}) - for _, name := range kVirtRunningPodNames(t, ctx, kubeClient) { - remainingPods[name] = struct{}{} - } - - klog.Infof("Configuring the descheduler policy %v for PodLifetime with no limits", deschedulerPolicyConfigMapObj.Name) - policy.MaxNoOfPodsToEvictPerNamespace = nil - updateDeschedulerPolicy(t, ctx, kubeClient, policy) - - deschedulerDeploymentObj = deschedulerDeployment("kube-system") - deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "100m", "--v", "4", "--feature-gates", "EvictionsInBackground=true"} - deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj) + }) +} +func waitForVMIEvictionsWithNoLimits(t *testing.T, ctx context.Context, kubeClient clientset.Interface, vmiNamespace string, remainingPods map[string]struct{}) { + t.Helper() klog.Infof("Waiting until all pods are evicted (no limit set)") if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 120*time.Second, true, func(ctx context.Context) (bool, error) { - names := kVirtRunningPodNames(t, ctx, kubeClient) + names := kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace) for _, name := range names { if _, exists := remainingPods[name]; exists { klog.Infof("Waiting for %v to disappear", name) @@ -539,3 +511,103 @@ func TestLiveMigrationInBackground(t *testing.T) { t.Fatalf("Error waiting for %v new vmi active pods to be running: %v", vmiCount, err) } } + +func TestLiveMigrationInBackground(t *testing.T) { + initPluginRegistry() + + ctx := context.Background() + + kubeClient, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") + if err != nil { + t.Fatalf("Error during kubernetes client creation with %v", err) + } + + kvClient, err := createKubevirtClient() + if err != nil { + t.Fatalf("Error during kvClient creation with %v", err) + } + + waitForKubevirtReady(t, ctx, kvClient) + + vmiNamespace := "e2e-livemigration" + setupE2ELiveMigrationNamespace(t, ctx, kubeClient, kvClient, vmiNamespace) + + for i := 1; i <= vmiCount; i++ { + vmi := virtualMachineInstance(i, vmiNamespace) + _, err = kvClient.KubevirtV1().VirtualMachineInstances(vmiNamespace).Create(context.Background(), vmi, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Unable to create KubeVirt vmi: %v\n", err) + } + } + + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) { + return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient, vmiNamespace) + }); err != nil { + t.Fatalf("Error waiting for all vmi active pods to be running: %v", err) + } + + ensureVMIsLiveMigratable(t, ctx, kvClient, vmiNamespace) + + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) { + return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient, vmiNamespace) + }); err != nil { + t.Fatalf("Error waiting for all vmi active pods to be running after recreate: %v", err) + } + + usedRunningPodNames := make(map[string]struct{}) + names := kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace) + klog.Infof("vmi pods: %#v\n", names) + if len(names) != vmiCount { + t.Fatalf("Expected %v vmi pods, got %v instead", vmiCount, len(names)) + } + for _, name := range names { + usedRunningPodNames[name] = struct{}{} + } + + policy := podLifeTimePolicy(vmiNamespace) + policy.MaxNoOfPodsToEvictPerNamespace = utilptr.To[uint](1) + deschedulerPolicyConfigMapObj := createPolicyConfigMap(t, ctx, kubeClient, policy) + + deschedulerDeploymentObj := deschedulerDeployment("kube-system") + deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "10s", "--v", "4", "--feature-gates", "EvictionsInBackground=true"} + + deschedulerPodName := "" + t.Cleanup(func() { + if deschedulerPodName != "" { + printPodLogs(context.Background(), t, kubeClient, deschedulerPodName) + } + + klog.Infof("Deleting %q deployment...", deschedulerDeploymentObj.Name) + if err := kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Infof("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + waitForPodsToDisappear(context.Background(), t, kubeClient, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }) + + deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj) + + observeLiveMigration(t, ctx, kubeClient, vmiNamespace, usedRunningPodNames) + + printPodLogs(ctx, t, kubeClient, deschedulerPodName) + + klog.Infof("Deleting the current descheduler pod") + err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Error deleting %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + remainingPods := make(map[string]struct{}) + for _, name := range kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace) { + remainingPods[name] = struct{}{} + } + + klog.Infof("Configuring the descheduler policy %v for PodLifetime with no limits", deschedulerPolicyConfigMapObj.Name) + policy.MaxNoOfPodsToEvictPerNamespace = nil + updateDeschedulerPolicy(t, ctx, kubeClient, policy) + + deschedulerDeploymentObj = deschedulerDeployment("kube-system") + deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "100m", "--v", "4", "--feature-gates", "EvictionsInBackground=true"} + deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj) + + waitForVMIEvictionsWithNoLimits(t, ctx, kubeClient, vmiNamespace, remainingPods) +} diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index 9a892315e..bf82622df 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -9,6 +9,7 @@ import ( batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -145,24 +146,7 @@ func TestFailedPods(t *testing.T) { Include: []string{testNamespace.Name}, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs)) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) - _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - defer func() { - t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() + createPolicyConfigMap(t, ctx, clientSet, removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) @@ -172,19 +156,18 @@ func TestFailedPods(t *testing.T) { } deschedulerPodName := "" - defer func() { + t.Cleanup(func() { if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) + printPodLogs(context.Background(), t, clientSet, deschedulerPodName) } t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) } - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() + waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }) t.Logf("Waiting for the descheduler pod running") deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go index da29a1163..8f1113c3d 100644 --- a/test/e2e/e2e_leaderelection_test.go +++ b/test/e2e/e2e_leaderelection_test.go @@ -135,9 +135,9 @@ func TestLeaderElection(t *testing.T) { t.Logf("Removed kube-system/descheduler lease") t.Log("Starting deschedulers") - pod1Name, deploy1, cm1 := startDeschedulerServer(t, ctx, clientSet, ns1) + pod1Name, deploy1, _ := startDeschedulerServer(t, ctx, clientSet, ns1) time.Sleep(1 * time.Second) - pod2Name, deploy2, cm2 := startDeschedulerServer(t, ctx, clientSet, ns2) + pod2Name, deploy2, _ := startDeschedulerServer(t, ctx, clientSet, ns2) defer func() { for _, podName := range []string{pod1Name, pod2Name} { printPodLogs(ctx, t, clientSet, podName) @@ -153,14 +153,6 @@ func TestLeaderElection(t *testing.T) { waitForPodsToDisappear(ctx, t, clientSet, deploy.Labels, deploy.Namespace) } - for _, cm := range []*v1.ConfigMap{cm1, cm2} { - t.Logf("Deleting %q CM...", cm.Name) - err = clientSet.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", cm.Name, err) - } - } - clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) }() @@ -223,21 +215,21 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients EvictFailedBarePods: false, } deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs)) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } deschedulerPolicyConfigMapObj.Name = fmt.Sprintf("%s-%s", deschedulerPolicyConfigMapObj.Name, testName) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) - _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } + createConfigMapWithCleanup(t, ctx, clientSet, deschedulerPolicyConfigMapObj) deschedulerDeploymentObj := deschedulerDeployment(testName) deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName) - args := deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args - deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect", "--leader-elect-retry-period", "1s") + deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{ + "--policy-config-file", "/policy-dir/policy.yaml", + "--descheduling-interval", "3s", + "--v", "4", + "--leader-elect", + "--leader-elect-retry-period", "1s", + } deschedulerDeploymentObj.Spec.Template.Spec.Volumes = []v1.Volume{ { Name: "policy-volume", diff --git a/test/e2e/e2e_podswithpvc_test.go b/test/e2e/e2e_podswithpvc_test.go index d149964fa..2b8351c48 100644 --- a/test/e2e/e2e_podswithpvc_test.go +++ b/test/e2e/e2e_podswithpvc_test.go @@ -289,21 +289,7 @@ func TestProtectPodsWithPVC(t *testing.T) { t.Fatalf("Error creating %q CM: %v", policycm.Name, err) } - t.Logf("creating %q policy CM with PodsWithPVC protection enabled...", policycm.Name) - if _, err = cli.CoreV1().ConfigMaps(policycm.Namespace).Create( - ctx, policycm, metav1.CreateOptions{}, - ); err != nil { - t.Fatalf("error creating %q CM: %v", policycm.Name, err) - } - - defer func() { - t.Logf("deleting %q CM...", policycm.Name) - if err := cli.CoreV1().ConfigMaps(policycm.Namespace).Delete( - ctx, policycm.Name, metav1.DeleteOptions{}, - ); err != nil { - t.Fatalf("unable to delete %q CM: %v", policycm.Name, err) - } - }() + createConfigMapWithCleanup(t, ctx, cli, policycm) desdep := deschedulerDeployment(namespace.Name) t.Logf("creating descheduler deployment %v", desdep.Name) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 09f0fbdf4..0cf6b8245 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -31,6 +31,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -84,7 +85,12 @@ func TestMain(m *testing.M) { } func isClientRateLimiterError(err error) bool { - return strings.Contains(err.Error(), "client rate limiter") + if err == nil { + return false + } + return strings.Contains(err.Error(), "client rate limiter") && + !strings.Contains(err.Error(), "context deadline exceeded") && + !strings.Contains(err.Error(), "context canceled") } func initFeatureGates() featuregate.FeatureGate { @@ -112,6 +118,41 @@ func deschedulerPolicyConfigMap(policy *deschedulerapiv1alpha2.DeschedulerPolicy return cm, nil } +// createPolicyConfigMap generates, creates (or recreates if already existing), and registers cleanup for a descheduler policy ConfigMap. +func createPolicyConfigMap(t *testing.T, ctx context.Context, kubeClient clientset.Interface, policy *deschedulerapiv1alpha2.DeschedulerPolicy) *v1.ConfigMap { + t.Helper() + cm, err := deschedulerPolicyConfigMap(policy) + if err != nil { + t.Fatalf("Error creating policy CM object: %v", err) + } + return createConfigMapWithCleanup(t, ctx, kubeClient, cm) +} + +// createConfigMapWithCleanup creates (or recreates if already existing) a ConfigMap and registers a t.Cleanup callback to delete it. +func createConfigMapWithCleanup(t *testing.T, ctx context.Context, kubeClient clientset.Interface, cm *v1.ConfigMap) *v1.ConfigMap { + t.Helper() + t.Logf("Creating %q policy CM...", cm.Name) + _, err := kubeClient.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + _ = kubeClient.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{}) + _, err = kubeClient.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{}) + } + if err != nil { + t.Fatalf("Error creating %q CM: %v", cm.Name, err) + } + } + + t.Cleanup(func() { + t.Logf("Deleting %q CM...", cm.Name) + if err := kubeClient.CoreV1().ConfigMaps(cm.Namespace).Delete(context.Background(), cm.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Logf("Unable to delete %q CM: %v", cm.Name, err) + } + }) + + return cm +} + func deschedulerDeployment(testName string) *appsv1.Deployment { deploymentObject := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index e48bb4413..d4fee2b43 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -153,24 +153,7 @@ func TestTooManyRestarts(t *testing.T) { rs.DefaultFeatureGates = initFeatureGates() preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) - // Deploy the descheduler with the configured policy - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tc.policy) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name) - _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - defer func() { - t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() + createPolicyConfigMap(t, ctx, clientSet, tc.policy) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index 46170673f..3a10c2942 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -219,24 +219,7 @@ func TestTopologySpreadConstraint(t *testing.T) { Include: []string{testNamespace.Name}, }, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs)) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name) - _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - - defer func() { - t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() + createPolicyConfigMap(t, ctx, clientSet, topologySpreadConstraintPolicy(constraintArgs, evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) From b5a214909c3a81ac0ea090a6daca0e7a7e827b45 Mon Sep 17 00:00:00 2001 From: Amir Alavi Date: Sun, 9 Aug 2026 12:35:43 -0400 Subject: [PATCH 3/3] Fix e2e test descheduler deployment cleanup and KubeVirt live migration wait --- test/e2e/e2e_duplicatepods_test.go | 27 +---- test/e2e/e2e_evictioninbackground_test.go | 107 +++++++----------- test/e2e/e2e_failedpods_test.go | 27 +---- test/e2e/e2e_lownodeutilization_test.go | 41 ++----- test/e2e/e2e_test.go | 37 ++++++ test/e2e/e2e_toomanyrestarts_test.go | 26 +---- test/e2e/e2e_topologyspreadconstraint_test.go | 26 +---- 7 files changed, 90 insertions(+), 201 deletions(-) diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 3db674666..ff68dde34 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -192,32 +192,7 @@ func TestRemoveDuplicates(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemoveDuplicates strategy var meetsExpectations bool diff --git a/test/e2e/e2e_evictioninbackground_test.go b/test/e2e/e2e_evictioninbackground_test.go index 47a6625cb..236bead2e 100644 --- a/test/e2e/e2e_evictioninbackground_test.go +++ b/test/e2e/e2e_evictioninbackground_test.go @@ -106,86 +106,51 @@ func formatContainerStatuses(pod *corev1.Pod) string { return strings.Join(parts, "; ") } -// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable -// condition with status True. If a VMI fails to become migratable within the -// per-attempt timeout, it is deleted and recreated. This works around an -// upstream KubeVirt race where virt-handler computes the containerdisk -// checksum before the disk socket is ready, fails, and never retries; the -// recreated VMI lands on a node that already has the containerdisk image -// cached, so the socket comes up before virt-handler's first attempt. -// See https://github.com/kubernetes-sigs/descheduler/pull/1874 for context. +// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable condition with status True. func ensureVMIsLiveMigratable(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface, namespace string) { t.Helper() - const ( - maxAttempts = 3 - perAttemptWait = 120 * time.Second - deleteWait = 60 * time.Second - ) - - isLiveMigratable := func(vmi *kvcorev1.VirtualMachineInstance) bool { - for _, c := range vmi.Status.Conditions { - if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue { - return true - } + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, true, func(ctx context.Context) (bool, error) { + vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).List(ctx, metav1.ListOptions{}) + if err != nil || len(vmiList.Items) != vmiCount { + return false, nil } - return false - } - - for i := 1; i <= vmiCount; i++ { - name := fmt.Sprintf("kubevirtvmi-%v", i) - var lastVMI *kvcorev1.VirtualMachineInstance - for attempt := 1; attempt <= maxAttempts; attempt++ { - err := wait.PollUntilContextTimeout(ctx, 5*time.Second, perAttemptWait, true, func(ctx context.Context) (bool, error) { - vmi, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - klog.Infof("Unable to get vmi %v: %v", name, err) - return false, nil - } - lastVMI = vmi - return isLiveMigratable(vmi), nil - }) - if err == nil { - klog.Infof("vmi %v is LiveMigratable (attempt %d/%d)", name, attempt, maxAttempts) - break - } - if attempt == maxAttempts { - if lastVMI != nil { - klog.Infof("Final vmi %v status: phase=%v, conditions=%#v", name, lastVMI.Status.Phase, lastVMI.Status.Conditions) + for _, vmi := range vmiList.Items { + migratable := false + for _, c := range vmi.Status.Conditions { + if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue { + migratable = true + break } - t.Fatalf("vmi %v never became LiveMigratable after %d attempts", name, maxAttempts) - } - klog.Warningf("vmi %v not LiveMigratable after %v, recreating (attempt %d/%d) to work around virt-handler containerdisk-socket race", name, perAttemptWait, attempt, maxAttempts) - if err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - t.Fatalf("Unable to delete vmi %v for retry: %v", name, err) - } - if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, deleteWait, true, func(ctx context.Context) (bool, error) { - _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{}) - return apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("Timed out waiting for vmi %v to be deleted: %v", name, err) } - if _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Create(ctx, virtualMachineInstance(i, namespace), metav1.CreateOptions{}); err != nil { - t.Fatalf("Unable to recreate vmi %v: %v", name, err) + if !migratable { + return false, nil } } + return true, nil + }) + if err != nil { + t.Fatalf("VMIs never became LiveMigratable: %v", err) } + klog.Infof("All VMIs are LiveMigratable") } func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface) { - obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{}) - if err != nil { - t.Fatalf("Unable to get kubevirt/kubevirt: %v", err) - } - available := false - for _, condition := range obj.Status.Conditions { - if condition.Type == kvcorev1.KubeVirtConditionAvailable { - if condition.Status == corev1.ConditionTrue { - available = true + t.Helper() + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, true, func(ctx context.Context) (bool, error) { + obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{}) + if err != nil { + klog.Infof("Unable to get kubevirt/kubevirt: %v", err) + return false, nil + } + for _, condition := range obj.Status.Conditions { + if condition.Type == kvcorev1.KubeVirtConditionAvailable && condition.Status == corev1.ConditionTrue { + return true, nil } } - } - if !available { - t.Fatalf("Kubevirt is not available") + return false, nil + }) + if err != nil { + t.Fatalf("Kubevirt is not available: %v", err) } klog.Infof("Kubevirt is available") } @@ -415,7 +380,13 @@ func createAndWaitForDeschedulerRunning(t *testing.T, ctx context.Context, kubeC klog.Infof("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err := kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) + if apierrors.IsAlreadyExists(err) { + _ = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + _, err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) + } + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) + } } klog.Infof("Waiting for the descheduler pod running") diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index bf82622df..c3e06a40b 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -9,7 +9,6 @@ import ( batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -149,31 +148,7 @@ func TestFailedPods(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - t.Cleanup(func() { - if deschedulerPodName != "" { - printPodLogs(context.Background(), t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }) - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemoveDuplicates strategy var meetsExpectations bool diff --git a/test/e2e/e2e_lownodeutilization_test.go b/test/e2e/e2e_lownodeutilization_test.go index 225143ee7..1a79b7c03 100644 --- a/test/e2e/e2e_lownodeutilization_test.go +++ b/test/e2e/e2e_lownodeutilization_test.go @@ -25,7 +25,6 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -106,7 +105,10 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil { t.Fatalf("Unable to create ns %v: %v", testNamespace.Name, err) } - defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) + + t.Cleanup(func() { + clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) + }) t.Log("Creating duplicates pods") testLabel := map[string]string{"app": "test-lownodeutilization-kubernetes-metrics", "name": "test-lownodeutilization-kubernetes-metrics"} @@ -165,11 +167,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { expectedEvictedPodCount: 2, lowNodeUtilizationArgs: &nodeutilization.LowNodeUtilizationArgs{ Thresholds: api.ResourceThresholds{ - v1.ResourceCPU: 10, + v1.ResourceCPU: 8, v1.ResourcePods: 30, }, TargetThresholds: api.ResourceThresholds{ - v1.ResourceCPU: 20, + v1.ResourceCPU: 15, v1.ResourcePods: 50, }, MetricsUtilization: &nodeutilization.MetricsUtilization{ @@ -195,10 +197,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { } return } - defer func() { + + t.Cleanup(func() { clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace) - }() + }) waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace) // wait until workerNodes[0].Name has the right actual cpu utilization and all the testing pods are running // and producing ~4 cores in total @@ -239,31 +242,7 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - t.Cleanup(func() { - if deschedulerPodName != "" { - printPodLogs(context.Background(), t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }) - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run LowNodeUtilization plugin var meetsExpectations bool diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0cf6b8245..2f161fd08 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -153,6 +153,43 @@ func createConfigMapWithCleanup(t *testing.T, ctx context.Context, kubeClient cl return cm } +// createDeschedulerDeploymentWithCleanup creates (or recreates if already existing) a descheduler deployment and registers a t.Cleanup callback. +func createDeschedulerDeploymentWithCleanup(t *testing.T, ctx context.Context, kubeClient clientset.Interface, deployment *appsv1.Deployment) string { + t.Helper() + t.Logf("Creating descheduler deployment %v", deployment.Name) + _, err := kubeClient.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + _ = kubeClient.AppsV1().Deployments(deployment.Namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{}) + _, err = kubeClient.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + } + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deployment.Name, err) + } + } + + deschedulerPodName := "" + t.Cleanup(func() { + if deschedulerPodName != "" { + printPodLogs(context.Background(), t, kubeClient, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deployment.Name) + if err := kubeClient.AppsV1().Deployments(deployment.Namespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Logf("Unable to delete %q deployment: %v", deployment.Name, err) + } + + waitForPodsToDisappear(context.Background(), t, kubeClient, deployment.Labels, deployment.Namespace) + }) + + t.Logf("Waiting for the descheduler pod running") + deschedulerPods := waitForPodsRunning(ctx, t, kubeClient, deployment.Labels, 1, deployment.Namespace) + if len(deschedulerPods) != 0 { + deschedulerPodName = deschedulerPods[0].Name + } + return deschedulerPodName +} + func deschedulerDeployment(testName string) *appsv1.Deployment { deploymentObject := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index d4fee2b43..db264314f 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -156,31 +156,7 @@ func TestTooManyRestarts(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, tc.policy) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemovePodsHavingTooManyRestarts strategy if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (bool, error) { currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index 3a10c2942..129cc865b 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -221,31 +221,7 @@ func TestTopologySpreadConstraint(t *testing.T) { } createPolicyConfigMap(t, ctx, clientSet, topologySpreadConstraintPolicy(constraintArgs, evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemovePodsHavingTooManyRestarts strategy var meetsEvictedExpectations bool