Fix e2e test descheduler deployment cleanup and KubeVirt live migration wait

test/e2e/e2e_duplicatepods_test.go

@@ -192,32 +192,7 @@ func TestRemoveDuplicates(t *testing.T) {
 			createPolicyConfigMap(t, ctx, clientSet, removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs))
 
 			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
-			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
-			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
-			}
-
-			deschedulerPodName := ""
-			defer func() {
-				if deschedulerPodName != "" {
-					printPodLogs(ctx, t, clientSet, deschedulerPodName)
-				}
-
-				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
-				err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
-				if err != nil {
-					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
-				}
-
-				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
-			}()
-
-			t.Logf("Waiting for the descheduler pod running")
-			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
-			if len(deschedulerPods) != 0 {
-				deschedulerPodName = deschedulerPods[0].Name
-			}
+			createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj)
 
 			// Run RemoveDuplicates strategy
 			var meetsExpectations bool

test/e2e/e2e_evictioninbackground_test.go

@@ -106,86 +106,51 @@ func formatContainerStatuses(pod *corev1.Pod) string {
 	return strings.Join(parts, "; ")
 }
 
-// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable
-// condition with status True. If a VMI fails to become migratable within the
-// per-attempt timeout, it is deleted and recreated. This works around an
-// upstream KubeVirt race where virt-handler computes the containerdisk
-// checksum before the disk socket is ready, fails, and never retries; the
-// recreated VMI lands on a node that already has the containerdisk image
-// cached, so the socket comes up before virt-handler's first attempt.
-// See https://github.com/kubernetes-sigs/descheduler/pull/1874 for context.
+// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable condition with status True.
 func ensureVMIsLiveMigratable(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface, namespace string) {
 	t.Helper()
-	const (
-		maxAttempts    = 3
-		perAttemptWait = 120 * time.Second
-		deleteWait     = 60 * time.Second
-	)
-
-	isLiveMigratable := func(vmi *kvcorev1.VirtualMachineInstance) bool {
-		for _, c := range vmi.Status.Conditions {
-			if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue {
-				return true
-			}
+	err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, true, func(ctx context.Context) (bool, error) {
+		vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).List(ctx, metav1.ListOptions{})
+		if err != nil || len(vmiList.Items) != vmiCount {
+			return false, nil
 		}
-		return false
-	}
-
-	for i := 1; i <= vmiCount; i++ {
-		name := fmt.Sprintf("kubevirtvmi-%v", i)
-		var lastVMI *kvcorev1.VirtualMachineInstance
-		for attempt := 1; attempt <= maxAttempts; attempt++ {
-			err := wait.PollUntilContextTimeout(ctx, 5*time.Second, perAttemptWait, true, func(ctx context.Context) (bool, error) {
-				vmi, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{})
-				if err != nil {
-					klog.Infof("Unable to get vmi %v: %v", name, err)
-					return false, nil
-				}
-				lastVMI = vmi
-				return isLiveMigratable(vmi), nil
-			})
-			if err == nil {
-				klog.Infof("vmi %v is LiveMigratable (attempt %d/%d)", name, attempt, maxAttempts)
-				break
-			}
-			if attempt == maxAttempts {
-				if lastVMI != nil {
-					klog.Infof("Final vmi %v status: phase=%v, conditions=%#v", name, lastVMI.Status.Phase, lastVMI.Status.Conditions)
+		for _, vmi := range vmiList.Items {
+			migratable := false
+			for _, c := range vmi.Status.Conditions {
+				if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue {
+					migratable = true
+					break
 				}
-				t.Fatalf("vmi %v never became LiveMigratable after %d attempts", name, maxAttempts)
-			}
-			klog.Warningf("vmi %v not LiveMigratable after %v, recreating (attempt %d/%d) to work around virt-handler containerdisk-socket race", name, perAttemptWait, attempt, maxAttempts)
-			if err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
-				t.Fatalf("Unable to delete vmi %v for retry: %v", name, err)
-			}
-			if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, deleteWait, true, func(ctx context.Context) (bool, error) {
-				_, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{})
-				return apierrors.IsNotFound(err), nil
-			}); err != nil {
-				t.Fatalf("Timed out waiting for vmi %v to be deleted: %v", name, err)
 			}
-			if _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Create(ctx, virtualMachineInstance(i, namespace), metav1.CreateOptions{}); err != nil {
-				t.Fatalf("Unable to recreate vmi %v: %v", name, err)
+			if !migratable {
+				return false, nil
 			}
 		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("VMIs never became LiveMigratable: %v", err)
 	}
+	klog.Infof("All VMIs are LiveMigratable")
 }
 
 func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface) {
-	obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{})
-	if err != nil {
-		t.Fatalf("Unable to get kubevirt/kubevirt: %v", err)
-	}
-	available := false
-	for _, condition := range obj.Status.Conditions {
-		if condition.Type == kvcorev1.KubeVirtConditionAvailable {
-			if condition.Status == corev1.ConditionTrue {
-				available = true
+	t.Helper()
+	err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, true, func(ctx context.Context) (bool, error) {
+		obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{})
+		if err != nil {
+			klog.Infof("Unable to get kubevirt/kubevirt: %v", err)
+			return false, nil
+		}
+		for _, condition := range obj.Status.Conditions {
+			if condition.Type == kvcorev1.KubeVirtConditionAvailable && condition.Status == corev1.ConditionTrue {
+				return true, nil
 			}
 		}
-	}
-	if !available {
-		t.Fatalf("Kubevirt is not available")
+		return false, nil
+	})
+	if err != nil {
+		t.Fatalf("Kubevirt is not available: %v", err)
 	}
 	klog.Infof("Kubevirt is available")
 }
@@ -415,7 +380,13 @@ func createAndWaitForDeschedulerRunning(t *testing.T, ctx context.Context, kubeC
 	klog.Infof("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
 	_, err := kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
 	if err != nil {
-		t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+		if apierrors.IsAlreadyExists(err) {
+			_ = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
+			_, err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
+		}
+		if err != nil {
+			t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+		}
 	}
 
 	klog.Infof("Waiting for the descheduler pod running")

test/e2e/e2e_failedpods_test.go

@@ -9,7 +9,6 @@ import (
 
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -149,31 +148,7 @@ func TestFailedPods(t *testing.T) {
 			createPolicyConfigMap(t, ctx, clientSet, removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs))
 
 			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
-			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
-			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
-			}
-
-			deschedulerPodName := ""
-			t.Cleanup(func() {
-				if deschedulerPodName != "" {
-					printPodLogs(context.Background(), t, clientSet, deschedulerPodName)
-				}
-
-				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
-				if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
-					t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
-				}
-
-				waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
-			})
-
-			t.Logf("Waiting for the descheduler pod running")
-			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
-			if len(deschedulerPods) != 0 {
-				deschedulerPodName = deschedulerPods[0].Name
-			}
+			createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj)
 
 			// Run RemoveDuplicates strategy
 			var meetsExpectations bool

test/e2e/e2e_lownodeutilization_test.go

@@ -25,7 +25,6 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -106,7 +105,10 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
 	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
 		t.Fatalf("Unable to create ns %v: %v", testNamespace.Name, err)
 	}
-	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})
+
+	t.Cleanup(func() {
+		clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})
+	})
 
 	t.Log("Creating duplicates pods")
 	testLabel := map[string]string{"app": "test-lownodeutilization-kubernetes-metrics", "name": "test-lownodeutilization-kubernetes-metrics"}
@@ -165,11 +167,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
 			expectedEvictedPodCount: 2,
 			lowNodeUtilizationArgs: &nodeutilization.LowNodeUtilizationArgs{
 				Thresholds: api.ResourceThresholds{
-					v1.ResourceCPU:  10,
+					v1.ResourceCPU:  8,
 					v1.ResourcePods: 30,
 				},
 				TargetThresholds: api.ResourceThresholds{
-					v1.ResourceCPU:  20,
+					v1.ResourceCPU:  15,
 					v1.ResourcePods: 50,
 				},
 				MetricsUtilization: &nodeutilization.MetricsUtilization{
@@ -195,10 +197,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
 				}
 				return
 			}
-			defer func() {
+
+			t.Cleanup(func() {
 				clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{})
 				waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace)
-			}()
+			})
 			waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace)
 			// wait until workerNodes[0].Name has the right actual cpu utilization and all the testing pods are running
 			// and producing ~4 cores in total
@@ -239,31 +242,7 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
 			createPolicyConfigMap(t, ctx, clientSet, lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled))
 
 			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
-			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
-			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
-			}
-
-			deschedulerPodName := ""
-			t.Cleanup(func() {
-				if deschedulerPodName != "" {
-					printPodLogs(context.Background(), t, clientSet, deschedulerPodName)
-				}
-
-				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
-				if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
-					t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
-				}
-
-				waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
-			})
-
-			t.Logf("Waiting for the descheduler pod running")
-			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
-			if len(deschedulerPods) != 0 {
-				deschedulerPodName = deschedulerPods[0].Name
-			}
+			createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj)
 
 			// Run LowNodeUtilization plugin
 			var meetsExpectations bool

test/e2e/e2e_test.go

@@ -153,6 +153,43 @@ func createConfigMapWithCleanup(t *testing.T, ctx context.Context, kubeClient cl
 	return cm
 }
 
+// createDeschedulerDeploymentWithCleanup creates (or recreates if already existing) a descheduler deployment and registers a t.Cleanup callback.
+func createDeschedulerDeploymentWithCleanup(t *testing.T, ctx context.Context, kubeClient clientset.Interface, deployment *appsv1.Deployment) string {
+	t.Helper()
+	t.Logf("Creating descheduler deployment %v", deployment.Name)
+	_, err := kubeClient.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			_ = kubeClient.AppsV1().Deployments(deployment.Namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{})
+			_, err = kubeClient.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})
+		}
+		if err != nil {
+			t.Fatalf("Error creating %q deployment: %v", deployment.Name, err)
+		}
+	}
+
+	deschedulerPodName := ""
+	t.Cleanup(func() {
+		if deschedulerPodName != "" {
+			printPodLogs(context.Background(), t, kubeClient, deschedulerPodName)
+		}
+
+		t.Logf("Deleting %q deployment...", deployment.Name)
+		if err := kubeClient.AppsV1().Deployments(deployment.Namespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+			t.Logf("Unable to delete %q deployment: %v", deployment.Name, err)
+		}
+
+		waitForPodsToDisappear(context.Background(), t, kubeClient, deployment.Labels, deployment.Namespace)
+	})
+
+	t.Logf("Waiting for the descheduler pod running")
+	deschedulerPods := waitForPodsRunning(ctx, t, kubeClient, deployment.Labels, 1, deployment.Namespace)
+	if len(deschedulerPods) != 0 {
+		deschedulerPodName = deschedulerPods[0].Name
+	}
+	return deschedulerPodName
+}
+
 func deschedulerDeployment(testName string) *appsv1.Deployment {
 	deploymentObject := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{

test/e2e/e2e_toomanyrestarts_test.go

@@ -156,31 +156,7 @@ func TestTooManyRestarts(t *testing.T) {
 			createPolicyConfigMap(t, ctx, clientSet, tc.policy)
 
 			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
-			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
-			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
-			}
-
-			deschedulerPodName := ""
-			defer func() {
-				if deschedulerPodName != "" {
-					printPodLogs(ctx, t, clientSet, deschedulerPodName)
-				}
-
-				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
-				err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
-				if err != nil {
-					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
-				}
-				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
-			}()
-
-			t.Logf("Waiting for the descheduler pod running")
-			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
-			if len(deschedulerPods) != 0 {
-				deschedulerPodName = deschedulerPods[0].Name
-			}
+			createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj)
 			// Run RemovePodsHavingTooManyRestarts strategy
 			if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (bool, error) {
 				currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)

test/e2e/e2e_topologyspreadconstraint_test.go

@@ -221,31 +221,7 @@ func TestTopologySpreadConstraint(t *testing.T) {
 			}
 			createPolicyConfigMap(t, ctx, clientSet, topologySpreadConstraintPolicy(constraintArgs, evictorArgs))
 			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
-			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
-			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
-			}
-
-			deschedulerPodName := ""
-			defer func() {
-				if deschedulerPodName != "" {
-					printPodLogs(ctx, t, clientSet, deschedulerPodName)
-				}
-
-				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
-				err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
-				if err != nil {
-					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
-				}
-				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
-			}()
-
-			t.Logf("Waiting for the descheduler pod running")
-			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
-			if len(deschedulerPods) != 0 {
-				deschedulerPodName = deschedulerPods[0].Name
-			}
+			createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj)
 
 			// Run RemovePodsHavingTooManyRestarts strategy
 			var meetsEvictedExpectations bool