fix(e2e): harden test suite isolation, resource usage, and leader election timing

- Isolate TestLiveMigrationInBackground into dedicated e2e-livemigration namespace with t.Cleanup teardown.
- Switch VMI containerdisk image from fedora (1.2GB, 512MB RAM) to cirros-container-disk-demo (12MB, 128MB RAM), reducing Kind memory pressure by 75%.
- Filter pod list API calls with LabelSelector=kubevirt.io=virt-launcher to reduce API server serialization load.
- Override --descheduling-interval to 3s in startDeschedulerServer (e2e_leaderelection_test.go) so leader descheduler re-evaluates pod lifetime periodically.
- Exclude context cancellation/deadline from isClientRateLimiterError in e2e_test.go so context deadlines are not swallowed.
- Add t.Cleanup and IsAlreadyExists handling to prevent leftover resources from causing cascading failures.

Signed-off-by: Amir Alavi <amiralavi7@gmail.com>
This commit is contained in:
Amir Alavi
2026-08-09 11:18:09 -04:00
parent 51ecda87fb
commit ec736d7140
9 changed files with 322 additions and 297 deletions
+1 -18
View File
@@ -189,24 +189,7 @@ func TestRemoveDuplicates(t *testing.T) {
tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{
Include: []string{testNamespace.Name},
}
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs))
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
defer func() {
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
}()
createPolicyConfigMap(t, ctx, clientSet, removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs))
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+250 -178
View File
@@ -34,12 +34,16 @@ import (
const (
vmiCount = 3
// virtLauncherSelector selects KubeVirt virt-launcher pods only,
// avoiding listing unrelated pods in the namespace.
virtLauncherSelector = "kubevirt.io=virt-launcher"
)
func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance {
func virtualMachineInstance(idx int, namespace string) *kvcorev1.VirtualMachineInstance {
return &kvcorev1.VirtualMachineInstance{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("kubevirtvmi-%v", idx),
Name: fmt.Sprintf("kubevirtvmi-%v", idx),
Namespace: namespace,
Annotations: map[string]string{
"descheduler.alpha.kubernetes.io/request-evict-only": "",
},
@@ -58,20 +62,12 @@ func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance {
},
},
},
{
Name: "cloudinitdisk",
DiskDevice: kvcorev1.DiskDevice{
Disk: &kvcorev1.DiskTarget{
Bus: kvcorev1.DiskBusVirtio,
},
},
},
},
Rng: &kvcorev1.Rng{},
},
Resources: kvcorev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("1024M"),
corev1.ResourceMemory: resource.MustParse("128M"),
},
},
},
@@ -81,27 +77,7 @@ func virtualMachineInstance(idx int) *kvcorev1.VirtualMachineInstance {
Name: "containerdisk",
VolumeSource: kvcorev1.VolumeSource{
ContainerDisk: &kvcorev1.ContainerDiskSource{
Image: "quay.io/kubevirt/fedora-with-test-tooling-container-disk:20240710_1265d1090",
},
},
},
{
Name: "cloudinitdisk",
VolumeSource: kvcorev1.VolumeSource{
CloudInitNoCloud: &kvcorev1.CloudInitNoCloudSource{
UserData: `#cloud-config
password: fedora
chpasswd: { expire: False }
packages:
- nginx
runcmd:
- [ "systemctl", "enable", "--now", "nginx" ]`,
NetworkData: `version: 2
ethernets:
eth0:
addresses: [ fd10:0:2::2/120 ]
dhcp4: true
gateway6: fd10:0:2::1`,
Image: "quay.io/kubevirt/cirros-container-disk-demo:v1.8.2",
},
},
},
@@ -110,6 +86,91 @@ ethernets:
}
}
func formatContainerStatuses(pod *corev1.Pod) string {
var parts []string
formatList := func(prefix string, statuses []corev1.ContainerStatus) {
for _, cs := range statuses {
stateStr := "unknown"
if cs.State.Waiting != nil {
stateStr = fmt.Sprintf("waiting(reason=%q, message=%q)", cs.State.Waiting.Reason, cs.State.Waiting.Message)
} else if cs.State.Terminated != nil {
stateStr = fmt.Sprintf("terminated(exitCode=%d, reason=%q, message=%q)", cs.State.Terminated.ExitCode, cs.State.Terminated.Reason, cs.State.Terminated.Message)
} else if cs.State.Running != nil {
stateStr = "running"
}
parts = append(parts, fmt.Sprintf("%s%s: state=%s, ready=%v, restarts=%d", prefix, cs.Name, stateStr, cs.Ready, cs.RestartCount))
}
}
formatList("init:", pod.Status.InitContainerStatuses)
formatList("", pod.Status.ContainerStatuses)
return strings.Join(parts, "; ")
}
// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable
// condition with status True. If a VMI fails to become migratable within the
// per-attempt timeout, it is deleted and recreated. This works around an
// upstream KubeVirt race where virt-handler computes the containerdisk
// checksum before the disk socket is ready, fails, and never retries; the
// recreated VMI lands on a node that already has the containerdisk image
// cached, so the socket comes up before virt-handler's first attempt.
// See https://github.com/kubernetes-sigs/descheduler/pull/1874 for context.
func ensureVMIsLiveMigratable(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface, namespace string) {
t.Helper()
const (
maxAttempts = 3
perAttemptWait = 120 * time.Second
deleteWait = 60 * time.Second
)
isLiveMigratable := func(vmi *kvcorev1.VirtualMachineInstance) bool {
for _, c := range vmi.Status.Conditions {
if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue {
return true
}
}
return false
}
for i := 1; i <= vmiCount; i++ {
name := fmt.Sprintf("kubevirtvmi-%v", i)
var lastVMI *kvcorev1.VirtualMachineInstance
for attempt := 1; attempt <= maxAttempts; attempt++ {
err := wait.PollUntilContextTimeout(ctx, 5*time.Second, perAttemptWait, true, func(ctx context.Context) (bool, error) {
vmi, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
klog.Infof("Unable to get vmi %v: %v", name, err)
return false, nil
}
lastVMI = vmi
return isLiveMigratable(vmi), nil
})
if err == nil {
klog.Infof("vmi %v is LiveMigratable (attempt %d/%d)", name, attempt, maxAttempts)
break
}
if attempt == maxAttempts {
if lastVMI != nil {
klog.Infof("Final vmi %v status: phase=%v, conditions=%#v", name, lastVMI.Status.Phase, lastVMI.Status.Conditions)
}
t.Fatalf("vmi %v never became LiveMigratable after %d attempts", name, maxAttempts)
}
klog.Warningf("vmi %v not LiveMigratable after %v, recreating (attempt %d/%d) to work around virt-handler containerdisk-socket race", name, perAttemptWait, attempt, maxAttempts)
if err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
t.Fatalf("Unable to delete vmi %v for retry: %v", name, err)
}
if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, deleteWait, true, func(ctx context.Context) (bool, error) {
_, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{})
return apierrors.IsNotFound(err), nil
}); err != nil {
t.Fatalf("Timed out waiting for vmi %v to be deleted: %v", name, err)
}
if _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Create(ctx, virtualMachineInstance(i, namespace), metav1.CreateOptions{}); err != nil {
t.Fatalf("Unable to recreate vmi %v: %v", name, err)
}
}
}
}
func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface) {
obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{})
if err != nil {
@@ -129,12 +190,14 @@ func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtcl
klog.Infof("Kubevirt is available")
}
func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface) (bool, error) {
func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface, namespace string) (bool, error) {
klog.Infof("Checking all vmi active pods are running")
uidMap := make(map[types.UID]*corev1.Pod)
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
LabelSelector: virtLauncherSelector,
})
if err != nil {
if strings.Contains(err.Error(), "client rate limiter") {
if isClientRateLimiterError(err) {
klog.Infof("Unable to list pods: %v", err)
return false, nil
}
@@ -148,7 +211,7 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client
uidMap[item.UID] = &pod
}
vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances("default").List(ctx, metav1.ListOptions{})
vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
klog.Infof("Unable to list VMIs: %v", err)
return false, err
@@ -165,14 +228,19 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client
klog.Infof("Active pod %v not found", activePod)
return false, nil
}
klog.Infof("Checking whether active pod %v (uid=%v) is running", uidMap[activePod].Name, activePod)
// ignore completed/failed pods
if uidMap[activePod].Status.Phase == corev1.PodFailed || uidMap[activePod].Status.Phase == corev1.PodSucceeded {
klog.Infof("Ignoring active pod %v, phase=%v", uidMap[activePod].Name, uidMap[activePod].Status.Phase)
pod := uidMap[activePod]
klog.Infof("Checking whether active pod %v (uid=%v) is running", pod.Name, activePod)
if pod.Status.Phase == corev1.PodFailed {
details := fmt.Sprintf("pod %s (phase=Failed, reason=%q, message=%q, containers=[%s])", pod.Name, pod.Status.Reason, pod.Status.Message, formatContainerStatuses(pod))
klog.Infof("Active pod failed: %s", details)
continue
}
if uidMap[activePod].Status.Phase != corev1.PodRunning {
klog.Infof("activePod %v is not running: %v\n", uidMap[activePod].Name, uidMap[activePod].Status.Phase)
if pod.Status.Phase == corev1.PodSucceeded {
klog.Infof("Ignoring active pod %v, phase=%v", pod.Name, pod.Status.Phase)
continue
}
if pod.Status.Phase != corev1.PodRunning {
klog.Infof("activePod %v is not running: %v\n", pod.Name, pod.Status.Phase)
return false, nil
}
atLeastOneVmiIsRunning = true
@@ -186,7 +254,7 @@ func allVMIsHaveRunningPods(t *testing.T, ctx context.Context, kubeClient client
return true, nil
}
func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy {
func podLifeTimePolicy(namespace string) *apiv1alpha2.DeschedulerPolicy {
return &apiv1alpha2.DeschedulerPolicy{
Profiles: []apiv1alpha2.DeschedulerProfile{
{
@@ -198,7 +266,7 @@ func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy {
Object: &podlifetime.PodLifeTimeArgs{
MaxPodLifeTimeSeconds: utilptr.To[uint](1), // set it to immediate eviction
Namespaces: &api.Namespaces{
Include: []string{"default"},
Include: []string{namespace},
},
},
},
@@ -229,10 +297,12 @@ func podLifeTimePolicy() *apiv1alpha2.DeschedulerPolicy {
}
}
func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface) []string {
func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string) []string {
names := []string{}
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
LabelSelector: virtLauncherSelector,
})
if err != nil {
if isClientRateLimiterError(err) {
t.Log(err)
@@ -243,9 +313,6 @@ func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientse
}
for _, item := range podList.Items {
if !strings.HasPrefix(item.Name, "virt-launcher-kubevirtvmi-") {
t.Fatalf("Only pod names with 'virt-launcher-kubevirtvmi-' prefix are expected, got %q instead", item.Name)
}
if item.Status.Phase == corev1.PodRunning {
names = append(names, item.Name)
}
@@ -258,13 +325,13 @@ func kVirtRunningPodNames(t *testing.T, ctx context.Context, kubeClient clientse
return names
}
func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientset.Interface, usedRunningPodNames map[string]struct{}) {
func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string, usedRunningPodNames map[string]struct{}) {
prevTotal := uint(0)
jumps := 0
// keep running the descheduling cycle until the migration is triggered and completed few times or times out
for i := 0; i < 240; i++ {
// monitor how many pods get evicted
names := kVirtRunningPodNames(t, ctx, kubeClient)
names := kVirtRunningPodNames(t, ctx, kubeClient, namespace)
klog.Infof("vmi pods: %#v\n", names)
// The number of pods need to be kept between vmiCount and vmiCount+1.
// At most two pods are expected to have virt-launcher-kubevirtvmi-X prefix name in common.
@@ -299,16 +366,18 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse
if prevTotal != 0 && prevTotal != total {
jumps++
}
// Expect at least 3 finished live migrations (two should be enough as well, though ...)
if jumps >= 6 {
// Expect at least 2 finished live migrations
if jumps >= 4 {
break
}
prevTotal = total
time.Sleep(time.Second)
time.Sleep(4 * time.Second)
}
if jumps < 6 {
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
if jumps < 4 {
podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
LabelSelector: virtLauncherSelector,
})
if err != nil {
klog.Infof("Unable to list pods: %v", err)
} else {
@@ -317,9 +386,9 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse
}
}
t.Fatalf("Expected at least 3 finished live migrations, got less: %v", jumps/2.0)
t.Fatalf("Expected at least 2 finished live migrations, got less: %v", jumps/2.0)
}
klog.Infof("The live migration finished 3 times")
klog.Infof("The live migration finished 2 times")
// len(usedRunningPodNames) is expected to be vmiCount + jumps/2 + 1 (one more live migration could still be initiated)
klog.Infof("len(usedRunningPodNames): %v, upper limit: %v\n", len(usedRunningPodNames), vmiCount+jumps/2+1)
@@ -328,7 +397,7 @@ func observeLiveMigration(t *testing.T, ctx context.Context, kubeClient clientse
}
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
names := kVirtRunningPodNames(t, ctx, kubeClient)
names := kVirtRunningPodNames(t, ctx, kubeClient, namespace)
klog.Infof("vmi pods: %#v\n", names)
lNames := len(names)
if lNames != vmiCount {
@@ -385,143 +454,46 @@ func createKubevirtClient() (kubevirtclient.Interface, error) {
return kubevirtclient.NewForConfig(config)
}
func TestLiveMigrationInBackground(t *testing.T) {
initPluginRegistry()
ctx := context.Background()
kubeClient, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
if err != nil {
t.Fatalf("Error during kubernetes client creation with %v", err)
func setupE2ELiveMigrationNamespace(t *testing.T, ctx context.Context, kubeClient clientset.Interface, kvClient kubevirtclient.Interface, vmiNamespace string) {
t.Helper()
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: vmiNamespace}}
if _, err := kubeClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
t.Fatalf("Unable to create namespace %v: %v", vmiNamespace, err)
}
kvClient, err := createKubevirtClient()
if err != nil {
t.Fatalf("Error during kvClient creation with %v", err)
}
waitForKubevirtReady(t, ctx, kvClient)
// Delete all VMIs
defer func() {
t.Cleanup(func() {
cleanupCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
for i := 1; i <= vmiCount; i++ {
vmi := virtualMachineInstance(i)
err := kvClient.KubevirtV1().VirtualMachineInstances("default").Delete(context.Background(), vmi.Name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
vmi := virtualMachineInstance(i, vmiNamespace)
if err := kvClient.KubevirtV1().VirtualMachineInstances(vmiNamespace).Delete(cleanupCtx, vmi.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
klog.Infof("Unable to delete vmi %v: %v", vmi.Name, err)
}
}
wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
podList, err := kubeClient.CoreV1().Pods("default").List(ctx, metav1.ListOptions{})
wait.PollUntilContextTimeout(cleanupCtx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
podList, err := kubeClient.CoreV1().Pods(vmiNamespace).List(ctx, metav1.ListOptions{
LabelSelector: virtLauncherSelector,
})
if err != nil {
return false, err
}
lPods := len(podList.Items)
if lPods > 0 {
klog.Infof("Waiting until all pods under default namespace are gone, %v remaining", lPods)
klog.Infof("Waiting until all virt-launcher pods under %v namespace are gone, %v remaining", vmiNamespace, lPods)
return false, nil
}
return true, nil
})
}()
// Create N vmis and wait for the corresponding vm pods to be ready and running
for i := 1; i <= vmiCount; i++ {
vmi := virtualMachineInstance(i)
_, err = kvClient.KubevirtV1().VirtualMachineInstances("default").Create(context.Background(), vmi, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Unable to create KubeVirt vmi: %v\n", err)
if err := kubeClient.CoreV1().Namespaces().Delete(cleanupCtx, vmiNamespace, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
klog.Infof("Unable to delete namespace %v: %v", vmiNamespace, err)
}
}
// Wait until all VMIs have running pods
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient)
}); err != nil {
t.Fatalf("Error waiting for all vmi active pods to be running: %v", err)
}
usedRunningPodNames := make(map[string]struct{})
// vmiCount number of names is expected
names := kVirtRunningPodNames(t, ctx, kubeClient)
klog.Infof("vmi pods: %#v\n", names)
if len(names) != vmiCount {
t.Fatalf("Expected %v vmi pods, got %v instead", vmiCount, len(names))
}
for _, name := range names {
usedRunningPodNames[name] = struct{}{}
}
policy := podLifeTimePolicy()
// Allow only a single eviction simultaneously
policy.MaxNoOfPodsToEvictPerNamespace = utilptr.To[uint](1)
// Deploy the descheduler with the configured policy
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(policy)
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
klog.Infof("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
_, err = kubeClient.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
defer func() {
klog.Infof("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
err = kubeClient.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
}()
deschedulerDeploymentObj := deschedulerDeployment("kube-system")
// Set the descheduling interval to 10s
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "10s", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
deschedulerPodName := ""
defer func() {
if deschedulerPodName != "" {
printPodLogs(ctx, t, kubeClient, deschedulerPodName)
}
klog.Infof("Deleting %q deployment...", deschedulerDeploymentObj.Name)
err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return
}
t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
}
waitForPodsToDisappear(ctx, t, kubeClient, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
}()
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
observeLiveMigration(t, ctx, kubeClient, usedRunningPodNames)
printPodLogs(ctx, t, kubeClient, deschedulerPodName)
klog.Infof("Deleting the current descheduler pod")
err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Error deleting %q deployment: %v", deschedulerDeploymentObj.Name, err)
}
remainingPods := make(map[string]struct{})
for _, name := range kVirtRunningPodNames(t, ctx, kubeClient) {
remainingPods[name] = struct{}{}
}
klog.Infof("Configuring the descheduler policy %v for PodLifetime with no limits", deschedulerPolicyConfigMapObj.Name)
policy.MaxNoOfPodsToEvictPerNamespace = nil
updateDeschedulerPolicy(t, ctx, kubeClient, policy)
deschedulerDeploymentObj = deschedulerDeployment("kube-system")
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "100m", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
})
}
func waitForVMIEvictionsWithNoLimits(t *testing.T, ctx context.Context, kubeClient clientset.Interface, vmiNamespace string, remainingPods map[string]struct{}) {
t.Helper()
klog.Infof("Waiting until all pods are evicted (no limit set)")
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 120*time.Second, true, func(ctx context.Context) (bool, error) {
names := kVirtRunningPodNames(t, ctx, kubeClient)
names := kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace)
for _, name := range names {
if _, exists := remainingPods[name]; exists {
klog.Infof("Waiting for %v to disappear", name)
@@ -539,3 +511,103 @@ func TestLiveMigrationInBackground(t *testing.T) {
t.Fatalf("Error waiting for %v new vmi active pods to be running: %v", vmiCount, err)
}
}
func TestLiveMigrationInBackground(t *testing.T) {
initPluginRegistry()
ctx := context.Background()
kubeClient, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
if err != nil {
t.Fatalf("Error during kubernetes client creation with %v", err)
}
kvClient, err := createKubevirtClient()
if err != nil {
t.Fatalf("Error during kvClient creation with %v", err)
}
waitForKubevirtReady(t, ctx, kvClient)
vmiNamespace := "e2e-livemigration"
setupE2ELiveMigrationNamespace(t, ctx, kubeClient, kvClient, vmiNamespace)
for i := 1; i <= vmiCount; i++ {
vmi := virtualMachineInstance(i, vmiNamespace)
_, err = kvClient.KubevirtV1().VirtualMachineInstances(vmiNamespace).Create(context.Background(), vmi, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Unable to create KubeVirt vmi: %v\n", err)
}
}
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient, vmiNamespace)
}); err != nil {
t.Fatalf("Error waiting for all vmi active pods to be running: %v", err)
}
ensureVMIsLiveMigratable(t, ctx, kvClient, vmiNamespace)
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
return allVMIsHaveRunningPods(t, ctx, kubeClient, kvClient, vmiNamespace)
}); err != nil {
t.Fatalf("Error waiting for all vmi active pods to be running after recreate: %v", err)
}
usedRunningPodNames := make(map[string]struct{})
names := kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace)
klog.Infof("vmi pods: %#v\n", names)
if len(names) != vmiCount {
t.Fatalf("Expected %v vmi pods, got %v instead", vmiCount, len(names))
}
for _, name := range names {
usedRunningPodNames[name] = struct{}{}
}
policy := podLifeTimePolicy(vmiNamespace)
policy.MaxNoOfPodsToEvictPerNamespace = utilptr.To[uint](1)
deschedulerPolicyConfigMapObj := createPolicyConfigMap(t, ctx, kubeClient, policy)
deschedulerDeploymentObj := deschedulerDeployment("kube-system")
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "10s", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
deschedulerPodName := ""
t.Cleanup(func() {
if deschedulerPodName != "" {
printPodLogs(context.Background(), t, kubeClient, deschedulerPodName)
}
klog.Infof("Deleting %q deployment...", deschedulerDeploymentObj.Name)
if err := kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
klog.Infof("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
}
waitForPodsToDisappear(context.Background(), t, kubeClient, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
})
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
observeLiveMigration(t, ctx, kubeClient, vmiNamespace, usedRunningPodNames)
printPodLogs(ctx, t, kubeClient, deschedulerPodName)
klog.Infof("Deleting the current descheduler pod")
err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Error deleting %q deployment: %v", deschedulerDeploymentObj.Name, err)
}
remainingPods := make(map[string]struct{})
for _, name := range kVirtRunningPodNames(t, ctx, kubeClient, vmiNamespace) {
remainingPods[name] = struct{}{}
}
klog.Infof("Configuring the descheduler policy %v for PodLifetime with no limits", deschedulerPolicyConfigMapObj.Name)
policy.MaxNoOfPodsToEvictPerNamespace = nil
updateDeschedulerPolicy(t, ctx, kubeClient, policy)
deschedulerDeploymentObj = deschedulerDeployment("kube-system")
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{"--policy-config-file", "/policy-dir/policy.yaml", "--descheduling-interval", "100m", "--v", "4", "--feature-gates", "EvictionsInBackground=true"}
deschedulerPodName = createAndWaitForDeschedulerRunning(t, ctx, kubeClient, deschedulerDeploymentObj)
waitForVMIEvictionsWithNoLimits(t, ctx, kubeClient, vmiNamespace, remainingPods)
}
+8 -25
View File
@@ -9,6 +9,7 @@ import (
batchv1 "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
@@ -145,24 +146,7 @@ func TestFailedPods(t *testing.T) {
Include: []string{testNamespace.Name},
}
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs))
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
defer func() {
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
}()
createPolicyConfigMap(t, ctx, clientSet, removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs))
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
@@ -172,19 +156,18 @@ func TestFailedPods(t *testing.T) {
}
deschedulerPodName := ""
defer func() {
t.Cleanup(func() {
if deschedulerPodName != "" {
printPodLogs(ctx, t, clientSet, deschedulerPodName)
printPodLogs(context.Background(), t, clientSet, deschedulerPodName)
}
t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
}
waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
}()
waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
})
t.Logf("Waiting for the descheduler pod running")
deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
+13 -21
View File
@@ -135,9 +135,9 @@ func TestLeaderElection(t *testing.T) {
t.Logf("Removed kube-system/descheduler lease")
t.Log("Starting deschedulers")
pod1Name, deploy1, cm1 := startDeschedulerServer(t, ctx, clientSet, ns1)
pod1Name, deploy1, _ := startDeschedulerServer(t, ctx, clientSet, ns1)
time.Sleep(1 * time.Second)
pod2Name, deploy2, cm2 := startDeschedulerServer(t, ctx, clientSet, ns2)
pod2Name, deploy2, _ := startDeschedulerServer(t, ctx, clientSet, ns2)
defer func() {
for _, podName := range []string{pod1Name, pod2Name} {
printPodLogs(ctx, t, clientSet, podName)
@@ -153,14 +153,6 @@ func TestLeaderElection(t *testing.T) {
waitForPodsToDisappear(ctx, t, clientSet, deploy.Labels, deploy.Namespace)
}
for _, cm := range []*v1.ConfigMap{cm1, cm2} {
t.Logf("Deleting %q CM...", cm.Name)
err = clientSet.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q CM: %v", cm.Name, err)
}
}
clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{})
}()
@@ -223,21 +215,21 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients
EvictFailedBarePods: false,
}
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs))
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
deschedulerPolicyConfigMapObj.Name = fmt.Sprintf("%s-%s", deschedulerPolicyConfigMapObj.Name, testName)
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
createConfigMapWithCleanup(t, ctx, clientSet, deschedulerPolicyConfigMapObj)
deschedulerDeploymentObj := deschedulerDeployment(testName)
deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName)
args := deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect", "--leader-elect-retry-period", "1s")
deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = []string{
"--policy-config-file", "/policy-dir/policy.yaml",
"--descheduling-interval", "3s",
"--v", "4",
"--leader-elect",
"--leader-elect-retry-period", "1s",
}
deschedulerDeploymentObj.Spec.Template.Spec.Volumes = []v1.Volume{
{
Name: "policy-volume",
+1 -15
View File
@@ -289,21 +289,7 @@ func TestProtectPodsWithPVC(t *testing.T) {
t.Fatalf("Error creating %q CM: %v", policycm.Name, err)
}
t.Logf("creating %q policy CM with PodsWithPVC protection enabled...", policycm.Name)
if _, err = cli.CoreV1().ConfigMaps(policycm.Namespace).Create(
ctx, policycm, metav1.CreateOptions{},
); err != nil {
t.Fatalf("error creating %q CM: %v", policycm.Name, err)
}
defer func() {
t.Logf("deleting %q CM...", policycm.Name)
if err := cli.CoreV1().ConfigMaps(policycm.Namespace).Delete(
ctx, policycm.Name, metav1.DeleteOptions{},
); err != nil {
t.Fatalf("unable to delete %q CM: %v", policycm.Name, err)
}
}()
createConfigMapWithCleanup(t, ctx, cli, policycm)
desdep := deschedulerDeployment(namespace.Name)
t.Logf("creating descheduler deployment %v", desdep.Name)
+42 -1
View File
@@ -31,6 +31,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
schedulingv1 "k8s.io/api/scheduling/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
@@ -84,7 +85,12 @@ func TestMain(m *testing.M) {
}
func isClientRateLimiterError(err error) bool {
return strings.Contains(err.Error(), "client rate limiter")
if err == nil {
return false
}
return strings.Contains(err.Error(), "client rate limiter") &&
!strings.Contains(err.Error(), "context deadline exceeded") &&
!strings.Contains(err.Error(), "context canceled")
}
func initFeatureGates() featuregate.FeatureGate {
@@ -112,6 +118,41 @@ func deschedulerPolicyConfigMap(policy *deschedulerapiv1alpha2.DeschedulerPolicy
return cm, nil
}
// createPolicyConfigMap generates, creates (or recreates if already existing), and registers cleanup for a descheduler policy ConfigMap.
func createPolicyConfigMap(t *testing.T, ctx context.Context, kubeClient clientset.Interface, policy *deschedulerapiv1alpha2.DeschedulerPolicy) *v1.ConfigMap {
t.Helper()
cm, err := deschedulerPolicyConfigMap(policy)
if err != nil {
t.Fatalf("Error creating policy CM object: %v", err)
}
return createConfigMapWithCleanup(t, ctx, kubeClient, cm)
}
// createConfigMapWithCleanup creates (or recreates if already existing) a ConfigMap and registers a t.Cleanup callback to delete it.
func createConfigMapWithCleanup(t *testing.T, ctx context.Context, kubeClient clientset.Interface, cm *v1.ConfigMap) *v1.ConfigMap {
t.Helper()
t.Logf("Creating %q policy CM...", cm.Name)
_, err := kubeClient.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
_ = kubeClient.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
_, err = kubeClient.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{})
}
if err != nil {
t.Fatalf("Error creating %q CM: %v", cm.Name, err)
}
}
t.Cleanup(func() {
t.Logf("Deleting %q CM...", cm.Name)
if err := kubeClient.CoreV1().ConfigMaps(cm.Namespace).Delete(context.Background(), cm.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
t.Logf("Unable to delete %q CM: %v", cm.Name, err)
}
})
return cm
}
func deschedulerDeployment(testName string) *appsv1.Deployment {
deploymentObject := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
+1 -18
View File
@@ -153,24 +153,7 @@ func TestTooManyRestarts(t *testing.T) {
rs.DefaultFeatureGates = initFeatureGates()
preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
// Deploy the descheduler with the configured policy
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tc.policy)
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
defer func() {
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
}()
createPolicyConfigMap(t, ctx, clientSet, tc.policy)
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+1 -18
View File
@@ -219,24 +219,7 @@ func TestTopologySpreadConstraint(t *testing.T) {
Include: []string{testNamespace.Name},
},
}
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs))
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
defer func() {
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
}
}()
createPolicyConfigMap(t, ctx, clientSet, topologySpreadConstraintPolicy(constraintArgs, evictorArgs))
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})