open-cluster-management/pkg/placement/controllers/scheduling/enqueue_test.go

package scheduling

import (
	"strings"
	"testing"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog/v2/ktesting"

	clusterclient "open-cluster-management.io/api/client/cluster/clientset/versioned"
	clusterfake "open-cluster-management.io/api/client/cluster/clientset/versioned/fake"
	clusterinformers "open-cluster-management.io/api/client/cluster/informers/externalversions"
	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
	clusterapiv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1"
	clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
	clusterapiv1beta2 "open-cluster-management.io/api/cluster/v1beta2"

	testingcommon "open-cluster-management.io/ocm/pkg/common/testing"
	testinghelpers "open-cluster-management.io/ocm/pkg/placement/helpers/testing"
)

func newClusterInformerFactory(t *testing.T, clusterClient clusterclient.Interface, objects ...runtime.Object) clusterinformers.SharedInformerFactory {
	clusterInformerFactory := clusterinformers.NewSharedInformerFactory(clusterClient, time.Minute*10)

	err := clusterInformerFactory.Cluster().V1beta1().Placements().Informer().AddIndexers(cache.Indexers{
		placementsByScore:             indexPlacementsByScore,
		placementsByClusterSetBinding: indexPlacementByClusterSetBinding,
	})
	if err != nil {
		t.Fatal(err)
	}

	err = clusterInformerFactory.Cluster().V1beta2().ManagedClusterSetBindings().Informer().AddIndexers(cache.Indexers{
		clustersetBindingsByClusterSet: indexClusterSetBindingByClusterSet,
	})
	if err != nil {
		t.Fatal(err)
	}

	clusterStore := clusterInformerFactory.Cluster().V1().ManagedClusters().Informer().GetStore()
	clusterSetStore := clusterInformerFactory.Cluster().V1beta2().ManagedClusterSets().Informer().GetStore()
	clusterSetBindingStore := clusterInformerFactory.Cluster().V1beta2().ManagedClusterSetBindings().Informer().GetStore()
	placementStore := clusterInformerFactory.Cluster().V1beta1().Placements().Informer().GetStore()
	placementDecisionStore := clusterInformerFactory.Cluster().V1beta1().PlacementDecisions().Informer().GetStore()
	addOnPlacementStore := clusterInformerFactory.Cluster().V1alpha1().AddOnPlacementScores().Informer().GetStore()

	for _, obj := range objects {
		var err error
		switch obj.(type) {
		case *clusterapiv1.ManagedCluster:
			err = clusterStore.Add(obj)
		case *clusterapiv1beta2.ManagedClusterSet:
			err = clusterSetStore.Add(obj)
		case *clusterapiv1beta2.ManagedClusterSetBinding:
			err = clusterSetBindingStore.Add(obj)
		case *clusterapiv1beta1.Placement:
			err = placementStore.Add(obj)
		case *clusterapiv1beta1.PlacementDecision:
			err = placementDecisionStore.Add(obj)
		case *clusterapiv1alpha1.AddOnPlacementScore:
			err = addOnPlacementStore.Add(obj)
		}
		if err != nil {
			t.Fatal(err)
		}
	}

	return clusterInformerFactory
}

func TestEnqueuePlacementsByClusterSet(t *testing.T) {
	cases := []struct {
		name       string
		clusterSet interface{}
		initObjs   []runtime.Object
		queuedKeys []string
	}{
		{
			name:       "enqueue placements in a namespace",
			clusterSet: testinghelpers.NewClusterSet("clusterset1").Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewClusterSet("clusterset2").Build(),
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewClusterSetBinding("ns1", "clusterset2"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
				testinghelpers.NewPlacement("ns1", "placement2").WithClusterSets("clusterset2").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name:       "invalid object type",
			clusterSet: "invalid object type",
		},
		{
			name:       "clusterset selector type is LegacyClusterSetLabel",
			clusterSet: testinghelpers.NewClusterSet("clusterset1").Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name: "clusterset selector type is LabelSelector",
			clusterSet: testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta2.ManagedClusterSelector{
				SelectorType: clusterapiv1beta2.LabelSelector,
				LabelSelector: &metav1.LabelSelector{
					MatchLabels: map[string]string{
						"cloud": "Amazon",
					},
				},
			}).Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name:       "clusterset selector type is LegacyClusterSetLabel",
			clusterSet: testinghelpers.NewClusterSet("clusterset1").Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name: "clusterset selector type is LabelSelector",
			clusterSet: testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta2.ManagedClusterSelector{
				SelectorType: clusterapiv1beta2.LabelSelector,
				LabelSelector: &metav1.LabelSelector{
					MatchLabels: map[string]string{
						"cloud": "Amazon",
					},
				},
			}).Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name: "tombstone",
			clusterSet: cache.DeletedFinalStateUnknown{
				Key: "clusterset1",
				Obj: testinghelpers.NewClusterSet("clusterset1").Build(),
			},
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name: "tombstone with invalid object type",
			clusterSet: cache.DeletedFinalStateUnknown{
				Obj: "invalid object type",
			},
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			_, ctx := ktesting.NewTestContext(t)
			clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
			clusterInformerFactory := newClusterInformerFactory(t, clusterClient, c.initObjs...)

			syncCtx := testingcommon.NewFakeSyncContext(t, "fake")
			q := newEnqueuer(
				ctx,
				syncCtx.Queue(),
				clusterInformerFactory.Cluster().V1().ManagedClusters(),
				clusterInformerFactory.Cluster().V1beta2().ManagedClusterSets(),
				clusterInformerFactory.Cluster().V1beta1().Placements(),
				clusterInformerFactory.Cluster().V1beta2().ManagedClusterSetBindings(),
			)
			queuedKeys := sets.NewString()
			fakeEnqueuePlacement := func(obj interface{}, queue workqueue.RateLimitingInterface) {
				key, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
				queuedKeys.Insert(key)
			}
			q.enqueuePlacementFunc = fakeEnqueuePlacement
			q.enqueueClusterSet(c.clusterSet)

			expectedQueuedKeys := sets.NewString(c.queuedKeys...)
			if !queuedKeys.Equal(expectedQueuedKeys) {
				t.Errorf("expected queued placements %q, but got %s", strings.Join(expectedQueuedKeys.List(), ","), strings.Join(queuedKeys.List(), ","))
			}
		})
	}
}

func TestEnqueuePlacementsByClusterSetBinding(t *testing.T) {
	cases := []struct {
		name              string
		namespace         string
		clusterSetBinding interface{}
		initObjs          []runtime.Object
		queuedKeys        []string
	}{
		{
			name:              "enqueue placements by clusterSetBinding",
			namespace:         "ns1",
			clusterSetBinding: testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
			initObjs: []runtime.Object{
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
				testinghelpers.NewPlacement("ns1", "placement2").WithClusterSets("clusterset2").Build(),
				testinghelpers.NewPlacement("ns2", "placement3").WithClusterSets("clusterset1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name:              "invalid resource type",
			clusterSetBinding: "invalid resource type",
			initObjs: []runtime.Object{
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
				testinghelpers.NewPlacement("ns2", "placement2").Build(),
			},
		},
		{
			name:              "on clustersetbinding change",
			clusterSetBinding: testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
				testinghelpers.NewPlacement("ns2", "placement2").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name:              "clusterset",
			clusterSetBinding: testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name: "tombstone",
			clusterSetBinding: cache.DeletedFinalStateUnknown{
				Key: "ns1/clusterset1",
				Obj: testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
			},
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
		{
			name: "tombstone with invalid object type",
			clusterSetBinding: cache.DeletedFinalStateUnknown{
				Obj: "invalid object type",
			},
			initObjs: []runtime.Object{
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewPlacement("ns1", "placement1").Build(),
			},
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			_, ctx := ktesting.NewTestContext(t)
			clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
			clusterInformerFactory := newClusterInformerFactory(t, clusterClient, c.initObjs...)

			syncCtx := testingcommon.NewFakeSyncContext(t, "fake")
			q := newEnqueuer(
				ctx,
				syncCtx.Queue(),
				clusterInformerFactory.Cluster().V1().ManagedClusters(),
				clusterInformerFactory.Cluster().V1beta2().ManagedClusterSets(),
				clusterInformerFactory.Cluster().V1beta1().Placements(),
				clusterInformerFactory.Cluster().V1beta2().ManagedClusterSetBindings(),
			)
			queuedKeys := sets.NewString()
			fakeEnqueuePlacement := func(obj interface{}, queue workqueue.RateLimitingInterface) {
				key, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
				queuedKeys.Insert(key)
			}
			q.enqueuePlacementFunc = fakeEnqueuePlacement
			q.enqueueClusterSetBinding(c.clusterSetBinding)

			expectedQueuedKeys := sets.NewString(c.queuedKeys...)
			if !queuedKeys.Equal(expectedQueuedKeys) {
				t.Errorf("expected queued placements %q, but got %s", strings.Join(expectedQueuedKeys.List(), ","), strings.Join(queuedKeys.List(), ","))
			}
		})
	}
}

func TestEnqueuePlacementsByScore(t *testing.T) {
	cases := []struct {
		name       string
		namespace  string
		score      interface{}
		initObjs   []runtime.Object
		queuedKeys []string
	}{
		{
			name:  "ensueue score",
			score: testinghelpers.NewAddOnPlacementScore("cluster1", "score1").Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewPlacement("ns1", "placement1").WithScoreCoordinateAddOn("score1", "cpu", 1).Build(),
				testinghelpers.NewPlacement("ns2", "placement2").WithScoreCoordinateAddOn("score2", "cpu", 1).Build(),
				testinghelpers.NewPlacement("ns3", "placement3").WithScoreCoordinateAddOn("score1", "cpu", 1).Build(),
				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, "clusterset1").Build(),
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
				testinghelpers.NewClusterSetBinding("ns3", "clusterset1"),
			},
			queuedKeys: []string{
				"ns1/placement1",
				"ns3/placement3",
			},
		},
		{
			name:  "only enqueue score with filtered placement",
			score: testinghelpers.NewAddOnPlacementScore("cluster1", "score1").Build(),
			initObjs: []runtime.Object{
				testinghelpers.NewPlacement("ns1", "placement1").WithScoreCoordinateAddOn("score1", "cpu", 1).Build(),
				testinghelpers.NewPlacement("ns2", "placement2").WithScoreCoordinateAddOn("score2", "cpu", 1).Build(),
				testinghelpers.NewPlacement("ns3", "placement3").WithScoreCoordinateAddOn("score1", "cpu", 1).Build(),
				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, "clusterset1").Build(),
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewClusterSetBinding("ns1", "clusterset2"),
				testinghelpers.NewClusterSetBinding("ns3", "clusterset1"),
			},
			queuedKeys: []string{
				"ns3/placement3",
			},
		},
		{
			name: "tombstone",
			score: cache.DeletedFinalStateUnknown{
				Key: "cluster1/score1",
				Obj: testinghelpers.NewAddOnPlacementScore("cluster1", "score1").Build(),
			},
			initObjs: []runtime.Object{
				testinghelpers.NewPlacement("ns1", "placement1").WithScoreCoordinateAddOn("score1", "cpu", 1).Build(),
				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, "clusterset1").Build(),
				testinghelpers.NewClusterSet("clusterset1").Build(),
				testinghelpers.NewClusterSetBinding("ns1", "clusterset1"),
			},
			queuedKeys: []string{
				"ns1/placement1",
			},
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			_, ctx := ktesting.NewTestContext(t)
			clusterClient := clusterfake.NewSimpleClientset(c.initObjs...)
			clusterInformerFactory := newClusterInformerFactory(t, clusterClient, c.initObjs...)

			syncCtx := testingcommon.NewFakeSyncContext(t, "fake")
			q := newEnqueuer(
				ctx,
				syncCtx.Queue(),
				clusterInformerFactory.Cluster().V1().ManagedClusters(),
				clusterInformerFactory.Cluster().V1beta2().ManagedClusterSets(),
				clusterInformerFactory.Cluster().V1beta1().Placements(),
				clusterInformerFactory.Cluster().V1beta2().ManagedClusterSetBindings(),
			)
			queuedKeys := sets.NewString()
			fakeEnqueuePlacement := func(obj interface{}, queue workqueue.RateLimitingInterface) {
				key, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
				queuedKeys.Insert(key)
			}
			q.enqueuePlacementFunc = fakeEnqueuePlacement
			q.enqueuePlacementScore(c.score)

			expectedQueuedKeys := sets.NewString(c.queuedKeys...)
			if !queuedKeys.Equal(expectedQueuedKeys) {
				t.Errorf("expected queued placements %q, but got %s", strings.Join(expectedQueuedKeys.List(), ","), strings.Join(queuedKeys.List(), ","))
			}
		})
	}
}