Files
open-cluster-management/test/integration/placement/assertion_test.go
Jian Qiu bede3edd92 Switch to patcher in sdk-go (#349)
Signed-off-by: Jian Qiu <jqiu@redhat.com>
2024-01-22 02:04:49 +00:00

488 lines
18 KiB
Go

package placement
import (
"context"
"fmt"
"reflect"
"sort"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
clusterapiv1 "open-cluster-management.io/api/cluster/v1"
clusterapiv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1"
clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
clusterapiv1beta2 "open-cluster-management.io/api/cluster/v1beta2"
"open-cluster-management.io/sdk-go/pkg/patcher"
"open-cluster-management.io/ocm/test/integration/util"
)
// assert placement
func assertCreatingPlacement(placement *clusterapiv1beta1.Placement) *clusterapiv1beta1.Placement {
ginkgo.By("Create placement")
newplacement, err := clusterClient.ClusterV1beta1().Placements(placement.Namespace).Create(context.Background(), placement, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
return newplacement
}
func assertCreatingPlacementWithDecision(placement *clusterapiv1beta1.Placement, numberOfDecisionClusters, numberOfPlacementDecisions int) {
newplacement := assertCreatingPlacement(placement)
assertPlacementDecisionCreated(newplacement)
assertPlacementDecisionNumbers(newplacement.Name, newplacement.Namespace, numberOfDecisionClusters, numberOfPlacementDecisions)
if placement.Spec.NumberOfClusters != nil {
assertPlacementConditionSatisfied(
newplacement.Name, newplacement.Namespace, numberOfDecisionClusters,
numberOfDecisionClusters == int(*placement.Spec.NumberOfClusters))
}
}
func assertPatchingPlacementSpec(newPlacement *clusterapiv1beta1.Placement) {
ginkgo.By("Patching placement spec")
placementPatcher := patcher.NewPatcher[
*clusterapiv1beta1.Placement, clusterapiv1beta1.PlacementSpec, clusterapiv1beta1.PlacementStatus](
clusterClient.ClusterV1beta1().Placements(newPlacement.Namespace))
gomega.Eventually(func() error {
oldPlacement, err := clusterClient.ClusterV1beta1().Placements(newPlacement.Namespace).Get(
context.Background(), newPlacement.Name, metav1.GetOptions{})
if err != nil {
return err
}
_, err = placementPatcher.PatchSpec(context.Background(), newPlacement, newPlacement.Spec, oldPlacement.Spec)
return err
}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}
func assertPlacementDeleted(placementName, namespace string) {
ginkgo.By("Check if placement is gone")
gomega.Eventually(func() bool {
_, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
if err == nil {
return false
}
return errors.IsNotFound(err)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
func assertPlacementStatusDecisionGroups(placementName, namespace string, decisionGroupStatus []clusterapiv1beta1.DecisionGroupStatus) {
ginkgo.By("Check the group status of placement")
gomega.Eventually(func() bool {
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
if err != nil {
return false
}
ginkgo.By(fmt.Sprintf("actual decision groups %v", placement.Status.DecisionGroups))
return reflect.DeepEqual(placement.Status.DecisionGroups, decisionGroupStatus)
}, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeTrue())
}
func assertPlacementConditionSatisfied(placementName, namespace string, numOfSelectedClusters int, satisfied bool) {
ginkgo.By("Check the condition PlacementSatisfied of placement")
gomega.Eventually(func() bool {
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
if err != nil {
return false
}
if satisfied && !util.HasCondition(
placement.Status.Conditions,
clusterapiv1beta1.PlacementConditionSatisfied,
"AllDecisionsScheduled",
metav1.ConditionTrue,
) {
return false
}
if !satisfied && !util.HasCondition(
placement.Status.Conditions,
clusterapiv1beta1.PlacementConditionSatisfied,
"NotAllDecisionsScheduled",
metav1.ConditionFalse,
) {
return false
}
return placement.Status.NumberOfSelectedClusters == int32(numOfSelectedClusters)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
func assertPlacementConditionMisconfigured(placementName, namespace string, misConfigured bool) {
ginkgo.By("Check the condition PlacementMisconfigured of placement")
gomega.Eventually(func() bool {
placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
if err != nil {
return false
}
if !misConfigured && !util.HasCondition(
placement.Status.Conditions,
clusterapiv1beta1.PlacementConditionMisconfigured,
"Succeedconfigured",
metav1.ConditionFalse,
) {
return false
}
if misConfigured && !util.HasCondition(
placement.Status.Conditions,
clusterapiv1beta1.PlacementConditionMisconfigured,
"Misconfigured",
metav1.ConditionTrue,
) {
return false
}
return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
// assert placement decision
func assertCreatingPlacementDecision(name, namespace string, clusterNames []string) {
ginkgo.By(fmt.Sprintf("Create placementdecision %s", name))
placementDecision := &clusterapiv1beta1.PlacementDecision{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
Labels: map[string]string{
placementLabel: name,
},
},
}
placementDecision, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).Create(
context.Background(), placementDecision, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
var clusterDecisions []clusterapiv1beta1.ClusterDecision
for _, clusterName := range clusterNames {
clusterDecisions = append(clusterDecisions, clusterapiv1beta1.ClusterDecision{
ClusterName: clusterName,
})
}
placementDecision.Status.Decisions = clusterDecisions
placementDecision, err = clusterClient.ClusterV1beta1().PlacementDecisions(namespace).UpdateStatus(
context.Background(), placementDecision, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
func assertPlacementDecisionCreated(placement *clusterapiv1beta1.Placement) {
ginkgo.By("Check if placementdecision is created")
gomega.Eventually(func() bool {
pdl, err := clusterClient.ClusterV1beta1().PlacementDecisions(placement.Namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: placementLabel + "=" + placement.Name,
})
if err != nil {
return false
}
if len(pdl.Items) == 0 {
return false
}
for _, pd := range pdl.Items {
objectMeta := pd.ObjectMeta
if controlled := metav1.IsControlledBy(&objectMeta, placement); !controlled {
return false
}
}
return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
func assertPlacementDecisionNumbers(placementName, namespace string, desiredNumOfDecisionClusters, desiredNumOfDecisions int) {
ginkgo.By("Check the number of decisions in placementdecisions")
gomega.Eventually(func() bool {
pdl, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: placementLabel + "=" + placementName,
})
if err != nil {
return false
}
if len(pdl.Items) != desiredNumOfDecisions {
return false
}
actualNumOfDecisionClusters := 0
for _, pd := range pdl.Items {
actualNumOfDecisionClusters += len(pd.Status.Decisions)
}
return actualNumOfDecisionClusters == desiredNumOfDecisionClusters
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
func assertPlacementDecisionClusterNames(placementName, namespace string, desiredClusters []string) {
ginkgo.By(fmt.Sprintf("Check the cluster names of placementdecisions %s", placementName))
gomega.Eventually(func() bool {
pdl, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: placementLabel + "=" + placementName,
})
if err != nil {
return false
}
actualClusters := sets.NewString()
desiredClusters := sets.NewString(desiredClusters...)
for _, pd := range pdl.Items {
for _, d := range pd.Status.Decisions {
actualClusters.Insert(d.ClusterName)
}
}
if actualClusters.Equal(desiredClusters) {
return true
}
ginkgo.By(fmt.Sprintf("Expect %v, but got %v", desiredClusters.List(), actualClusters.List()))
return false
}, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeTrue())
}
// assert clusterset
func assertCreatingClusterSet(clusterSetName string, labels ...string) {
ginkgo.By(fmt.Sprintf("Create clusterset %s", clusterSetName))
clusterset := &clusterapiv1beta2.ManagedClusterSet{
ObjectMeta: metav1.ObjectMeta{
Name: clusterSetName,
Labels: map[string]string{},
},
Spec: clusterapiv1beta2.ManagedClusterSetSpec{
ClusterSelector: clusterapiv1beta2.ManagedClusterSelector{
SelectorType: clusterapiv1beta2.LabelSelector,
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{},
},
},
},
}
if len(labels) > 0 {
for i := 1; i < len(labels); i += 2 {
clusterset.Spec.ClusterSelector.LabelSelector.MatchLabels[labels[i-1]] = labels[i]
}
}
_, err := clusterClient.ClusterV1beta2().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
func assertBindingClusterSet(clusterSetName, namespace string) {
ginkgo.By("Create clusterset/clustersetbinding")
clusterset := &clusterapiv1beta2.ManagedClusterSet{
ObjectMeta: metav1.ObjectMeta{
Name: clusterSetName,
},
}
_, err := clusterClient.ClusterV1beta2().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
csb := &clusterapiv1beta2.ManagedClusterSetBinding{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: clusterSetName,
},
Spec: clusterapiv1beta2.ManagedClusterSetBindingSpec{
ClusterSet: clusterSetName,
},
}
_, err = clusterClient.ClusterV1beta2().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
func assertDeletingClusterSet(clusterSetName string) {
ginkgo.By(fmt.Sprintf("Delete clusterset %s", clusterSetName))
err := clusterClient.ClusterV1beta2().ManagedClusterSets().Delete(context.Background(), clusterSetName, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
ginkgo.By("Check if clusterset is gone")
gomega.Eventually(func() bool {
_, err := clusterClient.ClusterV1beta2().ManagedClusterSets().Get(context.Background(), clusterSetName, metav1.GetOptions{})
if err == nil {
return false
}
return errors.IsNotFound(err)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
func assertCreatingClusterSetBinding(clusterSetName, namespace string) {
ginkgo.By(fmt.Sprintf("Create clustersetbinding %s", clusterSetName))
csb := &clusterapiv1beta2.ManagedClusterSetBinding{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: clusterSetName,
},
Spec: clusterapiv1beta2.ManagedClusterSetBindingSpec{
ClusterSet: clusterSetName,
},
}
_, err := clusterClient.ClusterV1beta2().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
func assertDeletingClusterSetBinding(clusterSetName, namespace string) {
ginkgo.By(fmt.Sprintf("Delete clustersetbinding %s", clusterSetName))
err := clusterClient.ClusterV1beta2().ManagedClusterSetBindings(namespace).Delete(context.Background(), clusterSetName, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
ginkgo.By("Check if clustersetbinding is gone")
gomega.Eventually(func() bool {
_, err := clusterClient.ClusterV1beta2().ManagedClusterSetBindings(namespace).Get(context.Background(), clusterSetName, metav1.GetOptions{})
if err == nil {
return false
}
return errors.IsNotFound(err)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
}
// assert clusters
func assertCreatingClusters(clusterSetName string, num int, labels ...string) []string {
ginkgo.By(fmt.Sprintf("Create %d clusters", num))
var names []string
for i := 0; i < num; i++ {
cluster := &clusterapiv1.ManagedCluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Labels: map[string]string{
clusterSetLabel: clusterSetName,
},
},
}
for i := 1; i < len(labels); i += 2 {
cluster.Labels[labels[i-1]] = labels[i]
}
cluster, err := clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
cluster.Status.Conditions = []metav1.Condition{}
for i := 1; i < len(labels); i += 2 {
cluster.Status.ClusterClaims = append(cluster.Status.ClusterClaims, clusterapiv1.ManagedClusterClaim{Name: labels[i-1], Value: labels[i]})
}
_, err = clusterClient.ClusterV1().ManagedClusters().UpdateStatus(context.Background(), cluster, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
names = append(names, cluster.Name)
}
sort.SliceStable(names, func(i, j int) bool {
return names[i] < names[j]
})
return names
}
func assertCleanupClusters() []string {
ginkgo.By("Cleanup all managed clusters")
var clusterNames []string
clusters, err := clusterClient.ClusterV1().ManagedClusters().List(context.Background(), metav1.ListOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
for _, cluster := range clusters.Items {
clusterNames = append(clusterNames, cluster.Name)
}
assertDeletingClusters(clusterNames...)
return clusterNames
}
func assertPatchingClusterStatusWithResources(managedClusterName string, res []string) {
ginkgo.By(fmt.Sprintf("Updating ManagedClusters %s cluster resources", managedClusterName))
oldmc, err := clusterClient.ClusterV1().ManagedClusters().Get(context.Background(), managedClusterName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
allocatable := map[clusterapiv1.ResourceName]resource.Quantity{}
capacity := map[clusterapiv1.ResourceName]resource.Quantity{}
allocatable[clusterapiv1.ResourceCPU], err = resource.ParseQuantity(res[0])
gomega.Expect(err).ToNot(gomega.HaveOccurred())
allocatable[clusterapiv1.ResourceMemory], err = resource.ParseQuantity(res[2])
gomega.Expect(err).ToNot(gomega.HaveOccurred())
capacity[clusterapiv1.ResourceCPU], err = resource.ParseQuantity(res[1])
gomega.Expect(err).ToNot(gomega.HaveOccurred())
capacity[clusterapiv1.ResourceMemory], err = resource.ParseQuantity(res[3])
gomega.Expect(err).ToNot(gomega.HaveOccurred())
newmc := oldmc.DeepCopy()
newmc.Status.Allocatable = allocatable
newmc.Status.Capacity = capacity
managedClusterPatcher := patcher.NewPatcher[
*clusterapiv1.ManagedCluster, clusterapiv1.ManagedClusterSpec, clusterapiv1.ManagedClusterStatus](
clusterClient.ClusterV1().ManagedClusters())
_, err = managedClusterPatcher.PatchStatus(context.Background(), newmc, newmc.Status, oldmc.Status)
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
func assertPatchingClusterSpecWithTaint(managedClusterName string, taint *clusterapiv1.Taint) {
ginkgo.By(fmt.Sprintf("Updating ManagedClusters %s taint", managedClusterName))
if taint == nil {
return
}
oldmc, err := clusterClient.ClusterV1().ManagedClusters().Get(context.Background(), managedClusterName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
newmc := oldmc.DeepCopy()
newmc.Spec.Taints = append(newmc.Spec.Taints, *taint)
managedClusterPatcher := patcher.NewPatcher[
*clusterapiv1.ManagedCluster, clusterapiv1.ManagedClusterSpec, clusterapiv1.ManagedClusterStatus](
clusterClient.ClusterV1().ManagedClusters())
_, err = managedClusterPatcher.PatchSpec(context.Background(), newmc, newmc.Spec, oldmc.Spec)
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
func assertDeletingClusters(clusterNames ...string) {
for _, clusterName := range clusterNames {
ginkgo.By(fmt.Sprintf("Delete cluster %s", clusterName))
err := clusterClient.ClusterV1().ManagedClusters().Delete(context.Background(), clusterName, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
ginkgo.By("Check if cluster is gone")
gomega.Eventually(func() bool {
_, err := clusterClient.ClusterV1().ManagedClusters().Get(context.Background(), clusterName, metav1.GetOptions{})
if err == nil {
return false
}
return errors.IsNotFound(err)
}, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeTrue())
}
}
func assertCreatingAddOnPlacementScores(clusternamespace, crname, scorename string, score int32) {
ginkgo.By(fmt.Sprintf("Create namespace %s for addonplacementscores %s", clusternamespace, crname))
if _, err := kubeClient.CoreV1().Namespaces().Get(context.Background(), clusternamespace, metav1.GetOptions{}); err != nil {
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: clusternamespace,
},
}
_, err := kubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
ginkgo.By(fmt.Sprintf("Create addonplacementscores %s in %s", crname, clusternamespace))
addOn, err := clusterClient.ClusterV1alpha1().AddOnPlacementScores(clusternamespace).Get(context.Background(), crname, metav1.GetOptions{})
if err != nil {
newAddOn := &clusterapiv1alpha1.AddOnPlacementScore{
ObjectMeta: metav1.ObjectMeta{
Namespace: clusternamespace,
Name: crname,
},
}
addOn, err = clusterClient.ClusterV1alpha1().AddOnPlacementScores(clusternamespace).Create(context.Background(), newAddOn, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
vu := metav1.NewTime(time.Now().Add(10 * time.Second))
addOn.Status = clusterapiv1alpha1.AddOnPlacementScoreStatus{
Scores: []clusterapiv1alpha1.AddOnPlacementScoreItem{
{
Name: scorename,
Value: score,
},
},
ValidUntil: &vu,
}
_, err = clusterClient.ClusterV1alpha1().AddOnPlacementScores(clusternamespace).UpdateStatus(context.Background(), addOn, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}