open-cluster-management/test/integration/addon/assertion_test.go

package integration

import (
	"context"
	"fmt"
	"time"

	"github.com/onsi/ginkgo/v2"
	"github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"
	apiequality "k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
	addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1"
	addonclient "open-cluster-management.io/api/client/addon/clientset/versioned"
	clusterv1client "open-cluster-management.io/api/client/cluster/clientset/versioned"
	workclientset "open-cluster-management.io/api/client/work/clientset/versioned"
	clusterv1 "open-cluster-management.io/api/cluster/v1"
	clusterv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
	workapiv1 "open-cluster-management.io/api/work/v1"
	"open-cluster-management.io/sdk-go/pkg/patcher"
)

// TODO: The spec hash is hardcoded here and will break the test once the API changes.
// We need a better way to handle the spec hash.
// Config spec hashes (same for both v1alpha1 and v1beta1 since structure is identical)
const addOnDefaultConfigSpecHash = "ea4e6089e8b2f584c1cd8b86e16a15df3186b30744f433ede7436091c93a4cc7" //nolint:gosec
const addOnTest1ConfigSpecHash = "b746cc98936955dd2168640f0011a14f2a74be306c747bc8ba829328cc772690"   //nolint:gosec
const addOnTest2ConfigSpecHash = "3e6348b0c0f0a341ce28d0a2f99bc41fc63fb17d947eb5a45284729fbeb63b32"   //nolint:gosec
const addOnTest3ConfigSpecHash = "9b3c6d09731ed254589fa4fc2936e6fb00537de92da5e035183da313eb98d27e"   //nolint:gosec

// v1alpha1 config specs
var addOnDefaultConfigSpecAlpha = addonapiv1alpha1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
		{
			Name:  "test",
			Value: "test",
		},
	},
}

var addOnTest1ConfigSpecAlpha = addonapiv1alpha1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
		{
			Name:  "test1",
			Value: "test1",
		},
	},
}

var addOnTest2ConfigSpecAlpha = addonapiv1alpha1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
		{
			Name:  "test2",
			Value: "test2",
		},
	},
}

var addOnTest3ConfigSpecAlpha = addonapiv1alpha1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
		{
			Name:  "test3",
			Value: "test3",
		},
	},
}

// v1beta1 config specs (structure is identical to v1alpha1)
var addOnDefaultConfigSpecBeta = addonapiv1beta1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1beta1.CustomizedVariable{
		{
			Name:  "test",
			Value: "test",
		},
	},
}

var addOnTest1ConfigSpecBeta = addonapiv1beta1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1beta1.CustomizedVariable{
		{
			Name:  "test1",
			Value: "test1",
		},
	},
}

var addOnTest2ConfigSpecBeta = addonapiv1beta1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1beta1.CustomizedVariable{
		{
			Name:  "test2",
			Value: "test2",
		},
	},
}

var addOnTest3ConfigSpecBeta = addonapiv1beta1.AddOnDeploymentConfigSpec{
	CustomizedVariables: []addonapiv1beta1.CustomizedVariable{
		{
			Name:  "test3",
			Value: "test3",
		},
	},
}

func createClusterManagementAddOnAlpha(name, defaultConfigNamespace, defaultConfigName string) (*addonapiv1alpha1.ClusterManagementAddOn, error) {
	clusterManagementAddon, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
	if errors.IsNotFound(err) {
		clusterManagementAddon, err = hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Create(
			context.Background(),
			&addonapiv1alpha1.ClusterManagementAddOn{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
				},
				Spec: addonapiv1alpha1.ClusterManagementAddOnSpec{
					SupportedConfigs: []addonapiv1alpha1.ConfigMeta{
						{
							ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
								Group:    addOnDeploymentConfigGVR.Group,
								Resource: addOnDeploymentConfigGVR.Resource,
							},
							DefaultConfig: &addonapiv1alpha1.ConfigReferent{
								Name:      defaultConfigName,
								Namespace: defaultConfigNamespace,
							},
						},
					},
					InstallStrategy: addonapiv1alpha1.InstallStrategy{
						Type: addonapiv1alpha1.AddonInstallStrategyManual,
					},
				},
			},
			metav1.CreateOptions{},
		)
		if err != nil {
			return nil, err
		}
		return clusterManagementAddon, nil
	}

	if err != nil {
		return nil, err
	}

	return clusterManagementAddon, nil
}

func createClusterManagementAddOnBeta(name, defaultConfigNamespace, defaultConfigName string) (*addonapiv1beta1.ClusterManagementAddOn, error) {
	clusterManagementAddon, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
	if errors.IsNotFound(err) {
		clusterManagementAddon, err = hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Create(
			context.Background(),
			&addonapiv1beta1.ClusterManagementAddOn{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
				},
				Spec: addonapiv1beta1.ClusterManagementAddOnSpec{
					DefaultConfigs: []addonapiv1beta1.AddOnConfig{
						{
							ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{
								Group:    addOnDeploymentConfigGVR.Group,
								Resource: addOnDeploymentConfigGVR.Resource,
							},
							ConfigReferent: addonapiv1beta1.ConfigReferent{
								Name:      defaultConfigName,
								Namespace: defaultConfigNamespace,
							},
						},
					},
					InstallStrategy: addonapiv1beta1.InstallStrategy{
						Type: addonapiv1beta1.AddonInstallStrategyManual,
					},
				},
			},
			metav1.CreateOptions{},
		)
		if err != nil {
			return nil, err
		}
		return clusterManagementAddon, nil
	}

	if err != nil {
		return nil, err
	}

	return clusterManagementAddon, nil
}

func patchClusterManagementAddOnAlpha(_ context.Context, newaddon *addonapiv1alpha1.ClusterManagementAddOn) {
	gomega.Eventually(func() error {
		old, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), newaddon.Name, metav1.GetOptions{})
		if err != nil {
			return err
		}
		new := old.DeepCopy()
		new.Spec = *newaddon.Spec.DeepCopy()
		new.Annotations = newaddon.Annotations

		cmaPatcher := patcher.NewPatcher[
			*addonapiv1alpha1.ClusterManagementAddOn, addonapiv1alpha1.ClusterManagementAddOnSpec, addonapiv1alpha1.ClusterManagementAddOnStatus](
			hubAddonClient.AddonV1alpha1().ClusterManagementAddOns())
		_, err = cmaPatcher.PatchSpec(context.Background(), new, new.Spec, old.Spec)
		if err != nil {
			return err
		}

		_, err = cmaPatcher.PatchLabelAnnotations(context.Background(), new, new.ObjectMeta, old.ObjectMeta)
		return err
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func patchClusterManagementAddOnBeta(_ context.Context, newaddon *addonapiv1beta1.ClusterManagementAddOn) {
	gomega.Eventually(func() error {
		old, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), newaddon.Name, metav1.GetOptions{})
		if err != nil {
			return err
		}
		new := old.DeepCopy()
		new.Spec = *newaddon.Spec.DeepCopy()
		new.Annotations = newaddon.Annotations

		_, err = hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Update(context.Background(), new, metav1.UpdateOptions{})
		return err
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func updateManagedClusterAddOnStatusAlpha(_ context.Context, new *addonapiv1alpha1.ManagedClusterAddOn) {
	gomega.Eventually(func() error {
		old, err := hubAddonClient.AddonV1alpha1().ManagedClusterAddOns(new.Namespace).Get(context.Background(), new.Name, metav1.GetOptions{})
		if err != nil {
			return err
		}
		old.Status = new.Status
		_, err = hubAddonClient.AddonV1alpha1().ManagedClusterAddOns(old.Namespace).UpdateStatus(context.Background(), old, metav1.UpdateOptions{})
		return err
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func updateManagedClusterAddOnStatusBeta(_ context.Context, new *addonapiv1beta1.ManagedClusterAddOn) {
	gomega.Eventually(func() error {
		old, err := hubAddonClient.AddonV1beta1().ManagedClusterAddOns(new.Namespace).Get(context.Background(), new.Name, metav1.GetOptions{})
		if err != nil {
			return err
		}
		old.Status = new.Status
		_, err = hubAddonClient.AddonV1beta1().ManagedClusterAddOns(old.Namespace).UpdateStatus(context.Background(), old, metav1.UpdateOptions{})
		return err
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func assertClusterManagementAddOnDefaultConfigReferencesAlpha(name string, expect ...addonapiv1alpha1.DefaultConfigReference) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s DefaultConfigReferences (Alpha)", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if len(actual.Status.DefaultConfigReferences) != len(expect) {
			return fmt.Errorf("expected %v default config reference, actual: %v", len(expect), len(actual.Status.DefaultConfigReferences))
		}

		for i, e := range expect {
			actualConfigReference := actual.Status.DefaultConfigReferences[i]

			if !apiequality.Semantic.DeepEqual(actualConfigReference, e) {
				return fmt.Errorf("expected default config is %v, actual: %v", e, actualConfigReference)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertClusterManagementAddOnDefaultConfigReferencesBeta(name string, expect ...addonapiv1beta1.DefaultConfigReference) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s DefaultConfigReferences (Beta)", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if len(actual.Status.DefaultConfigReferences) != len(expect) {
			return fmt.Errorf("expected %v default config reference, actual: %v", len(expect), len(actual.Status.DefaultConfigReferences))
		}

		for i, e := range expect {
			actualConfigReference := actual.Status.DefaultConfigReferences[i]

			if !apiequality.Semantic.DeepEqual(actualConfigReference, e) {
				return fmt.Errorf("expected default config is %v, actual: %v", e, actualConfigReference)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func createAddOnDeploymentConfigAlpha(
	hubAddonClient addonclient.Interface,
	configNamespace string,
	configName string,
	configSpec addonapiv1alpha1.AddOnDeploymentConfigSpec,
) error {
	config := &addonapiv1alpha1.AddOnDeploymentConfig{
		ObjectMeta: metav1.ObjectMeta{
			Name:      configName,
			Namespace: configNamespace,
		},
		Spec: configSpec,
	}
	_, err := hubAddonClient.AddonV1alpha1().AddOnDeploymentConfigs(configNamespace).Create(
		context.Background(), config, metav1.CreateOptions{})
	return err
}

func createAddOnDeploymentConfigBeta(
	hubAddonClient addonclient.Interface,
	configNamespace string,
	configName string,
	configSpec addonapiv1beta1.AddOnDeploymentConfigSpec,
) error {
	config := &addonapiv1beta1.AddOnDeploymentConfig{
		ObjectMeta: metav1.ObjectMeta{
			Name:      configName,
			Namespace: configNamespace,
		},
		Spec: configSpec,
	}
	_, err := hubAddonClient.AddonV1beta1().AddOnDeploymentConfigs(configNamespace).Create(
		context.Background(), config, metav1.CreateOptions{})
	return err
}

func updateAddOnDeploymentConfigSpecAlpha(
	hubAddonClient addonclient.Interface,
	configNamespace string,
	configName string,
	newConfigSpec addonapiv1alpha1.AddOnDeploymentConfigSpec,
) {
	gomega.Eventually(func() error {
		addOnConfig, err := hubAddonClient.AddonV1alpha1().AddOnDeploymentConfigs(configNamespace).Get(
			context.Background(), configName, metav1.GetOptions{})
		if err != nil {
			return err
		}

		addOnConfig.Spec = newConfigSpec
		_, err = hubAddonClient.AddonV1alpha1().AddOnDeploymentConfigs(configNamespace).Update(
			context.Background(), addOnConfig, metav1.UpdateOptions{})
		if err != nil {
			return err
		}
		return nil
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func updateAddOnDeploymentConfigSpecBeta(
	hubAddonClient addonclient.Interface,
	configNamespace string,
	configName string,
	newConfigSpec addonapiv1beta1.AddOnDeploymentConfigSpec,
) {
	gomega.Eventually(func() error {
		addOnConfig, err := hubAddonClient.AddonV1beta1().AddOnDeploymentConfigs(configNamespace).Get(
			context.Background(), configName, metav1.GetOptions{})
		if err != nil {
			return err
		}

		addOnConfig.Spec = newConfigSpec
		_, err = hubAddonClient.AddonV1beta1().AddOnDeploymentConfigs(configNamespace).Update(
			context.Background(), addOnConfig, metav1.UpdateOptions{})
		if err != nil {
			return err
		}
		return nil
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func assertClusterManagementAddOnInstallProgressionAlpha(name string, expect ...addonapiv1alpha1.InstallProgression) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s InstallProgression", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if len(actual.Status.InstallProgressions) != len(expect) {
			return fmt.Errorf("expected %v install progression, actual: %v", len(expect), len(actual.Status.InstallProgressions))
		}

		for i, e := range expect {
			actualInstallProgression := actual.Status.InstallProgressions[i]

			if !apiequality.Semantic.DeepEqual(actualInstallProgression.ConfigReferences, e.ConfigReferences) {
				return fmt.Errorf("expected InstallProgression.ConfigReferences is %v, actual: %v", e.ConfigReferences, actualInstallProgression.ConfigReferences)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertClusterManagementAddOnInstallProgressionBeta(name string, expect ...addonapiv1beta1.InstallProgression) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s InstallProgression (v1beta1)", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if len(actual.Status.InstallProgressions) != len(expect) {
			return fmt.Errorf("expected %v install progression, actual: %v", len(expect), len(actual.Status.InstallProgressions))
		}

		for i, e := range expect {
			actualInstallProgression := actual.Status.InstallProgressions[i]

			if !apiequality.Semantic.DeepEqual(actualInstallProgression.ConfigReferences, e.ConfigReferences) {
				return fmt.Errorf("expected InstallProgression.ConfigReferences is %v, actual: %v", e.ConfigReferences, actualInstallProgression.ConfigReferences)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertManagedClusterAddOnConfigReferencesAlpha(name, namespace string, expect ...addonapiv1alpha1.ConfigReference) {
	ginkgo.By(fmt.Sprintf("Check ManagedClusterAddOn %s/%s ConfigReferences", namespace, name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ManagedClusterAddOns(namespace).Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if len(actual.Status.ConfigReferences) != len(expect) {
			return fmt.Errorf("expected %v config reference, actual: %v", len(expect), len(actual.Status.ConfigReferences))
		}

		for i, e := range expect {
			actualConfigReference := actual.Status.ConfigReferences[i]

			if !apiequality.Semantic.DeepEqual(actualConfigReference, e) {
				return fmt.Errorf("expected mca config reference is %v %v, actual: %v %v",
					e.DesiredConfig,
					e.LastAppliedConfig,
					actualConfigReference.DesiredConfig,
					actualConfigReference.LastAppliedConfig,
				)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertManagedClusterAddOnConfigReferencesBeta(name, namespace string, expect ...addonapiv1beta1.ConfigReference) {
	ginkgo.By(fmt.Sprintf("Check ManagedClusterAddOn %s/%s ConfigReferences (v1beta1)", namespace, name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1beta1().ManagedClusterAddOns(namespace).Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if len(actual.Status.ConfigReferences) != len(expect) {
			return fmt.Errorf("expected %v config reference, actual: %v", len(expect), len(actual.Status.ConfigReferences))
		}

		for i, e := range expect {
			actualConfigReference := actual.Status.ConfigReferences[i]

			if !apiequality.Semantic.DeepEqual(actualConfigReference, e) {
				return fmt.Errorf("expected mca config reference is %v %v, actual: %v %v",
					e.DesiredConfig,
					e.LastAppliedConfig,
					actualConfigReference.DesiredConfig,
					actualConfigReference.LastAppliedConfig,
				)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertClusterManagementAddOnAnnotationsAlpha(name string) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %v Annotations (Alpha)", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		if _, ok := actual.Annotations[addonapiv1alpha1.AddonLifecycleAnnotationKey]; ok {
			return fmt.Errorf("expected annotation %v to be empty, actual: %v",
				addonapiv1alpha1.AddonLifecycleAnnotationKey,
				actual.Annotations[addonapiv1alpha1.AddonLifecycleAnnotationKey])
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertClusterManagementAddOnAnnotationsBeta(name string) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %v Annotations (Beta)", name))

	// Note: The AddonLifecycleAnnotationKey was removed in v1beta1, so this assertion
	// is a no-op for backward compatibility with test structure. In v1beta1, addon
	// lifecycle management is always handled by the addon-manager.
	gomega.Eventually(func() error {
		_, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		return err
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func updateManifestWorkStatus(client workclientset.Interface, clusterName string, manifestWorkName string, status metav1.ConditionStatus) {
	gomega.Eventually(func() error {
		work, err := client.WorkV1().ManifestWorks(clusterName).Get(context.Background(), manifestWorkName, metav1.GetOptions{})
		if err != nil {
			return err
		}

		meta.SetStatusCondition(
			&work.Status.Conditions,
			metav1.Condition{Type: workapiv1.WorkApplied, Status: status, Reason: "WorkApplied", ObservedGeneration: work.Generation})
		meta.SetStatusCondition(
			&work.Status.Conditions,
			metav1.Condition{Type: workapiv1.WorkAvailable, Status: status, Reason: "WorkAvailable", ObservedGeneration: work.Generation})

		_, err = client.WorkV1().ManifestWorks(clusterName).UpdateStatus(context.Background(), work, metav1.UpdateOptions{})
		return err
	}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
}

func createManagedCluster(hubClusterClient clusterv1client.Interface, managedClusterName string) error {
	managedCluster := &clusterv1.ManagedCluster{
		ObjectMeta: metav1.ObjectMeta{
			Name: managedClusterName,
		},
		Spec: clusterv1.ManagedClusterSpec{
			HubAcceptsClient: true,
		},
	}
	_, err := hubClusterClient.ClusterV1().ManagedClusters().Create(context.Background(), managedCluster, metav1.CreateOptions{})
	if err != nil {
		return err
	}

	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: managedClusterName}}
	_, err = hubKubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
	if err != nil {
		return err
	}
	return nil
}

func createPlacementDecision(hubClusterClient clusterv1client.Interface, placementNamespace, placementName, groupIndex string, clusterNames ...string) error {
	var err error
	decision := &clusterv1beta1.PlacementDecision{
		ObjectMeta: metav1.ObjectMeta{
			Name:      placementName + groupIndex,
			Namespace: placementNamespace,
			Labels: map[string]string{
				clusterv1beta1.PlacementLabel:          placementName,
				clusterv1beta1.DecisionGroupIndexLabel: groupIndex,
			},
		},
	}
	decision, err = hubClusterClient.ClusterV1beta1().PlacementDecisions(placementNamespace).Create(context.Background(), decision, metav1.CreateOptions{})
	if err != nil {
		return err
	}

	clusterDecisions := []clusterv1beta1.ClusterDecision{}
	for _, clusterName := range clusterNames {
		clusterDecisions = append(clusterDecisions, clusterv1beta1.ClusterDecision{ClusterName: clusterName})
	}
	decision.Status.Decisions = clusterDecisions

	_, err = hubClusterClient.ClusterV1beta1().PlacementDecisions(placementNamespace).UpdateStatus(context.Background(), decision, metav1.UpdateOptions{})
	if err != nil {
		return err
	}
	return nil
}

func assertClusterManagementAddOnConditionsAlpha(name string, expect ...metav1.Condition) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s Conditions (Alpha)", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		for i, ec := range expect {
			if i >= len(actual.Status.InstallProgressions) {
				return fmt.Errorf("expected %d install progressions, actual: %d", i+1, len(actual.Status.InstallProgressions))
			}
			cond := meta.FindStatusCondition(actual.Status.InstallProgressions[i].Conditions, ec.Type)
			if cond == nil ||
				cond.Status != ec.Status ||
				cond.Reason != ec.Reason ||
				cond.Message != ec.Message {
				return fmt.Errorf("expected cma progressing condition is %v, actual: %v", ec, cond)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertClusterManagementAddOnConditionsBeta(name string, expect ...metav1.Condition) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s Conditions (Beta)", name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		for i, ec := range expect {
			if i >= len(actual.Status.InstallProgressions) {
				return fmt.Errorf("expected %d install progressions, actual: %d", i+1, len(actual.Status.InstallProgressions))
			}
			cond := meta.FindStatusCondition(actual.Status.InstallProgressions[i].Conditions, ec.Type)
			if cond == nil ||
				cond.Status != ec.Status ||
				cond.Reason != ec.Reason ||
				cond.Message != ec.Message {
				return fmt.Errorf("expected cma progressing condition is %v, actual: %v", ec, cond)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertClusterManagementAddOnNoConditionsAlpha(name string, start metav1.Time, duration time.Duration, expect ...metav1.Condition) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s no conditions in duration %v (Alpha)", name, duration))

	gomega.Consistently(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		elapsedTime := metav1.Now().Sub(start.Time)

		// Only check if we haven't reached the expected timeout duration yet
		if elapsedTime < duration {
			for i, ec := range expect {
				if i >= len(actual.Status.InstallProgressions) {
					return fmt.Errorf("expected %d install progressions, actual: %d", i+1, len(actual.Status.InstallProgressions))
				}
				cond := meta.FindStatusCondition(actual.Status.InstallProgressions[i].Conditions, ec.Type)

				// The expected timeout condition should NOT appear before the duration
				if cond != nil &&
					cond.Status == ec.Status &&
					cond.Reason == ec.Reason &&
					cond.Message == ec.Message {
					return fmt.Errorf("unexpected condition matches before duration (elapsed: %v, expected: %v)", elapsedTime, duration)
				}
			}
		}

		return nil
	}, duration+2*time.Second, eventuallyInterval).Should(gomega.BeNil())
}

func assertClusterManagementAddOnNoConditionsBeta(name string, start metav1.Time, duration time.Duration, expect ...metav1.Condition) {
	ginkgo.By(fmt.Sprintf("Check ClusterManagementAddOn %s no conditions in duration %v (Beta)", name, duration))

	gomega.Consistently(func() error {
		actual, err := hubAddonClient.AddonV1beta1().ClusterManagementAddOns().Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		elapsedTime := metav1.Now().Sub(start.Time)

		// Only check if we haven't reached the expected timeout duration yet
		if elapsedTime < duration {
			for i, ec := range expect {
				if i >= len(actual.Status.InstallProgressions) {
					return fmt.Errorf("expected %d install progressions, actual: %d", i+1, len(actual.Status.InstallProgressions))
				}
				cond := meta.FindStatusCondition(actual.Status.InstallProgressions[i].Conditions, ec.Type)

				// The expected timeout condition should NOT appear before the duration
				if cond != nil &&
					cond.Status == ec.Status &&
					cond.Reason == ec.Reason &&
					cond.Message == ec.Message {
					return fmt.Errorf("unexpected condition matches before duration (elapsed: %v, expected: %v)", elapsedTime, duration)
				}
			}
		}

		return nil
	}, duration+2*time.Second, eventuallyInterval).Should(gomega.BeNil())
}

func assertManagedClusterAddOnConditionsAlpha(name, namespace string, expect ...metav1.Condition) {
	ginkgo.By(fmt.Sprintf("Check ManagedClusterAddOn %s/%s Conditions (Alpha)", namespace, name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1alpha1().ManagedClusterAddOns(namespace).Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		for _, ec := range expect {
			cond := meta.FindStatusCondition(actual.Status.Conditions, ec.Type)
			if cond == nil ||
				cond.Status != ec.Status ||
				cond.Reason != ec.Reason ||
				cond.Message != ec.Message {
				return fmt.Errorf("expected addon progressing condition is %v, actual: %v", ec, cond)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}

func assertManagedClusterAddOnConditionsBeta(name, namespace string, expect ...metav1.Condition) {
	ginkgo.By(fmt.Sprintf("Check ManagedClusterAddOn %s/%s Conditions (Beta)", namespace, name))

	gomega.Eventually(func() error {
		actual, err := hubAddonClient.AddonV1beta1().ManagedClusterAddOns(namespace).Get(context.Background(), name, metav1.GetOptions{})
		if err != nil {
			return err
		}

		for _, ec := range expect {
			cond := meta.FindStatusCondition(actual.Status.Conditions, ec.Type)
			if cond == nil ||
				cond.Status != ec.Status ||
				cond.Reason != ec.Reason ||
				cond.Message != ec.Message {
				return fmt.Errorf("expected addon progressing condition is %v, actual: %v", ec, cond)
			}
		}

		return nil
	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
}