From 12c257987ee4558b2047de97a67678e91fc395bf Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Wed, 3 Jun 2026 22:06:45 -0400 Subject: [PATCH] :seedling: add v1beta1 e2e tests for addon API - preserve v1alpha1 tests (#1553) * test: add v1beta1 e2e tests for addon lifecycle and install strategy Add v1beta1 e2e test coverage following the alpha/beta split pattern from integration tests. Rename framework helper functions to use explicit V1Alpha1/V1Beta1 suffixes for consistency. Framework changes (test/framework/managedclusteraddon.go): - Rename CreateManagedClusterAddOn to CreateManagedClusterAddOnV1Alpha1 - Rename CheckManagedClusterAddOnStatus to CheckManagedClusterAddOnStatusV1Alpha1 - Add CheckManagedClusterAddOnStatusV1Beta1 helper - Organize functions with Alpha version followed by Beta counterpart E2E test changes: - Rename addon_test.go to addon_alpha_test.go (v1alpha1 API) - Create new addon_test.go with v1beta1 API using V1Beta1 helpers - Rename addon_install_test.go to addon_install_alpha_test.go (v1alpha1 API) - Create new addon_install_test.go with v1beta1 API - Update addon_lease_test.go, addon_token_auth_test.go, addonmanagement_test.go to use V1Alpha1 helpers Test coverage (both alpha and v1beta1): - addon_test.go: Basic ManagedClusterAddOn lifecycle and availability - addon_install_test.go: ClusterManagementAddOn with placement-based install strategy, addon annotation syncing Naming pattern matches integration tests: *_alpha_test.go for v1alpha1, *_test.go (no suffix) for v1beta1. Co-Authored-By: Claude Sonnet 4.5 Signed-off-by: Tesshu Flower * test: add v1beta1 e2e tests for addon lease health check Add v1beta1 e2e test coverage for addon lease-based health checking following the alpha/beta split pattern. Changes: - Rename addon_lease_test.go to addon_lease_alpha_test.go (v1alpha1 API) - Create new addon_lease_test.go with v1beta1 API using V1Beta1 helpers - Update all AddonClient.AddonV1alpha1() calls to AddonV1beta1() in beta test Both alpha and v1beta1 tests cover: - Addon status remains available while lease is updated - Addon status changes to unavailable when lease stops updating - Addon status changes to unknown when no lease exists - Addon status changes to unknown when managed cluster lease stops updating Co-Authored-By: Claude Sonnet 4.5 Signed-off-by: Tesshu Flower * test: migrate addon_token_auth test to alpha/beta versions - Rename addon_token_auth_test.go to addon_token_auth_alpha_test.go - Update alpha Describe to include (v1alpha1) - Create new addon_token_auth_test.go for v1beta1 - Update imports and client calls to use v1beta1 API Signed-off-by: Tesshu Flower --------- Signed-off-by: Tesshu Flower Co-authored-by: Claude Sonnet 4.5 --- test/e2e/addon_alpha_test.go | 53 ++ test/e2e/addon_install_alpha_test.go | 251 ++++++ test/e2e/addon_install_test.go | 37 +- test/e2e/addon_lease_alpha_test.go | 337 ++++++++ test/e2e/addon_lease_test.go | 20 +- test/e2e/addon_test.go | 12 +- test/e2e/addon_token_auth_alpha_test.go | 359 ++++++++ test/e2e/addon_token_auth_test.go | 16 +- test/e2e/addonmanagement_alpha_test.go | 1045 +++++++++++++++++++++++ test/e2e/addonmanagement_test.go | 138 +-- test/framework/managedclusteraddon.go | 25 +- 11 files changed, 2183 insertions(+), 110 deletions(-) create mode 100644 test/e2e/addon_alpha_test.go create mode 100644 test/e2e/addon_install_alpha_test.go create mode 100644 test/e2e/addon_lease_alpha_test.go create mode 100644 test/e2e/addon_token_auth_alpha_test.go create mode 100644 test/e2e/addonmanagement_alpha_test.go diff --git a/test/e2e/addon_alpha_test.go b/test/e2e/addon_alpha_test.go new file mode 100644 index 000000000..4e10ed51b --- /dev/null +++ b/test/e2e/addon_alpha_test.go @@ -0,0 +1,53 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" +) + +var _ = Describe("Manage the managed cluster addons (v1alpha1)", Label("addon"), func() { + var addOnName string + BeforeEach(func() { + addOnName = fmt.Sprintf("e2e-addon-%s", rand.String(6)) + }) + + AfterEach(func() { + err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("Create one managed cluster addon and make sure it is available", func() { + By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) + err := hub.CreateManagedClusterAddOnV1Alpha1(universalClusterName, addOnName, addOnName) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("create the addon lease %v on addon install namespace %v", addOnName, addOnName)) + err = hub.CreateManagedClusterAddOnLease(addOnName, addOnName) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("wait the addon %v available condition to be true", addOnName)) + Eventually(func() error { + return hub.CheckManagedClusterAddOnStatusV1Alpha1(universalClusterName, addOnName) + }).Should(Succeed()) + }) + + It("Create one managed cluster addon and make sure it is available in Hosted mode", func() { + By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) + err := hub.CreateManagedClusterAddOnV1Alpha1(universalClusterName, addOnName, addOnName) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("create the addon lease %v on addon install namespace %v", addOnName, addOnName)) + err = hub.CreateManagedClusterAddOnLease(addOnName, addOnName) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("wait the addon %v available condition to be true", addOnName)) + Eventually(func() error { + return hub.CheckManagedClusterAddOnStatusV1Alpha1(universalClusterName, addOnName) + }).Should(Succeed()) + }) +}) diff --git a/test/e2e/addon_install_alpha_test.go b/test/e2e/addon_install_alpha_test.go new file mode 100644 index 000000000..dc4492e2a --- /dev/null +++ b/test/e2e/addon_install_alpha_test.go @@ -0,0 +1,251 @@ +package e2e + +import ( + "context" + "fmt" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + clusterv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" + clusterv1beta1 "open-cluster-management.io/api/cluster/v1beta1" + clusterv1beta2 "open-cluster-management.io/api/cluster/v1beta2" +) + +var _ = ginkgo.Describe("Addon install with install strategy (v1alpha1)", ginkgo.Ordered, ginkgo.Label("addon-install"), func() { + var addOnName string + var clusterNames []string + + ginkgo.BeforeAll(func() { + suffix := rand.String(6) + addOnName = fmt.Sprintf("addon-%s", suffix) + clusterNames = nil + + ginkgo.By("create namespace open-cluster-management-global-set") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "open-cluster-management-global-set", + }, + } + _, err := hub.KubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By("create Placement global in open-cluster-management-global-set") + placement := &clusterv1beta1.Placement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "global", + Namespace: "open-cluster-management-global-set", + }, + Spec: clusterv1beta1.PlacementSpec{ + ClusterSets: []string{"global"}, + Tolerations: []clusterv1beta1.Toleration{ + { + Key: "cluster.open-cluster-management.io/unreachable", + Operator: clusterv1beta1.TolerationOpEqual, + }, + { + Key: "cluster.open-cluster-management.io/unavailable", + Operator: clusterv1beta1.TolerationOpEqual, + }, + }, + DecisionStrategy: clusterv1beta1.DecisionStrategy{ + GroupStrategy: clusterv1beta1.GroupStrategy{}, + }, + PrioritizerPolicy: clusterv1beta1.PrioritizerPolicy{ + Mode: clusterv1beta1.PrioritizerPolicyModeAdditive, + }, + }, + } + _, err = hub.ClusterClient.ClusterV1beta1().Placements("open-cluster-management-global-set").Create( + context.TODO(), placement, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By("create ManagedClusterSetBinding global in open-cluster-management-global-set") + binding := &clusterv1beta2.ManagedClusterSetBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "global", + Namespace: "open-cluster-management-global-set", + }, + Spec: clusterv1beta2.ManagedClusterSetBindingSpec{ + ClusterSet: "global", + }, + } + _, err = hub.ClusterClient.ClusterV1beta2().ManagedClusterSetBindings("open-cluster-management-global-set").Create( + context.TODO(), binding, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By(fmt.Sprintf("create ClusterManagementAddOn %s with install strategy", addOnName)) + cma := &addonapiv1alpha1.ClusterManagementAddOn{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + }, + Spec: addonapiv1alpha1.ClusterManagementAddOnSpec{ + InstallStrategy: addonapiv1alpha1.InstallStrategy{ + Type: addonapiv1alpha1.AddonInstallStrategyPlacements, + Placements: []addonapiv1alpha1.PlacementStrategy{ + { + PlacementRef: addonapiv1alpha1.PlacementRef{ + Name: "global", + Namespace: "open-cluster-management-global-set", + }, + RolloutStrategy: clusterv1alpha1.RolloutStrategy{ + Type: clusterv1alpha1.All, + }, + }, + }, + }, + }, + } + _, err = hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Create( + context.TODO(), cma, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.AfterAll(func() { + ginkgo.By(fmt.Sprintf("delete ClusterManagementAddOn %s", addOnName)) + err := hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Delete( + context.TODO(), addOnName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + for _, clusterName := range clusterNames { + ginkgo.By(fmt.Sprintf("delete ManagedCluster %s", clusterName)) + err := hub.ClusterClient.ClusterV1().ManagedClusters().Delete( + context.TODO(), clusterName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + } + + ginkgo.By("delete namespace open-cluster-management-global-set") + err = hub.KubeClient.CoreV1().Namespaces().Delete( + context.TODO(), "open-cluster-management-global-set", metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + }) + + ginkgo.It("Should create addon without addon annotations when managed cluster has no addon annotations", func() { + clusterName := fmt.Sprintf("e2e-addon-install-%s", rand.String(6)) + clusterNames = append(clusterNames, clusterName) + + ginkgo.By(fmt.Sprintf("create ManagedCluster %s with non-addon annotations", clusterName)) + managedCluster := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "foo.example.com/bar": "value1", + "baz.example.com/qux": "value2", + }, + }, + Spec: clusterv1.ManagedClusterSpec{ + HubAcceptsClient: true, + }, + } + _, err := hub.ClusterClient.ClusterV1().ManagedClusters().Create( + context.TODO(), managedCluster, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("check ManagedClusterAddOn %s is created in cluster namespace %s", addOnName, clusterName)) + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + + if len(addon.Annotations) != 0 { + return fmt.Errorf("expected no annotations on ManagedClusterAddOn, got %v", addon.Annotations) + } + return nil + }).Should(gomega.Succeed()) + }) + + ginkgo.It("Should sync addon annotations from managed cluster to addon", func() { + clusterName := fmt.Sprintf("e2e-addon-install-%s", rand.String(6)) + clusterNames = append(clusterNames, clusterName) + + annotations := map[string]string{ + "addon.open-cluster-management.io/hosting-cluster-name": "hosting-cluster", + "addon.open-cluster-management.io/test-annotation": "test-value", + } + + ginkgo.By(fmt.Sprintf("create ManagedCluster %s with addon annotations", clusterName)) + managedCluster := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: annotations, + }, + Spec: clusterv1.ManagedClusterSpec{ + HubAcceptsClient: true, + }, + } + _, err := hub.ClusterClient.ClusterV1().ManagedClusters().Create( + context.TODO(), managedCluster, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("check ManagedClusterAddOn %s is created with synced annotations", addOnName)) + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + + for key, expectedValue := range annotations { + actualValue, ok := addon.Annotations[key] + if !ok { + return fmt.Errorf("expected annotation %s not found on ManagedClusterAddOn", key) + } + if actualValue != expectedValue { + return fmt.Errorf("annotation %s: expected %q, got %q", key, expectedValue, actualValue) + } + } + return nil + }).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("update annotation on ManagedCluster %s", clusterName)) + gomega.Eventually(func() error { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get( + context.TODO(), clusterName, metav1.GetOptions{}) + if err != nil { + return err + } + cluster.Annotations["addon.open-cluster-management.io/test-annotation"] = "updated-value" + _, err = hub.ClusterClient.ClusterV1().ManagedClusters().Update( + context.TODO(), cluster, metav1.UpdateOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("check updated annotation is synced to ManagedClusterAddOn %s", addOnName)) + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + + actualValue, ok := addon.Annotations["addon.open-cluster-management.io/test-annotation"] + if !ok { + return fmt.Errorf("expected annotation addon.open-cluster-management.io/test-annotation not found on ManagedClusterAddOn") + } + if actualValue != "updated-value" { + return fmt.Errorf("annotation addon.open-cluster-management.io/test-annotation: expected %q, got %q", "updated-value", actualValue) + } + return nil + }).Should(gomega.Succeed()) + }) +}) diff --git a/test/e2e/addon_install_test.go b/test/e2e/addon_install_test.go index 38071dfdd..6ebe73be4 100644 --- a/test/e2e/addon_install_test.go +++ b/test/e2e/addon_install_test.go @@ -11,14 +11,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" - addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1" clusterv1 "open-cluster-management.io/api/cluster/v1" clusterv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" clusterv1beta1 "open-cluster-management.io/api/cluster/v1beta1" clusterv1beta2 "open-cluster-management.io/api/cluster/v1beta2" ) -var _ = ginkgo.Describe("Addon install with install strategy", ginkgo.Ordered, ginkgo.Label("addon-install"), func() { +var _ = ginkgo.Describe("Addon install with install strategy (v1beta1)", ginkgo.Ordered, ginkgo.Label("addon-install"), func() { var addOnName string var clusterNames []string @@ -87,16 +87,16 @@ var _ = ginkgo.Describe("Addon install with install strategy", ginkgo.Ordered, g } ginkgo.By(fmt.Sprintf("create ClusterManagementAddOn %s with install strategy", addOnName)) - cma := &addonapiv1alpha1.ClusterManagementAddOn{ + cma := &addonapiv1beta1.ClusterManagementAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addOnName, }, - Spec: addonapiv1alpha1.ClusterManagementAddOnSpec{ - InstallStrategy: addonapiv1alpha1.InstallStrategy{ - Type: addonapiv1alpha1.AddonInstallStrategyPlacements, - Placements: []addonapiv1alpha1.PlacementStrategy{ + Spec: addonapiv1beta1.ClusterManagementAddOnSpec{ + InstallStrategy: addonapiv1beta1.InstallStrategy{ + Type: addonapiv1beta1.AddonInstallStrategyPlacements, + Placements: []addonapiv1beta1.PlacementStrategy{ { - PlacementRef: addonapiv1alpha1.PlacementRef{ + PlacementRef: addonapiv1beta1.PlacementRef{ Name: "global", Namespace: "open-cluster-management-global-set", }, @@ -108,14 +108,14 @@ var _ = ginkgo.Describe("Addon install with install strategy", ginkgo.Ordered, g }, }, } - _, err = hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Create( + _, err = hub.AddonClient.AddonV1beta1().ClusterManagementAddOns().Create( context.TODO(), cma, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) }) ginkgo.AfterAll(func() { ginkgo.By(fmt.Sprintf("delete ClusterManagementAddOn %s", addOnName)) - err := hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Delete( + err := hub.AddonClient.AddonV1beta1().ClusterManagementAddOns().Delete( context.TODO(), addOnName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -161,14 +161,21 @@ var _ = ginkgo.Describe("Addon install with install strategy", ginkgo.Ordered, g ginkgo.By(fmt.Sprintf("check ManagedClusterAddOn %s is created in cluster namespace %s", addOnName, clusterName)) gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(clusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } - if len(addon.Annotations) != 0 { - return fmt.Errorf("expected no annotations on ManagedClusterAddOn, got %v", addon.Annotations) + // In v1beta1, the installNamespace annotation is expected because: + // - v1alpha1 is the storage version + // - installNamespace field was removed from v1beta1 spec + // - It's preserved as an annotation during v1alpha1->v1beta1 conversion + // We only check that no *user* annotations are present (non-system annotations) + for key := range addon.Annotations { + if key != addonapiv1beta1.InstallNamespaceAnnotation { + return fmt.Errorf("expected no user annotations on ManagedClusterAddOn, got %v", addon.Annotations) + } } return nil }).Should(gomega.Succeed()) @@ -199,7 +206,7 @@ var _ = ginkgo.Describe("Addon install with install strategy", ginkgo.Ordered, g ginkgo.By(fmt.Sprintf("check ManagedClusterAddOn %s is created with synced annotations", addOnName)) gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(clusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err @@ -232,7 +239,7 @@ var _ = ginkgo.Describe("Addon install with install strategy", ginkgo.Ordered, g ginkgo.By(fmt.Sprintf("check updated annotation is synced to ManagedClusterAddOn %s", addOnName)) gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(clusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err diff --git a/test/e2e/addon_lease_alpha_test.go b/test/e2e/addon_lease_alpha_test.go new file mode 100644 index 000000000..611cbac32 --- /dev/null +++ b/test/e2e/addon_lease_alpha_test.go @@ -0,0 +1,337 @@ +package e2e + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + coordv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/klog/v2" + + clusterv1 "open-cluster-management.io/api/cluster/v1" + operatorapiv1 "open-cluster-management.io/api/operator/v1" + + "open-cluster-management.io/ocm/test/framework" +) + +var _ = ginkgo.Describe("Addon Health Check (v1alpha1)", ginkgo.Label("addon-lease"), func() { + const availableLabelValue = "available" + ginkgo.Context("Checking addon lease on managed cluster to update addon status", func() { + var addOnName string + ginkgo.BeforeEach(func() { + // create an addon on created managed cluster + addOnName = fmt.Sprintf("addon-%s", rand.String(6)) + ginkgo.By(fmt.Sprintf("Creating managed cluster addon %q", addOnName)) + err := hub.CreateManagedClusterAddOnV1Alpha1(universalClusterName, addOnName, addOnName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // create addon installation namespace + ginkgo.By(fmt.Sprintf("Creating managed cluster addon installation namespace %q", addOnName)) + _, err = spoke.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName)) + err := spoke.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Should keep addon status to available", func() { + ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOnName, addOnName)) + _, err := spoke.KubeClient.CoordinationV1().Leases(addOnName).Create(context.TODO(), &coordv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + Namespace: addOnName, + }, + Spec: coordv1.LeaseSpec{ + RenewTime: &metav1.MicroTime{Time: time.Now()}, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(found.Status.Conditions, "Available") { + return fmt.Errorf("condition should be available, got %v", found.Status.Conditions) + } + return nil + }).Should(gomega.Succeed()) + + // check if the cluster has a label for addon with expected value + gomega.Eventually(func() bool { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return false + } + if len(cluster.Labels) == 0 { + return false + } + key := fmt.Sprintf("feature.open-cluster-management.io/addon-%s", addOnName) + return cluster.Labels[key] == availableLabelValue + }).Should(gomega.BeTrue()) + }) + + ginkgo.It("Should update addon status to unavailable if addon stops to update its lease", func() { + ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOnName, addOnName)) + _, err := spoke.KubeClient.CoordinationV1().Leases(addOnName).Create(context.TODO(), &coordv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + Namespace: addOnName, + }, + Spec: coordv1.LeaseSpec{ + RenewTime: &metav1.MicroTime{Time: time.Now()}, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(found.Status.Conditions, "Available") { + return fmt.Errorf("condition should be available") + } + return nil + }).Should(gomega.Succeed()) + + // check if the cluster has a label for addon with expected value + gomega.Eventually(func() bool { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return false + } + if len(cluster.Labels) == 0 { + return false + } + key := fmt.Sprintf("feature.open-cluster-management.io/addon-%s", addOnName) + return cluster.Labels[key] == availableLabelValue + }).Should(gomega.BeTrue()) + + ginkgo.By(fmt.Sprintf("Updating lease %q with a past time", addOnName)) + lease, err := spoke.KubeClient.CoordinationV1().Leases(addOnName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + lease.Spec.RenewTime = &metav1.MicroTime{Time: time.Now().Add(-10 * time.Minute)} + _, err = spoke.KubeClient.CoordinationV1().Leases(addOnName).Update(context.TODO(), lease, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionFalse(found.Status.Conditions, "Available") { + return fmt.Errorf("condition should be available") + } + return nil + }).Should(gomega.Succeed()) + + // check if the cluster has a label for addon with expected value + gomega.Eventually(func() bool { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return false + } + if len(cluster.Labels) == 0 { + return false + } + key := fmt.Sprintf("feature.open-cluster-management.io/addon-%s", addOnName) + return cluster.Labels[key] == "unhealthy" + }).Should(gomega.BeTrue()) + }) + + ginkgo.It("Should update addon status to unknown if there is no lease for this addon", func() { + ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOnName, addOnName)) + _, err := spoke.KubeClient.CoordinationV1().Leases(addOnName).Create(context.TODO(), &coordv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + Namespace: addOnName, + }, + Spec: coordv1.LeaseSpec{ + RenewTime: &metav1.MicroTime{Time: time.Now()}, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(found.Status.Conditions, "Available") { + return fmt.Errorf("condition should be available") + } + return nil + }).Should(gomega.Succeed()) + + // check if the cluster has a label for addon with expected value + gomega.Eventually(func() bool { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return false + } + if len(cluster.Labels) == 0 { + return false + } + key := fmt.Sprintf("feature.open-cluster-management.io/addon-%s", addOnName) + return cluster.Labels[key] == availableLabelValue + }).Should(gomega.BeTrue()) + + ginkgo.By(fmt.Sprintf("Deleting lease %q", addOnName)) + err = spoke.KubeClient.CoordinationV1().Leases(addOnName).Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(found.Status.Conditions, "Available") { + return fmt.Errorf("condition should be available") + } + return nil + }).Should(gomega.Succeed()) + + // check if the cluster has a label for addon with expected value + gomega.Eventually(func() bool { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return false + } + if len(cluster.Labels) == 0 { + return false + } + key := fmt.Sprintf("feature.open-cluster-management.io/addon-%s", addOnName) + return cluster.Labels[key] == "unreachable" + }).Should(gomega.BeTrue()) + }) + }) + + ginkgo.Context("Checking managed cluster status to update addon status", func() { + var klusterletName, clusterName, addOnName string + ginkgo.BeforeEach(func() { + klusterletName = fmt.Sprintf("e2e-klusterlet-%s", rand.String(6)) + clusterName = fmt.Sprintf("e2e-managedcluster-%s", rand.String(6)) + agentNamespace := fmt.Sprintf("open-cluster-management-agent-%s", rand.String(6)) + framework.CreateAndApproveKlusterlet( + hub, spoke, + klusterletName, clusterName, agentNamespace, operatorapiv1.InstallMode(klusterletDeployMode), bootstrapHubKubeConfigSecret, images, registrationDriver) + // create an addon on created managed cluster + addOnName = fmt.Sprintf("addon-%s", rand.String(6)) + ginkgo.By(fmt.Sprintf("Creating managed cluster addon %q", addOnName)) + gomega.Expect(hub.CreateManagedClusterAddOnV1Alpha1(clusterName, addOnName, addOnName)).ToNot(gomega.HaveOccurred()) + + // create addon installation namespace + ginkgo.By(fmt.Sprintf("Creating managed cluster addon installation namespace %q", addOnName)) + _, err := spoke.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("Cleaning managed cluster addon installation namespace %q", addOnName)) + err := hub.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("clean klusterlet %v resources after the test case", klusterletName)) + framework.CleanKlusterletRelatedResources(hub, spoke, klusterletName, clusterName) + }) + + ginkgo.It("Should update addon status to unknown if managed cluster stops to update its lease", func() { + ginkgo.By(fmt.Sprintf("Creating lease %q for managed cluster addon %q", addOnName, addOnName)) + _, err := spoke.KubeClient.CoordinationV1().Leases(addOnName).Create(context.TODO(), &coordv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: addOnName, + Namespace: addOnName, + }, + Spec: coordv1.LeaseSpec{ + RenewTime: &metav1.MicroTime{Time: time.Now()}, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if found.Status.Conditions == nil { + return fmt.Errorf("condition should not be nil") + } + cond := meta.FindStatusCondition(found.Status.Conditions, "Available") + if cond.Status != metav1.ConditionTrue { + return fmt.Errorf("available status should be true") + } + return nil + }).Should(gomega.Succeed()) + + // delete registration agent to stop agent update its status + ginkgo.By("Stoping klusterlet") + err = spoke.OperatorClient.OperatorV1().Klusterlets().Delete(context.TODO(), klusterletName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + _, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(), klusterletName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + klog.Infof("klusterlet %s deleted successfully", klusterletName) + return nil + } + if err != nil { + klog.Infof("get klusterlet %s error: %v", klusterletName, err) + return err + } + return fmt.Errorf("klusterlet is still deleting") + }).Should(gomega.Succeed()) + + // for speeding up test, update managed cluster status to unknown manually + ginkgo.By(fmt.Sprintf("Updating managed cluster %s status to unknown", clusterName)) + found, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + found.Status = clusterv1.ManagedClusterStatus{ + Conditions: []metav1.Condition{ + { + Type: clusterv1.ManagedClusterConditionAvailable, + Status: metav1.ConditionUnknown, + Reason: "ManagedClusterLeaseUpdateStopped", + Message: "Registration agent stopped updating its lease.", + LastTransitionTime: metav1.Now(), + }, + }, + } + _, err = hub.ClusterClient.ClusterV1().ManagedClusters().UpdateStatus(context.TODO(), found, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + if found.Status.Conditions == nil { + return fmt.Errorf("condition should not be nil") + } + cond := meta.FindStatusCondition(found.Status.Conditions, "Available") + if cond.Status != metav1.ConditionUnknown { + return fmt.Errorf("available status should be unknown") + } + return nil + }).Should(gomega.Succeed()) + }) + }) +}) diff --git a/test/e2e/addon_lease_test.go b/test/e2e/addon_lease_test.go index 2c41ac147..90022a962 100644 --- a/test/e2e/addon_lease_test.go +++ b/test/e2e/addon_lease_test.go @@ -23,14 +23,14 @@ import ( const availableLabelValue = "available" -var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func() { +var _ = ginkgo.Describe("Addon Health Check (v1beta1)", ginkgo.Label("addon-lease"), func() { ginkgo.Context("Checking addon lease on managed cluster to update addon status", func() { var addOnName string ginkgo.BeforeEach(func() { // create an addon on created managed cluster addOnName = fmt.Sprintf("addon-%s", rand.String(6)) ginkgo.By(fmt.Sprintf("Creating managed cluster addon %q", addOnName)) - err := hub.CreateManagedClusterAddOn(universalClusterName, addOnName, addOnName) + err := hub.CreateManagedClusterAddOnV1Beta1(universalClusterName, addOnName, addOnName) gomega.Expect(err).ToNot(gomega.HaveOccurred()) // create addon installation namespace @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } @@ -101,7 +101,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } @@ -132,7 +132,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } @@ -170,7 +170,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } @@ -198,7 +198,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } @@ -235,7 +235,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( // create an addon on created managed cluster addOnName = fmt.Sprintf("addon-%s", rand.String(6)) ginkgo.By(fmt.Sprintf("Creating managed cluster addon %q", addOnName)) - gomega.Expect(hub.CreateManagedClusterAddOn(clusterName, addOnName, addOnName)).ToNot(gomega.HaveOccurred()) + gomega.Expect(hub.CreateManagedClusterAddOnV1Beta1(clusterName, addOnName, addOnName)).ToNot(gomega.HaveOccurred()) // create addon installation namespace ginkgo.By(fmt.Sprintf("Creating managed cluster addon installation namespace %q", addOnName)) @@ -269,7 +269,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(clusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } @@ -320,7 +320,7 @@ var _ = ginkgo.Describe("Addon Health Check", ginkgo.Label("addon-lease"), func( gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - found, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(clusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) + found, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(clusterName).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err } diff --git a/test/e2e/addon_test.go b/test/e2e/addon_test.go index fdd07e052..990466718 100644 --- a/test/e2e/addon_test.go +++ b/test/e2e/addon_test.go @@ -10,20 +10,20 @@ import ( "k8s.io/apimachinery/pkg/util/rand" ) -var _ = Describe("Manage the managed cluster addons", Label("addon"), func() { +var _ = Describe("Manage the managed cluster addons (v1beta1)", Label("addon"), func() { var addOnName string BeforeEach(func() { addOnName = fmt.Sprintf("e2e-addon-%s", rand.String(6)) }) AfterEach(func() { - err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) + err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Delete(context.TODO(), addOnName, metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) }) It("Create one managed cluster addon and make sure it is available", func() { By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) - err := hub.CreateManagedClusterAddOn(universalClusterName, addOnName, addOnName) + err := hub.CreateManagedClusterAddOnV1Beta1(universalClusterName, addOnName, addOnName) Expect(err).ToNot(HaveOccurred()) By(fmt.Sprintf("create the addon lease %v on addon install namespace %v", addOnName, addOnName)) @@ -32,13 +32,13 @@ var _ = Describe("Manage the managed cluster addons", Label("addon"), func() { By(fmt.Sprintf("wait the addon %v available condition to be true", addOnName)) Eventually(func() error { - return hub.CheckManagedClusterAddOnStatus(universalClusterName, addOnName) + return hub.CheckManagedClusterAddOnStatusV1Beta1(universalClusterName, addOnName) }).Should(Succeed()) }) It("Create one managed cluster addon and make sure it is available in Hosted mode", func() { By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) - err := hub.CreateManagedClusterAddOn(universalClusterName, addOnName, addOnName) + err := hub.CreateManagedClusterAddOnV1Beta1(universalClusterName, addOnName, addOnName) Expect(err).ToNot(HaveOccurred()) By(fmt.Sprintf("create the addon lease %v on addon install namespace %v", addOnName, addOnName)) @@ -47,7 +47,7 @@ var _ = Describe("Manage the managed cluster addons", Label("addon"), func() { By(fmt.Sprintf("wait the addon %v available condition to be true", addOnName)) Eventually(func() error { - return hub.CheckManagedClusterAddOnStatus(universalClusterName, addOnName) + return hub.CheckManagedClusterAddOnStatusV1Beta1(universalClusterName, addOnName) }).Should(Succeed()) }) }) diff --git a/test/e2e/addon_token_auth_alpha_test.go b/test/e2e/addon_token_auth_alpha_test.go new file mode 100644 index 000000000..d0c46b139 --- /dev/null +++ b/test/e2e/addon_token_auth_alpha_test.go @@ -0,0 +1,359 @@ +package e2e + +import ( + "context" + "fmt" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + operatorapiv1 "open-cluster-management.io/api/operator/v1" + + "open-cluster-management.io/ocm/pkg/addon/templateagent" + "open-cluster-management.io/ocm/test/e2e/manifests" +) + +var _ = ginkgo.Describe("Template addon with token-based authentication (v1alpha1)", ginkgo.Ordered, ginkgo.Label("addon-manager", "addon-token-auth"), func() { + addOnName := "hello-template" + addonInstallNamespace := "test-addon-template-token" + var signerSecretNamespace string + var originalAddOnDriver *operatorapiv1.AddOnRegistrationDriver + + var agentClient kubernetes.Interface + var agentNamespace string + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = addonapiv1alpha1.Install(s) + _ = clusterv1.Install(s) + + templateResources := []string{ + "addon/addon_template.yaml", + "addon/cluster_management_addon.yaml", + "addon/cluster_role.yaml", + "addon/signca_secret_role.yaml", + "addon/signca_secret_rolebinding.yaml", + } + + ginkgo.BeforeAll(func() { + ginkgo.By("Save original klusterlet configuration") + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if klusterlet.Spec.RegistrationConfiguration != nil { + originalAddOnDriver = klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver + } + + ginkgo.By("Get initial registration agent deployment generation before updating klusterlet") + var initialGeneration int64 + var registrationDeploymentName string + registrationDeploymentName = fmt.Sprintf("%s-registration-agent", klusterlet.Name) + if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeSingleton || + klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeSingletonHosted { + registrationDeploymentName = fmt.Sprintf("%s-agent", klusterlet.Name) + } + + // In hosted mode, agents run on the hub cluster, otherwise on the spoke cluster + agentClient = spoke.KubeClient + agentNamespace = universalAgentNamespace + if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted || + klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeSingletonHosted { + agentClient = hub.KubeClient + agentNamespace = klusterlet.Name + } + + deployment, err := agentClient.AppsV1().Deployments(agentNamespace).Get( + context.TODO(), registrationDeploymentName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + initialGeneration = deployment.Generation + + ginkgo.By("Update klusterlet to use token-based authentication for addons") + gomega.Eventually(func() error { + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + if klusterlet.Spec.RegistrationConfiguration == nil { + klusterlet.Spec.RegistrationConfiguration = &operatorapiv1.RegistrationConfiguration{} + } + + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver = &operatorapiv1.AddOnRegistrationDriver{ + AuthType: "token", + Token: &operatorapiv1.TokenConfig{ + ExpirationSeconds: 3600, // 1 hour for testing + }, + } + + _, err = spoke.OperatorClient.OperatorV1().Klusterlets().Update( + context.TODO(), klusterlet, metav1.UpdateOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By("Verify klusterlet is updated with token auth configuration") + gomega.Eventually(func() error { + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + if klusterlet.Spec.RegistrationConfiguration == nil || + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver == nil { + return fmt.Errorf("token auth configuration not set") + } + + if klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver.AuthType != "token" { + return fmt.Errorf("auth type is not token: %s", + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver.AuthType) + } + + return nil + }).Should(gomega.Succeed()) + + ginkgo.By("Wait for registration agent deployment to rollout with new token auth configuration") + gomega.Eventually(func() error { + deployment, err := agentClient.AppsV1().Deployments(agentNamespace).Get( + context.TODO(), registrationDeploymentName, metav1.GetOptions{}) + if err != nil { + return err + } + + // Wait for deployment generation to increment (indicates config change was applied) + if deployment.Generation <= initialGeneration { + return fmt.Errorf("deployment generation has not incremented yet: current=%d, initial=%d", + deployment.Generation, initialGeneration) + } + + // Ensure the deployment controller has observed the latest spec + if deployment.Status.ObservedGeneration != deployment.Generation { + return fmt.Errorf("deployment has not observed latest generation: observed=%d, current=%d", + deployment.Status.ObservedGeneration, deployment.Generation) + } + + // Ensure all replicas have been updated with the new configuration + if deployment.Status.UpdatedReplicas != deployment.Status.Replicas { + return fmt.Errorf("deployment has not updated all replicas: updated=%d, total=%d", + deployment.Status.UpdatedReplicas, deployment.Status.Replicas) + } + + // Ensure all updated replicas are ready + if deployment.Status.ReadyReplicas != deployment.Status.Replicas { + return fmt.Errorf("deployment not fully ready: ready=%d, total=%d", + deployment.Status.ReadyReplicas, deployment.Status.Replicas) + } + + // Ensure there are no unavailable replicas + if deployment.Status.UnavailableReplicas > 0 { + return fmt.Errorf("deployment has unavailable replicas: %d", deployment.Status.UnavailableReplicas) + } + + return nil + }, "2m", "5s").Should(gomega.Succeed()) + + signerSecretNamespace = "signer-secret-token-ns-" + rand.String(6) + ginkgo.By("Create addon custom sign secret namespace") + _, err = hub.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: signerSecretNamespace, + }, + }, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By("Create addon custom sign secret") + err = copySignerSecret(context.TODO(), hub.KubeClient, "open-cluster-management-hub", + "signer-secret", signerSecretNamespace, customSignerSecretName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + }) + + ginkgo.AfterAll(func() { + ginkgo.By("Delete addon custom sign secret") + err := hub.KubeClient.CoreV1().Secrets(signerSecretNamespace).Delete(context.TODO(), + customSignerSecretName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret %v/%v: %v", + signerSecretNamespace, customSignerSecretName, err)) + } + + ginkgo.By("Delete addon custom sign secret namespace") + err = hub.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), signerSecretNamespace, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret namespace %v: %v", signerSecretNamespace, err)) + } + + ginkgo.By("Restore original klusterlet AddOnKubeClientRegistrationDriver configuration") + gomega.Eventually(func() error { + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + if klusterlet.Spec.RegistrationConfiguration == nil { + klusterlet.Spec.RegistrationConfiguration = &operatorapiv1.RegistrationConfiguration{} + } + + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver = originalAddOnDriver + + _, err = spoke.OperatorClient.OperatorV1().Klusterlets().Update( + context.TODO(), klusterlet, metav1.UpdateOptions{}) + return err + }).Should(gomega.Succeed()) + + }) + + ginkgo.It("Should work with token-based authentication flow", func() { + var err error + + ginkgo.By("Step 1: Create addon template resources") + err = createResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s, + defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{ + "Namespace": universalClusterName, + "AddonInstallNamespace": addonInstallNamespace, + "CustomSignerName": customSignerName, + "AddonManagerNamespace": templateagent.AddonManagerNamespace(), + "CustomSignerSecretName": customSignerSecretName, + "CustomSignerSecretNamespace": signerSecretNamespace, + }), + templateResources, + ) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Step 2: Create the template addon") + err = hub.CreateManagedClusterAddOnV1Alpha1(universalClusterName, addOnName, addonInstallNamespace) + if err != nil { + gomega.Expect(errors.IsAlreadyExists(err)).To(gomega.BeTrue()) + } + + ginkgo.By("Step 3: Wait for addon to become available with token authentication") + gomega.Eventually(func() error { + return hub.CheckManagedClusterAddOnStatusV1Alpha1(universalClusterName, addOnName) + }, "5m", "10s").Should(gomega.Succeed()) + + ginkgo.By("Step 4: Verify hub kubeconfig secret is created with token authentication") + gomega.Eventually(func() error { + secret, err := agentClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(), + templateagent.HubKubeconfigSecretName(addOnName), metav1.GetOptions{}) + if err != nil { + return err + } + + // Verify the secret contains token-based kubeconfig + token, ok := secret.Data["token"] + if !ok { + return fmt.Errorf("token not found in secret") + } + + // Token should not be empty + if len(token) == 0 { + return fmt.Errorf("token is empty") + } + + return nil + }).Should(gomega.Succeed()) + + ginkgo.By("Step 5: Verify custom client cert secret is created") + gomega.Eventually(func() error { + _, err := agentClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(), + templateagent.CustomSignedSecretName(addOnName, customSignerName), metav1.GetOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By("Step 6: Test addon functionality - create configmap on hub") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-token-%s", rand.String(6)), + Namespace: universalClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err = hub.KubeClient.CoreV1().ConfigMaps(universalClusterName).Create( + context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Step 7: Verify addon copies configmap to spoke using token auth") + gomega.Eventually(func() error { + copiedConfig, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get( + context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !equality.Semantic.DeepEqual(copiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copiedConfig.Data) + } + return nil + }, "2m", "5s").ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Step 8: Cleanup - Delete the addon") + err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err == nil { + return fmt.Errorf("the managedClusterAddon %s should be deleted", addOnName) + } + if err != nil && !errors.IsNotFound(err) { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Step 9: Delete addon template resources") + err = deleteResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s, + defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{ + "Namespace": universalClusterName, + "AddonInstallNamespace": addonInstallNamespace, + "CustomSignerName": customSignerName, + "AddonManagerNamespace": templateagent.AddonManagerNamespace(), + "CustomSignerSecretName": customSignerSecretName, + "CustomSignerSecretNamespace": signerSecretNamespace, + }), + templateResources, + ) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Step 10: Cleanup CSRs") + gomega.Eventually(func() error { + csrs, err := hub.KubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), + metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1alpha1.AddonLabelKey, addOnName, + clusterv1.ClusterNameLabelKey, universalClusterName), + }) + if err != nil { + return err + } + + for _, csr := range csrs.Items { + err = hub.KubeClient.CertificatesV1().CertificateSigningRequests().Delete(context.TODO(), + csr.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + }) +}) diff --git a/test/e2e/addon_token_auth_test.go b/test/e2e/addon_token_auth_test.go index 40607f64e..eeaec63ec 100644 --- a/test/e2e/addon_token_auth_test.go +++ b/test/e2e/addon_token_auth_test.go @@ -15,7 +15,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" - addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1" clusterv1 "open-cluster-management.io/api/cluster/v1" operatorapiv1 "open-cluster-management.io/api/operator/v1" @@ -23,7 +23,7 @@ import ( "open-cluster-management.io/ocm/test/e2e/manifests" ) -var _ = ginkgo.Describe("Template addon with token-based authentication", ginkgo.Ordered, ginkgo.Label("addon-manager", "addon-token-auth"), func() { +var _ = ginkgo.Describe("Template addon with token-based authentication (v1beta1)", ginkgo.Ordered, ginkgo.Label("addon-manager", "addon-token-auth"), func() { addOnName := "hello-template" addonInstallNamespace := "test-addon-template-token" var signerSecretNamespace string @@ -33,7 +33,7 @@ var _ = ginkgo.Describe("Template addon with token-based authentication", ginkgo var agentNamespace string s := runtime.NewScheme() _ = scheme.AddToScheme(s) - _ = addonapiv1alpha1.Install(s) + _ = addonapiv1beta1.Install(s) _ = clusterv1.Install(s) templateResources := []string{ @@ -234,14 +234,14 @@ var _ = ginkgo.Describe("Template addon with token-based authentication", ginkgo gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Step 2: Create the template addon") - err = hub.CreateManagedClusterAddOn(universalClusterName, addOnName, addonInstallNamespace) + err = hub.CreateManagedClusterAddOnV1Beta1(universalClusterName, addOnName, addonInstallNamespace) if err != nil { gomega.Expect(errors.IsAlreadyExists(err)).To(gomega.BeTrue()) } ginkgo.By("Step 3: Wait for addon to become available with token authentication") gomega.Eventually(func() error { - return hub.CheckManagedClusterAddOnStatus(universalClusterName, addOnName) + return hub.CheckManagedClusterAddOnStatusV1Beta1(universalClusterName, addOnName) }, "5m", "10s").Should(gomega.Succeed()) ginkgo.By("Step 4: Verify hub kubeconfig secret is created with token authentication") @@ -304,12 +304,12 @@ var _ = ginkgo.Describe("Template addon with token-based authentication", ginkgo }, "2m", "5s").ShouldNot(gomega.HaveOccurred()) ginkgo.By("Step 8: Cleanup - Delete the addon") - err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Delete( context.TODO(), addOnName, metav1.DeleteOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + _, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err == nil { return fmt.Errorf("the managedClusterAddon %s should be deleted", addOnName) @@ -338,7 +338,7 @@ var _ = ginkgo.Describe("Template addon with token-based authentication", ginkgo gomega.Eventually(func() error { csrs, err := hub.KubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1alpha1.AddonLabelKey, addOnName, + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1beta1.AddonLabelKey, addOnName, clusterv1.ClusterNameLabelKey, universalClusterName), }) if err != nil { diff --git a/test/e2e/addonmanagement_alpha_test.go b/test/e2e/addonmanagement_alpha_test.go new file mode 100644 index 000000000..44a77779d --- /dev/null +++ b/test/e2e/addonmanagement_alpha_test.go @@ -0,0 +1,1045 @@ +package e2e + +import ( + "context" + "encoding/json" + "fmt" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + workapiv1 "open-cluster-management.io/api/work/v1" + + "open-cluster-management.io/ocm/pkg/addon/templateagent" + "open-cluster-management.io/ocm/test/e2e/manifests" +) + +var ( + registriesAlpha = []addonapiv1alpha1.ImageMirror{ + { + Source: "quay.io/open-cluster-management/addon-examples", + Mirror: "quay.io/ocm/addon-examples", + }, + } + proxyConfigAlpha = addonapiv1alpha1.ProxyConfig{ + HTTPProxy: "http://proxy.example.com", + HTTPSProxy: "http://proxy.example.com", + NoProxy: "localhost", + CABundle: []byte("test-ca-bundle"), + } + resourceRequirementsConfigAlpha = []addonapiv1alpha1.ContainerResourceRequirements{ + { + ContainerID: "*:*:helloworld-agent", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + }, + }, + } +) + +var _ = ginkgo.Describe("Addon management (v1alpha1)", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() { + addOnName := "hello-template" + addonInstallNamespace := "test-addon-template" + + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = clusterv1.Install(s) + _ = addonapiv1alpha1.Install(s) + + templateResources := []string{ + "addon/addon_template.yaml", + "addon/cluster_management_addon.yaml", + "addon/cluster_role.yaml", + "addon/signca_secret_role.yaml", + "addon/signca_secret_rolebinding.yaml", + } + + var signerSecretNamespace string + + ginkgo.BeforeEach(func() { + signerSecretNamespace = "signer-secret-test-ns-" + rand.String(6) + + ginkgo.By("create addon custom sign secret namespace") + _, err := hub.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: signerSecretNamespace, + }, + }, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By("create addon custom sign secret") + err = copySignerSecret(context.TODO(), hub.KubeClient, "open-cluster-management-hub", + "signer-secret", signerSecretNamespace, customSignerSecretName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // the addon manager deployment should be running + gomega.Eventually(func() error { + return hub.CheckHubReady() + }).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("create addon template resources for cluster %v", universalClusterName)) + err = createResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s, + defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{ + "Namespace": universalClusterName, + "AddonInstallNamespace": addonInstallNamespace, + "CustomSignerName": customSignerName, + "AddonManagerNamespace": templateagent.AddonManagerNamespace(), + "CustomSignerSecretName": customSignerSecretName, + "CustomSignerSecretNamespace": signerSecretNamespace, + }), + templateResources, + ) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) + err = hub.CreateManagedClusterAddOnV1Alpha1(universalClusterName, addOnName, addonInstallNamespace) + if err != nil { + klog.Errorf("failed to create managed cluster addon %v on the managed cluster namespace %v: %v", addOnName, universalClusterName, err) + gomega.Expect(errors.IsAlreadyExists(err)).To(gomega.BeTrue()) + } + + ginkgo.By(fmt.Sprintf("wait the addon %v/%v available condition to be true", universalClusterName, addOnName)) + gomega.Eventually(func() error { + return hub.CheckManagedClusterAddOnStatusV1Alpha1(universalClusterName, addOnName) + }).Should(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("delete the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) + err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + context.TODO(), addOnName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete managed cluster addon %v on cluster %v: %v", addOnName, universalClusterName, err)) + } + + gomega.Eventually(func() error { + _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err == nil { + return fmt.Errorf("the managedClusterAddon %s should be deleted", addOnName) + } + if err != nil && !errors.IsNotFound(err) { + return err + } + + // check works after addon is not found + works, err := hub.WorkClient.WorkV1().ManifestWorks(universalClusterName).List( + context.TODO(), metav1.ListOptions{}) + if err == nil && len(works.Items) != 0 { + return fmt.Errorf("expected no works,but got: %+v", works.Items) + } + if err != nil && !errors.IsNotFound(err) { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("delete addon template resources for cluster %v", universalClusterName)) + err = deleteResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s, + defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{ + "Namespace": universalClusterName, + "AddonInstallNamespace": addonInstallNamespace, + "CustomSignerName": customSignerName, + "AddonManagerNamespace": templateagent.AddonManagerNamespace(), + "CustomSignerSecretName": customSignerSecretName, + "CustomSignerSecretNamespace": signerSecretNamespace, + }), + templateResources, + ) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("delete addon custom sign secret") + err = hub.KubeClient.CoreV1().Secrets(signerSecretNamespace).Delete(context.TODO(), + customSignerSecretName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret %v/%v: %v", + signerSecretNamespace, customSignerSecretName, err)) + } + + ginkgo.By("delete addon custom sign secret namespace") + err = hub.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), signerSecretNamespace, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret namespace %v: %v", signerSecretNamespace, err)) + } + + // delete all CSR created for the addon on the hub cluster, otherwise if it reaches the limit number 10, the + // other tests will fail + ginkgo.By(fmt.Sprintf("delete CSRs for addon %v on cluster %v", addOnName, universalClusterName)) + gomega.Eventually(func() error { + listOpts := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1alpha1.AddonLabelKey, addOnName, + clusterv1.ClusterNameLabelKey, universalClusterName), + } + + csrs, err := hub.KubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), listOpts) + if err != nil { + return err + } + + if len(csrs.Items) > 0 { + klog.Infof("Found %d CSRs to delete:", len(csrs.Items)) + err := hub.KubeClient.CertificatesV1().CertificateSigningRequests().DeleteCollection(context.TODO(), + metav1.DeleteOptions{}, listOpts) + if err != nil && !errors.IsNotFound(err) { + return err + } + return fmt.Errorf("waiting for %d CSRs to be fully deleted", len(csrs.Items)) + } + + klog.Infof("All CSRs deleted successfully") + return nil + }).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Template type addon should be functioning", func() { + ginkgo.By("Check hub kubeconfig secret is created") + gomega.Eventually(func() error { + _, err := hub.KubeClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(), + templateagent.HubKubeconfigSecretName(addOnName), metav1.GetOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By("Check custom client cert secret is created") + gomega.Eventually(func() error { + _, err := hub.KubeClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(), + templateagent.CustomSignedSecretName(addOnName, customSignerName), metav1.GetOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By("Make sure addon is functioning") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-%s", rand.String(6)), + Namespace: universalClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err := hub.KubeClient.CoreV1().ConfigMaps(universalClusterName).Create( + context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + copiedConfig, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get( + context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !equality.Semantic.DeepEqual(copiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copiedConfig.Data) + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure manifestwork config is configured") + manifestWork, err := hub.WorkClient.WorkV1().ManifestWorks(universalClusterName).Get(context.Background(), + fmt.Sprintf("addon-%s-deploy-0", addOnName), metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + foundDeploymentConfig := false + expectedDeploymentResourceIdentifier := workapiv1.ResourceIdentifier{ + Group: "apps", + Resource: "deployments", + Name: "hello-template-agent", + Namespace: addonInstallNamespace, + } + foundDaemonSetConfig := false + expectedDaemonSetResourceIdentifier := workapiv1.ResourceIdentifier{ + Group: "apps", + Resource: "daemonsets", + Name: "hello-template-agent-ds", + Namespace: addonInstallNamespace, + } + for _, mc := range manifestWork.Spec.ManifestConfigs { + if mc.ResourceIdentifier == expectedDeploymentResourceIdentifier { + foundDeploymentConfig = true + gomega.Expect(mc.UpdateStrategy.Type).To(gomega.Equal(workapiv1.UpdateStrategyTypeServerSideApply)) + } + if mc.ResourceIdentifier == expectedDaemonSetResourceIdentifier { + foundDaemonSetConfig = true + } + } + if !foundDeploymentConfig || !foundDaemonSetConfig { + gomega.Expect(fmt.Errorf("expected manifestwork is not correct, %v", + manifestWork.Spec.ManifestConfigs)).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By(fmt.Sprintf("delete the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) + err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("The pre-delete job should clean up the configmap after the addon is deleted") + gomega.Eventually(func() error { + _, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get( + context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + + return fmt.Errorf("the configmap should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + + return fmt.Errorf("the managedClusterAddon should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("The pre-delete job should be deleted ") + gomega.Eventually(func() error { + _, err := spoke.KubeClient.BatchV1().Jobs(addonInstallNamespace).Get( + context.Background(), "hello-template-cleanup-configmap", metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + + return fmt.Errorf("the job should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Template type addon should be configured by addon deployment config for image override"+ + "even there are cluster annotation config", func() { + ginkgo.By("Prepare cluster annotation for addon image override config") + overrideRegistries := addonapiv1alpha1.AddOnDeploymentConfigSpec{ + // should be different from the registries in the addonDeploymentConfig + Registries: []addonapiv1alpha1.ImageMirror{ + { + Source: "quay.io/open-cluster-management/addon-examples", + Mirror: "quay.io/ocm/addon-examples-test", + }, + }, + } + registriesJson, err := json.Marshal(overrideRegistries) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Eventually(func() error { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get( + context.Background(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + newCluster := cluster.DeepCopy() + + annotations := cluster.Annotations + if annotations == nil { + annotations = make(map[string]string) + } + annotations[clusterv1.ClusterImageRegistriesAnnotationKey] = string(registriesJson) + + newCluster.Annotations = annotations + _, err = hub.ClusterClient.ClusterV1().ManagedClusters().Update( + context.Background(), newCluster, metav1.UpdateOptions{}) + return err + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Prepare a AddOnDeploymentConfig for addon image override config") + gomega.Eventually(func() error { + return prepareImageOverrideAddOnDeploymentConfigAlpha(universalClusterName, addonInstallNamespace) + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.Background(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: universalClusterName, + Name: imageOverrideDeploymentConfigName, + }, + }, + } + _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + containers := agentDeploy.Spec.Template.Spec.Containers + if len(containers) != 1 { + return fmt.Errorf("expect one container, but %v", containers) + } + + if containers[0].Image != overrideImageValue { + return fmt.Errorf("unexpected image %s", containers[0].Image) + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Restore the managed cluster annotation") + gomega.Eventually(func() error { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get( + context.Background(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + newCluster := cluster.DeepCopy() + delete(newCluster.Annotations, clusterv1.ClusterImageRegistriesAnnotationKey) + _, err = hub.ClusterClient.ClusterV1().ManagedClusters().Update( + context.Background(), newCluster, metav1.UpdateOptions{}) + return err + }).ShouldNot(gomega.HaveOccurred()) + + // restore the image override config, because the override image is not available + // but it is needed by the pre-delete job + ginkgo.By("Restore the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.Background(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{} + _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon config is restored") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + containers := agentDeploy.Spec.Template.Spec.Containers + if len(containers) != 1 { + return fmt.Errorf("expect one container, but %v", containers) + } + + if containers[0].Image != originalImageValue { + return fmt.Errorf("unexpected image %s", containers[0].Image) + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Template type addon should be configured by addon deployment config for node placement", func() { + ginkgo.By("Prepare a AddOnDeploymentConfig for addon image override config") + gomega.Eventually(func() error { + return prepareNodePlacementAddOnDeploymentConfigAlpha(universalClusterName, addonInstallNamespace) + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.Background(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: universalClusterName, + Name: nodePlacementDeploymentConfigName, + }, + }, + } + _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + if !equality.Semantic.DeepEqual(agentDeploy.Spec.Template.Spec.NodeSelector, nodeSelector) { + return fmt.Errorf("unexpected nodeSeletcor %v", agentDeploy.Spec.Template.Spec.NodeSelector) + } + + if !equality.Semantic.DeepEqual(agentDeploy.Spec.Template.Spec.Tolerations, tolerations) { + return fmt.Errorf("unexpected tolerations %v", agentDeploy.Spec.Template.Spec.Tolerations) + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + + }) + + ginkgo.It("Template type addon should be configured by addon deployment config for namespace", func() { + ginkgo.By("Prepare a AddOnDeploymentConfig for namespace config") + overrideNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "another-addon-namespace", + }, + } + _, err := spoke.KubeClient.CoreV1().Namespaces().Create(context.TODO(), overrideNamespace, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Eventually(func() error { + return prepareInstallNamespaceAlpha(universalClusterName, overrideNamespace.Name) + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.Background(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: universalClusterName, + Name: namespaceOverrideConfigName, + }, + }, + } + _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + _, err := spoke.KubeClient.AppsV1().Deployments(overrideNamespace.Name).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + return err + }).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Template type addon's image should be overrode by cluster annotation", func() { + ginkgo.By("Prepare cluster annotation for addon image override config") + overrideRegistries := addonapiv1alpha1.AddOnDeploymentConfigSpec{ + Registries: registriesAlpha, + } + registriesJson, err := json.Marshal(overrideRegistries) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Eventually(func() error { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get( + context.Background(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + newCluster := cluster.DeepCopy() + + annotations := cluster.Annotations + if annotations == nil { + annotations = make(map[string]string) + } + annotations[clusterv1.ClusterImageRegistriesAnnotationKey] = string(registriesJson) + + newCluster.Annotations = annotations + _, err = hub.ClusterClient.ClusterV1().ManagedClusters().Update( + context.Background(), newCluster, metav1.UpdateOptions{}) + return err + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + containers := agentDeploy.Spec.Template.Spec.Containers + if len(containers) != 1 { + return fmt.Errorf("expect one container, but %v", containers) + } + + if containers[0].Image != overrideImageValue { + return fmt.Errorf("unexpected image %s", containers[0].Image) + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + + // restore the image override config, because the override image is not available + // but it is needed by the pre-delete job + ginkgo.By("Restore the managed cluster annotation") + gomega.Eventually(func() error { + cluster, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get( + context.Background(), universalClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + newCluster := cluster.DeepCopy() + delete(newCluster.Annotations, clusterv1.ClusterImageRegistriesAnnotationKey) + _, err = hub.ClusterClient.ClusterV1().ManagedClusters().Update( + context.Background(), newCluster, metav1.UpdateOptions{}) + return err + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon config is restored") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + containers := agentDeploy.Spec.Template.Spec.Containers + if len(containers) != 1 { + return fmt.Errorf("expect one container, but %v", containers) + } + + if containers[0].Image != originalImageValue { + return fmt.Errorf("unexpected image %s", containers[0].Image) + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Template type addon should be configured by addon deployment config for proxy", func() { + ginkgo.By("Prepare a AddOnDeploymentConfig for addon proxy config") + gomega.Eventually(func() error { + return prepareProxyAddOnDeploymentConfigAlpha(universalClusterName, addonInstallNamespace) + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.Background(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: universalClusterName, + Name: proxyDeploymentConfigName, + }, + }, + } + _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + for _, container := range agentDeploy.Spec.Template.Spec.Containers { + found := 0 + for _, env := range container.Env { + if env.Name == "HTTP_PROXY" || env.Name == "http_proxy" { + if env.Value != proxyConfigAlpha.HTTPProxy { + return fmt.Errorf("unexpected HTTP_PROXY %s", env.Value) + } + found++ + } + if env.Name == "HTTPS_PROXY" || env.Name == "https_proxy" { + if env.Value != proxyConfigAlpha.HTTPSProxy { + return fmt.Errorf("unexpected HTTPS_PROXY %s", env.Value) + } + found++ + } + if env.Name == "NO_PROXY" || env.Name == "no_proxy" { + if env.Value != proxyConfigAlpha.NoProxy { + return fmt.Errorf("unexpected NO_PROXY %s", env.Value) + } + found++ + } + if env.Name == "CA_BUNDLE_FILE_PATH" { + if env.Value != "/managed/proxy-ca/ca-bundle.crt" { + return fmt.Errorf("unexpected CA_BUNDLE_FILE_PATH %s", env.Value) + } + found++ + } + } + if found != 7 { + return fmt.Errorf("unexpected env %v", container.Env) + } + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + + cm, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get(context.TODO(), + fmt.Sprintf("%s-proxy-ca", addOnName), metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(cm.Data).To(gomega.HaveKey("ca-bundle.crt")) + gomega.Expect(cm.Data["ca-bundle.crt"]).To(gomega.Equal(string(proxyConfigAlpha.CABundle))) + }) + + ginkgo.It("Template type addon should be configured by addon deployment config for resource requirement", func() { + ginkgo.By("Prepare a AddOnDeploymentConfig for addon resource requirement config") + gomega.Eventually(func() error { + return prepareResourceRequirementsAddOnDeploymentConfigAlpha(universalClusterName, addonInstallNamespace) + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.Background(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: universalClusterName, + Name: resourceRequirementsDeploymentConfigName, + }, + }, + } + _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := spoke.KubeClient.AppsV1().Deployments(addonInstallNamespace).Get( + context.Background(), "hello-template-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + for _, container := range agentDeploy.Spec.Template.Spec.Containers { + if container.Name == "helloworld-agent" { + if !equality.Semantic.DeepEqual(container.Resources, resourceRequirementsConfigAlpha[0].Resources) { + return fmt.Errorf("unexpected resource requirements for deployment: %v", container.Resources) + } + } + } + + agentDaemonset, err := spoke.KubeClient.AppsV1().DaemonSets(addonInstallNamespace).Get( + context.Background(), "hello-template-agent-ds", metav1.GetOptions{}) + if err != nil { + return err + } + for _, container := range agentDaemonset.Spec.Template.Spec.Containers { + if container.Name == "helloworld-agent" { + if !equality.Semantic.DeepEqual(container.Resources, resourceRequirementsConfigAlpha[0].Resources) { + return fmt.Errorf("unexpected resource requirements for daemonset: %v", container.Resources) + } + } + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("ClusterManagementAddon deletion should wait for ManagedClusterAddons cleanup", func() { + ginkgo.By("Make sure addon is functioning before deletion") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-%s", rand.String(6)), + Namespace: universalClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err := hub.KubeClient.CoreV1().ConfigMaps(universalClusterName).Create( + context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + copiedConfig, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get( + context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !equality.Semantic.DeepEqual(copiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copiedConfig.Data) + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Delete the ClusterManagementAddon to trigger cascading deletion") + err = hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Delete( + context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("The pre-delete job should clean up the configmap") + gomega.Eventually(func() error { + _, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get( + context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("the configmap should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("ManagedClusterAddon should eventually be deleted after pre-delete job completes") + gomega.Eventually(func() error { + _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("the ManagedClusterAddon should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("The pre-delete job should be cleaned up") + gomega.Eventually(func() error { + _, err := spoke.KubeClient.BatchV1().Jobs(addonInstallNamespace).Get( + context.Background(), "hello-template-cleanup-configmap", metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("the job should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Verify ClusterManagementAddon is deleted") + gomega.Eventually(func() error { + _, err := hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("the ClusterManagementAddon should be deleted") + }).ShouldNot(gomega.HaveOccurred()) + }) +}) + +func prepareInstallNamespaceAlpha(namespace, installNamespace string) error { + _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), namespaceOverrideConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceOverrideConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + AgentInstallNamespace: installNamespace, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareImageOverrideAddOnDeploymentConfigAlpha(namespace, installNamespace string) error { + _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), imageOverrideDeploymentConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageOverrideDeploymentConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + Registries: registriesAlpha, + AgentInstallNamespace: installNamespace, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareNodePlacementAddOnDeploymentConfigAlpha(namespace, installNamespace string) error { + _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), nodePlacementDeploymentConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePlacementDeploymentConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonapiv1alpha1.NodePlacement{ + NodeSelector: nodeSelector, + Tolerations: tolerations, + }, + AgentInstallNamespace: installNamespace, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareProxyAddOnDeploymentConfigAlpha(namespace, installNamespace string) error { + _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), proxyDeploymentConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: proxyDeploymentConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonapiv1alpha1.NodePlacement{ + NodeSelector: nodeSelector, + Tolerations: tolerations, + }, + AgentInstallNamespace: installNamespace, + ProxyConfig: proxyConfigAlpha, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareResourceRequirementsAddOnDeploymentConfigAlpha(namespace, installNamespace string) error { + _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), resourceRequirementsDeploymentConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceRequirementsDeploymentConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + AgentInstallNamespace: installNamespace, + ProxyConfig: proxyConfigAlpha, + ResourceRequirements: resourceRequirementsConfigAlpha, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} diff --git a/test/e2e/addonmanagement_test.go b/test/e2e/addonmanagement_test.go index 05bef162d..fa43d6c80 100644 --- a/test/e2e/addonmanagement_test.go +++ b/test/e2e/addonmanagement_test.go @@ -25,7 +25,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" - addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1" clusterv1 "open-cluster-management.io/api/cluster/v1" workapiv1 "open-cluster-management.io/api/work/v1" @@ -49,19 +49,19 @@ const ( var ( nodeSelector = map[string]string{"kubernetes.io/os": "linux"} tolerations = []corev1.Toleration{{Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}} - registries = []addonapiv1alpha1.ImageMirror{ + registries = []addonapiv1beta1.ImageMirror{ { Source: "quay.io/open-cluster-management/addon-examples", Mirror: "quay.io/ocm/addon-examples", }, } - proxyConfig = addonapiv1alpha1.ProxyConfig{ + proxyConfig = addonapiv1beta1.ProxyConfig{ HTTPProxy: "http://proxy.example.com", HTTPSProxy: "http://proxy.example.com", NoProxy: "localhost", CABundle: []byte("test-ca-bundle"), } - resourceRequirementsConfig = []addonapiv1alpha1.ContainerResourceRequirements{ + resourceRequirementsConfig = []addonapiv1beta1.ContainerResourceRequirements{ { ContainerID: "*:*:helloworld-agent", Resources: corev1.ResourceRequirements{ @@ -73,14 +73,14 @@ var ( } ) -var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() { +var _ = ginkgo.Describe("Addon management (v1beta1)", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() { addOnName := "hello-template" addonInstallNamespace := "test-addon-template" s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = clusterv1.Install(s) - _ = addonapiv1alpha1.Install(s) + _ = addonapiv1beta1.Install(s) templateResources := []string{ "addon/addon_template.yaml", @@ -130,7 +130,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) - err = hub.CreateManagedClusterAddOn(universalClusterName, addOnName, addonInstallNamespace) + err = hub.CreateManagedClusterAddOnV1Beta1(universalClusterName, addOnName, addonInstallNamespace) if err != nil { klog.Errorf("failed to create managed cluster addon %v on the managed cluster namespace %v: %v", addOnName, universalClusterName, err) gomega.Expect(errors.IsAlreadyExists(err)).To(gomega.BeTrue()) @@ -138,20 +138,20 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By(fmt.Sprintf("wait the addon %v/%v available condition to be true", universalClusterName, addOnName)) gomega.Eventually(func() error { - return hub.CheckManagedClusterAddOnStatus(universalClusterName, addOnName) + return hub.CheckManagedClusterAddOnStatusV1Beta1(universalClusterName, addOnName) }).Should(gomega.Succeed()) }) ginkgo.AfterEach(func() { ginkgo.By(fmt.Sprintf("delete the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) - err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Delete( context.TODO(), addOnName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.Fail(fmt.Sprintf("failed to delete managed cluster addon %v on cluster %v: %v", addOnName, universalClusterName, err)) } gomega.Eventually(func() error { - _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + _, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err == nil { return fmt.Errorf("the managedClusterAddon %s should be deleted", addOnName) @@ -205,7 +205,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By(fmt.Sprintf("delete CSRs for addon %v on cluster %v", addOnName, universalClusterName)) gomega.Eventually(func() error { listOpts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1alpha1.AddonLabelKey, addOnName, + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1beta1.AddonLabelKey, addOnName, clusterv1.ClusterNameLabelKey, universalClusterName), } @@ -306,7 +306,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- } ginkgo.By(fmt.Sprintf("delete the addon %v on the managed cluster namespace %v", addOnName, universalClusterName)) - err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Delete( context.TODO(), addOnName, metav1.DeleteOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -325,7 +325,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- }).ShouldNot(gomega.HaveOccurred()) gomega.Eventually(func() error { - _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + _, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -355,9 +355,9 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.It("Template type addon should be configured by addon deployment config for image override"+ "even there are cluster annotation config", func() { ginkgo.By("Prepare cluster annotation for addon image override config") - overrideRegistries := addonapiv1alpha1.AddOnDeploymentConfigSpec{ + overrideRegistries := addonapiv1beta1.AddOnDeploymentConfigSpec{ // should be different from the registries in the addonDeploymentConfig - Registries: []addonapiv1alpha1.ImageMirror{ + Registries: []addonapiv1beta1.ImageMirror{ { Source: "quay.io/open-cluster-management/addon-examples", Mirror: "quay.io/ocm/addon-examples-test", @@ -394,25 +394,25 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("Add the configs to ManagedClusterAddOn") gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.Background(), addOnName, metav1.GetOptions{}) if err != nil { return err } newAddon := addon.DeepCopy() - newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + newAddon.Spec.Configs = []addonapiv1beta1.AddOnConfig{ { - ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addondeploymentconfigs", }, - ConfigReferent: addonapiv1alpha1.ConfigReferent{ + ConfigReferent: addonapiv1beta1.ConfigReferent{ Namespace: universalClusterName, Name: imageOverrideDeploymentConfigName, }, }, } - _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + _, err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Update( context.Background(), newAddon, metav1.UpdateOptions{}) if err != nil { return err @@ -459,14 +459,14 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- // but it is needed by the pre-delete job ginkgo.By("Restore the configs to ManagedClusterAddOn") gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.Background(), addOnName, metav1.GetOptions{}) if err != nil { return err } newAddon := addon.DeepCopy() - newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{} - _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + newAddon.Spec.Configs = []addonapiv1beta1.AddOnConfig{} + _, err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Update( context.Background(), newAddon, metav1.UpdateOptions{}) if err != nil { return err @@ -503,25 +503,25 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("Add the configs to ManagedClusterAddOn") gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.Background(), addOnName, metav1.GetOptions{}) if err != nil { return err } newAddon := addon.DeepCopy() - newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + newAddon.Spec.Configs = []addonapiv1beta1.AddOnConfig{ { - ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addondeploymentconfigs", }, - ConfigReferent: addonapiv1alpha1.ConfigReferent{ + ConfigReferent: addonapiv1beta1.ConfigReferent{ Namespace: universalClusterName, Name: nodePlacementDeploymentConfigName, }, }, } - _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + _, err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Update( context.Background(), newAddon, metav1.UpdateOptions{}) if err != nil { return err @@ -565,25 +565,25 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("Add the configs to ManagedClusterAddOn") gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.Background(), addOnName, metav1.GetOptions{}) if err != nil { return err } newAddon := addon.DeepCopy() - newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + newAddon.Spec.Configs = []addonapiv1beta1.AddOnConfig{ { - ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addondeploymentconfigs", }, - ConfigReferent: addonapiv1alpha1.ConfigReferent{ + ConfigReferent: addonapiv1beta1.ConfigReferent{ Namespace: universalClusterName, Name: namespaceOverrideConfigName, }, }, } - _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + _, err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Update( context.Background(), newAddon, metav1.UpdateOptions{}) if err != nil { return err @@ -601,7 +601,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.It("Template type addon's image should be overrode by cluster annotation", func() { ginkgo.By("Prepare cluster annotation for addon image override config") - overrideRegistries := addonapiv1alpha1.AddOnDeploymentConfigSpec{ + overrideRegistries := addonapiv1beta1.AddOnDeploymentConfigSpec{ Registries: registries, } registriesJson, err := json.Marshal(overrideRegistries) @@ -693,25 +693,25 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("Add the configs to ManagedClusterAddOn") gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.Background(), addOnName, metav1.GetOptions{}) if err != nil { return err } newAddon := addon.DeepCopy() - newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + newAddon.Spec.Configs = []addonapiv1beta1.AddOnConfig{ { - ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addondeploymentconfigs", }, - ConfigReferent: addonapiv1alpha1.ConfigReferent{ + ConfigReferent: addonapiv1beta1.ConfigReferent{ Namespace: universalClusterName, Name: proxyDeploymentConfigName, }, }, } - _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + _, err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Update( context.Background(), newAddon, metav1.UpdateOptions{}) if err != nil { return err @@ -778,25 +778,25 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("Add the configs to ManagedClusterAddOn") gomega.Eventually(func() error { - addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + addon, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.Background(), addOnName, metav1.GetOptions{}) if err != nil { return err } newAddon := addon.DeepCopy() - newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + newAddon.Spec.Configs = []addonapiv1beta1.AddOnConfig{ { - ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addondeploymentconfigs", }, - ConfigReferent: addonapiv1alpha1.ConfigReferent{ + ConfigReferent: addonapiv1beta1.ConfigReferent{ Namespace: universalClusterName, Name: resourceRequirementsDeploymentConfigName, }, }, } - _, err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Update( + _, err = hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Update( context.Background(), newAddon, metav1.UpdateOptions{}) if err != nil { return err @@ -867,7 +867,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- }).ShouldNot(gomega.HaveOccurred()) ginkgo.By("Delete the ClusterManagementAddon to trigger cascading deletion") - err = hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Delete( + err = hub.AddonClient.AddonV1beta1().ClusterManagementAddOns().Delete( context.TODO(), addOnName, metav1.DeleteOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -886,7 +886,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("ManagedClusterAddon should eventually be deleted after pre-delete job completes") gomega.Eventually(func() error { - _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + _, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(universalClusterName).Get( context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -912,7 +912,7 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- ginkgo.By("Verify ClusterManagementAddon is deleted") gomega.Eventually(func() error { - _, err := hub.AddonClient.AddonV1alpha1().ClusterManagementAddOns().Get( + _, err := hub.AddonClient.AddonV1beta1().ClusterManagementAddOns().Get( context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -926,17 +926,17 @@ var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon- }) func prepareInstallNamespace(namespace, installNamespace string) error { - _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Get( context.Background(), namespaceOverrideConfigName, metav1.GetOptions{}) if errors.IsNotFound(err) { - if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + if _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Create( context.Background(), - &addonapiv1alpha1.AddOnDeploymentConfig{ + &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: namespaceOverrideConfigName, Namespace: namespace, }, - Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: installNamespace, }, }, @@ -952,17 +952,17 @@ func prepareInstallNamespace(namespace, installNamespace string) error { } func prepareImageOverrideAddOnDeploymentConfig(namespace, installNamespace string) error { - _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Get( context.Background(), imageOverrideDeploymentConfigName, metav1.GetOptions{}) if errors.IsNotFound(err) { - if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + if _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Create( context.Background(), - &addonapiv1alpha1.AddOnDeploymentConfig{ + &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: imageOverrideDeploymentConfigName, Namespace: namespace, }, - Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ Registries: registries, AgentInstallNamespace: installNamespace, }, @@ -979,18 +979,18 @@ func prepareImageOverrideAddOnDeploymentConfig(namespace, installNamespace strin } func prepareNodePlacementAddOnDeploymentConfig(namespace, installNamespace string) error { - _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Get( context.Background(), nodePlacementDeploymentConfigName, metav1.GetOptions{}) if errors.IsNotFound(err) { - if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + if _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Create( context.Background(), - &addonapiv1alpha1.AddOnDeploymentConfig{ + &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: nodePlacementDeploymentConfigName, Namespace: namespace, }, - Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ - NodePlacement: &addonapiv1alpha1.NodePlacement{ + Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonapiv1beta1.NodePlacement{ NodeSelector: nodeSelector, Tolerations: tolerations, }, @@ -1009,18 +1009,18 @@ func prepareNodePlacementAddOnDeploymentConfig(namespace, installNamespace strin } func prepareProxyAddOnDeploymentConfig(namespace, installNamespace string) error { - _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Get( context.Background(), proxyDeploymentConfigName, metav1.GetOptions{}) if errors.IsNotFound(err) { - if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + if _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Create( context.Background(), - &addonapiv1alpha1.AddOnDeploymentConfig{ + &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: proxyDeploymentConfigName, Namespace: namespace, }, - Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ - NodePlacement: &addonapiv1alpha1.NodePlacement{ + Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonapiv1beta1.NodePlacement{ NodeSelector: nodeSelector, Tolerations: tolerations, }, @@ -1040,17 +1040,17 @@ func prepareProxyAddOnDeploymentConfig(namespace, installNamespace string) error } func prepareResourceRequirementsAddOnDeploymentConfig(namespace, installNamespace string) error { - _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Get( context.Background(), resourceRequirementsDeploymentConfigName, metav1.GetOptions{}) if errors.IsNotFound(err) { - if _, err := hub.AddonClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Create( + if _, err := hub.AddonClient.AddonV1beta1().AddOnDeploymentConfigs(namespace).Create( context.Background(), - &addonapiv1alpha1.AddOnDeploymentConfig{ + &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: resourceRequirementsDeploymentConfigName, Namespace: namespace, }, - Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: installNamespace, ProxyConfig: proxyConfig, ResourceRequirements: resourceRequirementsConfig, diff --git a/test/framework/managedclusteraddon.go b/test/framework/managedclusteraddon.go index 2381c6654..c448d97aa 100644 --- a/test/framework/managedclusteraddon.go +++ b/test/framework/managedclusteraddon.go @@ -15,7 +15,8 @@ import ( addonv1beta1 "open-cluster-management.io/api/addon/v1beta1" ) -func (hub *Hub) CreateManagedClusterAddOn(managedClusterNamespace, addOnName, installNamespace string) error { +// CreateManagedClusterAddOnV1Alpha1 creates a ManagedClusterAddOn using v1alpha1 API +func (hub *Hub) CreateManagedClusterAddOnV1Alpha1(managedClusterNamespace, addOnName, installNamespace string) error { _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterNamespace).Create( context.TODO(), &addonv1alpha1.ManagedClusterAddOn{ @@ -78,7 +79,8 @@ func (hub *Hub) CreateManagedClusterAddOnLease(addOnInstallNamespace, addOnName return err } -func (hub *Hub) CheckManagedClusterAddOnStatus(managedClusterNamespace, addOnName string) error { +// CheckManagedClusterAddOnStatusV1Alpha1 checks the status of a ManagedClusterAddOn using v1alpha1 API +func (hub *Hub) CheckManagedClusterAddOnStatusV1Alpha1(managedClusterNamespace, addOnName string) error { addOn, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterNamespace).Get(context.TODO(), addOnName, metav1.GetOptions{}) if err != nil { return err @@ -96,6 +98,25 @@ func (hub *Hub) CheckManagedClusterAddOnStatus(managedClusterNamespace, addOnNam return nil } +// CheckManagedClusterAddOnStatusV1Beta1 checks the status of a ManagedClusterAddOn using v1beta1 API +func (hub *Hub) CheckManagedClusterAddOnStatusV1Beta1(managedClusterNamespace, addOnName string) error { + addOn, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(managedClusterNamespace).Get(context.TODO(), addOnName, metav1.GetOptions{}) + if err != nil { + return err + } + + if addOn.Status.Conditions == nil { + return fmt.Errorf("there is no conditions in addon %v/%v", managedClusterNamespace, addOnName) + } + + if !meta.IsStatusConditionTrue(addOn.Status.Conditions, "Available") { + return fmt.Errorf("the addon %v/%v available condition is not true, %v", + managedClusterNamespace, addOnName, addOn.Status.Conditions) + } + + return nil +} + // CreateManagedClusterAddOnV1Beta1 creates a ManagedClusterAddOn using v1beta1 API func (hub *Hub) CreateManagedClusterAddOnV1Beta1(managedClusterNamespace, addOnName, installNamespace string) error { _, err := hub.AddonClient.AddonV1beta1().ManagedClusterAddOns(managedClusterNamespace).Create(