diff --git a/test/e2e/addon_token_auth_test.go b/test/e2e/addon_token_auth_test.go new file mode 100644 index 000000000..40607f64e --- /dev/null +++ b/test/e2e/addon_token_auth_test.go @@ -0,0 +1,359 @@ +package e2e + +import ( + "context" + "fmt" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + operatorapiv1 "open-cluster-management.io/api/operator/v1" + + "open-cluster-management.io/ocm/pkg/addon/templateagent" + "open-cluster-management.io/ocm/test/e2e/manifests" +) + +var _ = ginkgo.Describe("Template addon with token-based authentication", ginkgo.Ordered, ginkgo.Label("addon-manager", "addon-token-auth"), func() { + addOnName := "hello-template" + addonInstallNamespace := "test-addon-template-token" + var signerSecretNamespace string + var originalAddOnDriver *operatorapiv1.AddOnRegistrationDriver + + var agentClient kubernetes.Interface + var agentNamespace string + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = addonapiv1alpha1.Install(s) + _ = clusterv1.Install(s) + + templateResources := []string{ + "addon/addon_template.yaml", + "addon/cluster_management_addon.yaml", + "addon/cluster_role.yaml", + "addon/signca_secret_role.yaml", + "addon/signca_secret_rolebinding.yaml", + } + + ginkgo.BeforeAll(func() { + ginkgo.By("Save original klusterlet configuration") + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if klusterlet.Spec.RegistrationConfiguration != nil { + originalAddOnDriver = klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver + } + + ginkgo.By("Get initial registration agent deployment generation before updating klusterlet") + var initialGeneration int64 + var registrationDeploymentName string + registrationDeploymentName = fmt.Sprintf("%s-registration-agent", klusterlet.Name) + if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeSingleton || + klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeSingletonHosted { + registrationDeploymentName = fmt.Sprintf("%s-agent", klusterlet.Name) + } + + // In hosted mode, agents run on the hub cluster, otherwise on the spoke cluster + agentClient = spoke.KubeClient + agentNamespace = universalAgentNamespace + if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted || + klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeSingletonHosted { + agentClient = hub.KubeClient + agentNamespace = klusterlet.Name + } + + deployment, err := agentClient.AppsV1().Deployments(agentNamespace).Get( + context.TODO(), registrationDeploymentName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + initialGeneration = deployment.Generation + + ginkgo.By("Update klusterlet to use token-based authentication for addons") + gomega.Eventually(func() error { + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + if klusterlet.Spec.RegistrationConfiguration == nil { + klusterlet.Spec.RegistrationConfiguration = &operatorapiv1.RegistrationConfiguration{} + } + + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver = &operatorapiv1.AddOnRegistrationDriver{ + AuthType: "token", + Token: &operatorapiv1.TokenConfig{ + ExpirationSeconds: 3600, // 1 hour for testing + }, + } + + _, err = spoke.OperatorClient.OperatorV1().Klusterlets().Update( + context.TODO(), klusterlet, metav1.UpdateOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By("Verify klusterlet is updated with token auth configuration") + gomega.Eventually(func() error { + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + if klusterlet.Spec.RegistrationConfiguration == nil || + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver == nil { + return fmt.Errorf("token auth configuration not set") + } + + if klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver.AuthType != "token" { + return fmt.Errorf("auth type is not token: %s", + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver.AuthType) + } + + return nil + }).Should(gomega.Succeed()) + + ginkgo.By("Wait for registration agent deployment to rollout with new token auth configuration") + gomega.Eventually(func() error { + deployment, err := agentClient.AppsV1().Deployments(agentNamespace).Get( + context.TODO(), registrationDeploymentName, metav1.GetOptions{}) + if err != nil { + return err + } + + // Wait for deployment generation to increment (indicates config change was applied) + if deployment.Generation <= initialGeneration { + return fmt.Errorf("deployment generation has not incremented yet: current=%d, initial=%d", + deployment.Generation, initialGeneration) + } + + // Ensure the deployment controller has observed the latest spec + if deployment.Status.ObservedGeneration != deployment.Generation { + return fmt.Errorf("deployment has not observed latest generation: observed=%d, current=%d", + deployment.Status.ObservedGeneration, deployment.Generation) + } + + // Ensure all replicas have been updated with the new configuration + if deployment.Status.UpdatedReplicas != deployment.Status.Replicas { + return fmt.Errorf("deployment has not updated all replicas: updated=%d, total=%d", + deployment.Status.UpdatedReplicas, deployment.Status.Replicas) + } + + // Ensure all updated replicas are ready + if deployment.Status.ReadyReplicas != deployment.Status.Replicas { + return fmt.Errorf("deployment not fully ready: ready=%d, total=%d", + deployment.Status.ReadyReplicas, deployment.Status.Replicas) + } + + // Ensure there are no unavailable replicas + if deployment.Status.UnavailableReplicas > 0 { + return fmt.Errorf("deployment has unavailable replicas: %d", deployment.Status.UnavailableReplicas) + } + + return nil + }, "2m", "5s").Should(gomega.Succeed()) + + signerSecretNamespace = "signer-secret-token-ns-" + rand.String(6) + ginkgo.By("Create addon custom sign secret namespace") + _, err = hub.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: signerSecretNamespace, + }, + }, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + ginkgo.By("Create addon custom sign secret") + err = copySignerSecret(context.TODO(), hub.KubeClient, "open-cluster-management-hub", + "signer-secret", signerSecretNamespace, customSignerSecretName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + }) + + ginkgo.AfterAll(func() { + ginkgo.By("Delete addon custom sign secret") + err := hub.KubeClient.CoreV1().Secrets(signerSecretNamespace).Delete(context.TODO(), + customSignerSecretName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret %v/%v: %v", + signerSecretNamespace, customSignerSecretName, err)) + } + + ginkgo.By("Delete addon custom sign secret namespace") + err = hub.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), signerSecretNamespace, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret namespace %v: %v", signerSecretNamespace, err)) + } + + ginkgo.By("Restore original klusterlet AddOnKubeClientRegistrationDriver configuration") + gomega.Eventually(func() error { + klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), universalKlusterletName, metav1.GetOptions{}) + if err != nil { + return err + } + + if klusterlet.Spec.RegistrationConfiguration == nil { + klusterlet.Spec.RegistrationConfiguration = &operatorapiv1.RegistrationConfiguration{} + } + + klusterlet.Spec.RegistrationConfiguration.AddOnKubeClientRegistrationDriver = originalAddOnDriver + + _, err = spoke.OperatorClient.OperatorV1().Klusterlets().Update( + context.TODO(), klusterlet, metav1.UpdateOptions{}) + return err + }).Should(gomega.Succeed()) + + }) + + ginkgo.It("Should work with token-based authentication flow", func() { + var err error + + ginkgo.By("Step 1: Create addon template resources") + err = createResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s, + defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{ + "Namespace": universalClusterName, + "AddonInstallNamespace": addonInstallNamespace, + "CustomSignerName": customSignerName, + "AddonManagerNamespace": templateagent.AddonManagerNamespace(), + "CustomSignerSecretName": customSignerSecretName, + "CustomSignerSecretNamespace": signerSecretNamespace, + }), + templateResources, + ) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Step 2: Create the template addon") + err = hub.CreateManagedClusterAddOn(universalClusterName, addOnName, addonInstallNamespace) + if err != nil { + gomega.Expect(errors.IsAlreadyExists(err)).To(gomega.BeTrue()) + } + + ginkgo.By("Step 3: Wait for addon to become available with token authentication") + gomega.Eventually(func() error { + return hub.CheckManagedClusterAddOnStatus(universalClusterName, addOnName) + }, "5m", "10s").Should(gomega.Succeed()) + + ginkgo.By("Step 4: Verify hub kubeconfig secret is created with token authentication") + gomega.Eventually(func() error { + secret, err := agentClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(), + templateagent.HubKubeconfigSecretName(addOnName), metav1.GetOptions{}) + if err != nil { + return err + } + + // Verify the secret contains token-based kubeconfig + token, ok := secret.Data["token"] + if !ok { + return fmt.Errorf("token not found in secret") + } + + // Token should not be empty + if len(token) == 0 { + return fmt.Errorf("token is empty") + } + + return nil + }).Should(gomega.Succeed()) + + ginkgo.By("Step 5: Verify custom client cert secret is created") + gomega.Eventually(func() error { + _, err := agentClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(), + templateagent.CustomSignedSecretName(addOnName, customSignerName), metav1.GetOptions{}) + return err + }).Should(gomega.Succeed()) + + ginkgo.By("Step 6: Test addon functionality - create configmap on hub") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-token-%s", rand.String(6)), + Namespace: universalClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err = hub.KubeClient.CoreV1().ConfigMaps(universalClusterName).Create( + context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Step 7: Verify addon copies configmap to spoke using token auth") + gomega.Eventually(func() error { + copiedConfig, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get( + context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !equality.Semantic.DeepEqual(copiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copiedConfig.Data) + } + return nil + }, "2m", "5s").ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Step 8: Cleanup - Delete the addon") + err = hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Delete( + context.TODO(), addOnName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + _, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get( + context.TODO(), addOnName, metav1.GetOptions{}) + if err == nil { + return fmt.Errorf("the managedClusterAddon %s should be deleted", addOnName) + } + if err != nil && !errors.IsNotFound(err) { + return err + } + return nil + }).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Step 9: Delete addon template resources") + err = deleteResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s, + defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{ + "Namespace": universalClusterName, + "AddonInstallNamespace": addonInstallNamespace, + "CustomSignerName": customSignerName, + "AddonManagerNamespace": templateagent.AddonManagerNamespace(), + "CustomSignerSecretName": customSignerSecretName, + "CustomSignerSecretNamespace": signerSecretNamespace, + }), + templateResources, + ) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Step 10: Cleanup CSRs") + gomega.Eventually(func() error { + csrs, err := hub.KubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), + metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", addonapiv1alpha1.AddonLabelKey, addOnName, + clusterv1.ClusterNameLabelKey, universalClusterName), + }) + if err != nil { + return err + } + + for _, csr := range csrs.Items { + err = hub.KubeClient.CertificatesV1().CertificateSigningRequests().Delete(context.TODO(), + csr.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + } + + return nil + }).ShouldNot(gomega.HaveOccurred()) + }) +}) diff --git a/test/e2e/addonmanagement_test.go b/test/e2e/addonmanagement_test.go index 03a507c82..cab7975df 100644 --- a/test/e2e/addonmanagement_test.go +++ b/test/e2e/addonmanagement_test.go @@ -73,7 +73,7 @@ var ( } ) -var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() { +var _ = ginkgo.Describe("Addon management", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() { addOnName := "hello-template" addonInstallNamespace := "test-addon-template" @@ -93,7 +93,7 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered, var signerSecretNamespace string ginkgo.BeforeEach(func() { - signerSecretNamespace = "signer-secret-test-ns" + rand.String(6) + signerSecretNamespace = "signer-secret-test-ns-" + rand.String(6) ginkgo.By("create addon custom sign secret namespace") _, err := hub.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{