package templateagent import ( "fmt" "os" "testing" "time" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" kubeinformers "k8s.io/client-go/informers" fakekube "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2/ktesting" "open-cluster-management.io/addon-framework/pkg/addonfactory" "open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting" "open-cluster-management.io/addon-framework/pkg/agent" "open-cluster-management.io/addon-framework/pkg/utils" addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1" fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake" addoninformers "open-cluster-management.io/api/client/addon/informers/externalversions" clusterv1 "open-cluster-management.io/api/cluster/v1" clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1" workapiv1 "open-cluster-management.io/api/work/v1" testingcommon "open-cluster-management.io/ocm/pkg/common/testing" ) func TestAddonTemplateAgentManifests(t *testing.T) { _, ctx := ktesting.NewTestContext(t) addonName := "hello-template" clusterName := "cluster1" s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = clusterv1apha1.Install(s) _ = addonapiv1alpha1.Install(s) _ = addonapiv1beta1.Install(s) decoder := serializer.NewCodecFactory(s).UniversalDeserializer() validatePodTemplate := func(t *testing.T, podTemplate corev1.PodTemplateSpec, expectedEnvs []corev1.EnvVar, caBundleConfigured bool) { image := podTemplate.Spec.Containers[0].Image if image != "quay.io/ocm/addon-examples:v1" { t.Errorf("unexpected image %v", image) } nodeSelector := podTemplate.Spec.NodeSelector expectedNodeSelector := map[string]string{"host": "ssd"} if !equality.Semantic.DeepEqual(nodeSelector, expectedNodeSelector) { t.Errorf("unexpected nodeSelector %v", nodeSelector) } tolerations := podTemplate.Spec.Tolerations expectedTolerations := []corev1.Toleration{ { Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, }, } if !equality.Semantic.DeepEqual(tolerations, expectedTolerations) { t.Errorf("unexpected tolerations %v", tolerations) } envs := podTemplate.Spec.Containers[0].Env if !equality.Semantic.DeepEqual(envs, expectedEnvs) { t.Errorf("unexpected envs %v", envs) } volumes := podTemplate.Spec.Volumes expectedVolumes := []corev1.Volume{ { Name: "hub-kubeconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "hello-template-hub-kubeconfig", }, }, }, { Name: "cert-example-com-signer-name", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "hello-template-example.com-signer-name-client-cert", }, }, }, } if caBundleConfigured { expectedVolumes = append(expectedVolumes, corev1.Volume{ Name: "proxy-ca-bundle", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: "hello-template-proxy-ca", }, }, }, }) } if !equality.Semantic.DeepEqual(volumes, expectedVolumes) { t.Errorf("expected volumes %+v, but got: %+v", expectedVolumes, volumes) } volumeMounts := podTemplate.Spec.Containers[0].VolumeMounts expectedVolumeMounts := []corev1.VolumeMount{ { Name: "hub-kubeconfig", MountPath: "/managed/hub-kubeconfig", }, { Name: "cert-example-com-signer-name", MountPath: "/managed/example.com-signer-name", }, } if caBundleConfigured { expectedVolumeMounts = append(expectedVolumeMounts, corev1.VolumeMount{ Name: "proxy-ca-bundle", MountPath: "/managed/proxy-ca", }) } if !equality.Semantic.DeepEqual(volumeMounts, expectedVolumeMounts) { t.Errorf("expected volumeMounts %v, but got: %v", expectedVolumeMounts, volumeMounts) } } validateAgentOptions := func(t *testing.T, addonOptions agent.AgentAddonOptions, managedClusterAddon *addonapiv1beta1.ManagedClusterAddOn, expectedNamespace string) { if addonOptions.Registration == nil { t.Fatal("registration is nil") } if addonOptions.AgentInstallNamespace == nil { t.Fatal("agentInstallNamespace func is nil") } registrationNamespace, err := addonOptions.AgentInstallNamespace(ctx, managedClusterAddon) if err != nil { t.Fatalf("execute agent install namespace func error: %v", err) } if registrationNamespace != expectedNamespace { t.Fatalf("agentInstallNamespace func does not return expected value, expect %s but got %s", expectedNamespace, registrationNamespace) } } cases := []struct { name string addonTemplatePath string addonDeploymentConfig *addonapiv1beta1.AddOnDeploymentConfig managedCluster *clusterv1.ManagedCluster expectedErr string validateObjects func(t *testing.T, objects []runtime.Object) validateAgentOptions func(t *testing.T, o agent.AgentAddonOptions, mca *addonapiv1beta1.ManagedClusterAddOn) }{ { name: "no addon template", addonTemplatePath: "", addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ CustomizedVariables: []addonapiv1beta1.CustomizedVariable{ { Name: "LOG_LEVEL", Value: "4", }, }, }, }, managedCluster: addonfactory.NewFakeManagedCluster(clusterName, "1.10.1"), expectedErr: fmt.Sprintf("addon %s/%s template not found in status", clusterName, addonName), }, { name: "manifests rendered successfully", addonTemplatePath: "./testmanifests/addontemplate.yaml", addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: "test-install-namespace", CustomizedVariables: []addonapiv1beta1.CustomizedVariable{ { Name: "LOG_LEVEL", Value: "4", }, }, NodePlacement: &addonapiv1beta1.NodePlacement{ NodeSelector: map[string]string{ "host": "ssd", }, Tolerations: []corev1.Toleration{ { Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, }, }, }, Registries: []addonapiv1beta1.ImageMirror{ { Source: "quay.io/open-cluster-management", Mirror: "quay.io/ocm", }, }, }, }, managedCluster: addonfactory.NewFakeManagedCluster(clusterName, "1.10.1"), validateObjects: func(t *testing.T, objects []runtime.Object) { if len(objects) != 5 { t.Fatalf("expected 5 objects, but got %v", len(objects)) } unstructDeployment, ok := objects[0].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be *appsv1.Deployment, but got %T", objects[0]) } deployment, err := utils.ConvertToDeployment(unstructDeployment) if err != nil { t.Fatal(err) } if deployment.Namespace != "test-install-namespace" { t.Errorf("unexpected namespace %s", deployment.Namespace) } validatePodTemplate(t, deployment.Spec.Template, []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "4"}, {Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"}, {Name: "CLUSTER_NAME", Value: clusterName}, {Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"}, }, false) unstructDaemonSet, ok := objects[1].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be *appsv1.DaemonSet, but got %T", objects[1]) } daemonSet, err := utils.ConvertToDaemonSet(unstructDaemonSet) if err != nil { t.Fatal(err) } if daemonSet.Namespace != "test-install-namespace" { t.Errorf("unexpected namespace %s", daemonSet.Namespace) } validatePodTemplate(t, daemonSet.Spec.Template, []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "4"}, {Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"}, {Name: "CLUSTER_NAME", Value: clusterName}, {Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"}, }, false) // check clusterrole unstructCRB, ok := objects[3].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be unstructured, but got %T", objects[3]) } clusterRoleBinding := &rbacv1.ClusterRoleBinding{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructCRB.Object, clusterRoleBinding) if err != nil { t.Fatal(err) } if clusterRoleBinding.Subjects[0].Namespace != "test-install-namespace" { t.Errorf("clusterRolebinding namespace does not match, got %s", clusterRoleBinding.Subjects[0].Namespace) } }, validateAgentOptions: func(t *testing.T, o agent.AgentAddonOptions, mca *addonapiv1beta1.ManagedClusterAddOn) { validateAgentOptions(t, o, mca, "test-install-namespace") }, }, { name: "manifests with proxy config rendered successfully", addonTemplatePath: "./testmanifests/addontemplate.yaml", addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: "test-install-namespace", CustomizedVariables: []addonapiv1beta1.CustomizedVariable{ { Name: "LOG_LEVEL", Value: "4", }, }, NodePlacement: &addonapiv1beta1.NodePlacement{ NodeSelector: map[string]string{ "host": "ssd", }, Tolerations: []corev1.Toleration{ { Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, }, }, }, Registries: []addonapiv1beta1.ImageMirror{ { Source: "quay.io/open-cluster-management", Mirror: "quay.io/ocm", }, }, ProxyConfig: addonapiv1beta1.ProxyConfig{ HTTPProxy: "http://proxy.example.com:8080", HTTPSProxy: "http://proxy.example.com:8080", NoProxy: "localhost", CABundle: []byte("fake-ca-bundle"), }, }, }, managedCluster: addonfactory.NewFakeManagedCluster(clusterName, "1.10.1"), validateObjects: func(t *testing.T, objects []runtime.Object) { if len(objects) != 6 { t.Fatalf("expected 6 objects, but got %v", len(objects)) } unstructDeployment, ok := objects[0].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be *appsv1.Deployment, but got %T", objects[0]) } deployment, err := utils.ConvertToDeployment(unstructDeployment) if err != nil { t.Fatal(err) } if deployment.Namespace != "test-install-namespace" { t.Errorf("unexpected namespace %s", deployment.Namespace) } validatePodTemplate(t, deployment.Spec.Template, []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "4"}, {Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"}, {Name: "CLUSTER_NAME", Value: clusterName}, {Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"}, {Name: "HTTP_PROXY", Value: "http://proxy.example.com:8080"}, {Name: "http_proxy", Value: "http://proxy.example.com:8080"}, {Name: "HTTPS_PROXY", Value: "http://proxy.example.com:8080"}, {Name: "https_proxy", Value: "http://proxy.example.com:8080"}, {Name: "NO_PROXY", Value: "localhost"}, {Name: "no_proxy", Value: "localhost"}, {Name: "CA_BUNDLE_FILE_PATH", Value: "/managed/proxy-ca/ca-bundle.crt"}, }, true) unstructDaemonSet, ok := objects[1].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be *appsv1.DaemonSet, but got %T", objects[1]) } daemonSet, err := utils.ConvertToDaemonSet(unstructDaemonSet) if err != nil { t.Fatal(err) } if daemonSet.Namespace != "test-install-namespace" { t.Errorf("unexpected namespace %s", daemonSet.Namespace) } validatePodTemplate(t, daemonSet.Spec.Template, []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "4"}, {Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"}, {Name: "CLUSTER_NAME", Value: clusterName}, {Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"}, {Name: "HTTP_PROXY", Value: "http://proxy.example.com:8080"}, {Name: "http_proxy", Value: "http://proxy.example.com:8080"}, {Name: "HTTPS_PROXY", Value: "http://proxy.example.com:8080"}, {Name: "https_proxy", Value: "http://proxy.example.com:8080"}, {Name: "NO_PROXY", Value: "localhost"}, {Name: "no_proxy", Value: "localhost"}, {Name: "CA_BUNDLE_FILE_PATH", Value: "/managed/proxy-ca/ca-bundle.crt"}, }, true) // check clusterrole unstructCRB, ok := objects[3].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be unstructured, but got %T", objects[3]) } clusterRoleBinding := &rbacv1.ClusterRoleBinding{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructCRB.Object, clusterRoleBinding) if err != nil { t.Fatal(err) } if clusterRoleBinding.Subjects[0].Namespace != "test-install-namespace" { t.Errorf("clusterRolebinding namespace does not match, got %s", clusterRoleBinding.Subjects[0].Namespace) } unstructConfigmap, ok := objects[5].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be unstructured, but got %T", objects[5]) } configmap := &corev1.ConfigMap{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructConfigmap.Object, configmap) if err != nil { t.Fatal(err) } if configmap.Namespace != "test-install-namespace" { t.Errorf("unexpected namespace %s", configmap.Namespace) } if configmap.Name != "hello-template-proxy-ca" { t.Errorf("unexpected name %s", configmap.Name) } if configmap.Data["ca-bundle.crt"] != "fake-ca-bundle" { t.Errorf("unexpected data %s", configmap.Data["ca-bundle.crt"]) } }, validateAgentOptions: func(t *testing.T, o agent.AgentAddonOptions, mca *addonapiv1beta1.ManagedClusterAddOn) { validateAgentOptions(t, o, mca, "test-install-namespace") }, }, { name: "manifests with only daemonset and no namespace configured by adc rendered successfully", addonTemplatePath: "./testmanifests/addontemplate_daemonset.yaml", addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ // AgentInstallNamespace: "test-install-namespace", CustomizedVariables: []addonapiv1beta1.CustomizedVariable{ { Name: "LOG_LEVEL", Value: "4", }, }, NodePlacement: &addonapiv1beta1.NodePlacement{ NodeSelector: map[string]string{ "host": "ssd", }, Tolerations: []corev1.Toleration{ { Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, }, }, }, Registries: []addonapiv1beta1.ImageMirror{ { Source: "quay.io/open-cluster-management", Mirror: "quay.io/ocm", }, }, }, }, managedCluster: addonfactory.NewFakeManagedCluster(clusterName, "1.10.1"), validateObjects: func(t *testing.T, objects []runtime.Object) { if len(objects) != 3 { t.Fatalf("expected 3 objects, but got %v", len(objects)) } unstructDaemonSet, ok := objects[0].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be *appsv1.DaemonSet, but got %T", objects[0]) } daemonSet, err := utils.ConvertToDaemonSet(unstructDaemonSet) if err != nil { t.Fatal(err) } if daemonSet.Namespace != "open-cluster-management-agent-addon-ds" { // first daemonset ns t.Errorf("unexpected namespace %s", daemonSet.Namespace) } validatePodTemplate(t, daemonSet.Spec.Template, []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "4"}, {Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"}, {Name: "CLUSTER_NAME", Value: clusterName}, // {Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"}, }, false) // check clusterrole unstructCRB, ok := objects[2].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be unstructured, but got %T", objects[2]) } clusterRoleBinding := &rbacv1.ClusterRoleBinding{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructCRB.Object, clusterRoleBinding) if err != nil { t.Fatal(err) } if clusterRoleBinding.Subjects[0].Namespace != "open-cluster-management-agent-addon-ds" { t.Errorf("clusterRolebinding namespace does not match, got %s", clusterRoleBinding.Subjects[0].Namespace) } }, validateAgentOptions: func(t *testing.T, o agent.AgentAddonOptions, mca *addonapiv1beta1.ManagedClusterAddOn) { validateAgentOptions(t, o, mca, "open-cluster-management-agent-addon-ds") }, }, { name: "manifests with only deployment rendered successfully", addonTemplatePath: "./testmanifests/addontemplate_deployment.yaml", addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: "test-install-namespace", CustomizedVariables: []addonapiv1beta1.CustomizedVariable{ { Name: "LOG_LEVEL", Value: "4", }, }, NodePlacement: &addonapiv1beta1.NodePlacement{ NodeSelector: map[string]string{ "host": "ssd", }, Tolerations: []corev1.Toleration{ { Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, }, }, }, Registries: []addonapiv1beta1.ImageMirror{ { Source: "quay.io/open-cluster-management", Mirror: "quay.io/ocm", }, }, }, }, managedCluster: addonfactory.NewFakeManagedCluster(clusterName, "1.10.1"), validateObjects: func(t *testing.T, objects []runtime.Object) { if len(objects) != 4 { t.Fatalf("expected 4 objects, but got %v", len(objects)) } unstructDeployment, ok := objects[0].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be *appsv1.Deployment, but got %T", objects[0]) } deployment, err := utils.ConvertToDeployment(unstructDeployment) if err != nil { t.Fatal(err) } if deployment.Namespace != "test-install-namespace" { t.Errorf("unexpected namespace %s", deployment.Namespace) } validatePodTemplate(t, deployment.Spec.Template, []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "4"}, {Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"}, {Name: "CLUSTER_NAME", Value: clusterName}, {Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"}, }, false) // check clusterrole unstructCRB, ok := objects[2].(*unstructured.Unstructured) if !ok { t.Errorf("expected object to be unstructured, but got %T", objects[2]) } clusterRoleBinding := &rbacv1.ClusterRoleBinding{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructCRB.Object, clusterRoleBinding) if err != nil { t.Fatal(err) } if clusterRoleBinding.Subjects[0].Namespace != "test-install-namespace" { t.Errorf("clusterRolebinding namespace does not match, got %s", clusterRoleBinding.Subjects[0].Namespace) } }, validateAgentOptions: func(t *testing.T, o agent.AgentAddonOptions, mca *addonapiv1beta1.ManagedClusterAddOn) { validateAgentOptions(t, o, mca, "test-install-namespace") }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cma := &addonapiv1beta1.ClusterManagementAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, }, Status: addonapiv1beta1.ClusterManagementAddOnStatus{ DefaultConfigReferences: []addonapiv1beta1.DefaultConfigReference{ { ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: utils.AddOnTemplateGVR.Group, Resource: utils.AddOnTemplateGVR.Resource, }, DesiredConfig: &addonapiv1beta1.ConfigSpecHash{ ConfigReferent: addonapiv1beta1.ConfigReferent{ Name: addonName, }, SpecHash: "fake-hash", }, }, }, }, } var addonTemplate *addonapiv1alpha1.AddOnTemplate if len(tc.addonTemplatePath) > 0 { data, err := os.ReadFile(tc.addonTemplatePath) if err != nil { t.Errorf("error reading file: %v", err) } addonTemplate = &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, }, } _, _, err = decoder.Decode(data, nil, addonTemplate) if err != nil { t.Errorf("error decoding file: %v", err) } } var managedClusterAddon *addonapiv1beta1.ManagedClusterAddOn var objs = []runtime.Object{cma} managedClusterAddonBuilder := newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ) if addonTemplate != nil { managedClusterAddonBuilder.withAddonTemplate(addonTemplate) objs = append(objs, addonTemplate) } if tc.addonDeploymentConfig != nil { managedClusterAddonBuilder.withAddonDeploymentConfig(tc.addonDeploymentConfig) objs = append(objs, tc.addonDeploymentConfig) } managedClusterAddon = managedClusterAddonBuilder.build() objs = append(objs, managedClusterAddon) hubKubeClient := fakekube.NewSimpleClientset() addonClient := fakeaddon.NewSimpleClientset(objs...) addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute) cmaStore := addonInformerFactory.Addon().V1beta1().ClusterManagementAddOns().Informer().GetStore() if err := cmaStore.Add(cma); err != nil { t.Fatal(err) } if managedClusterAddon != nil { mcaStore := addonInformerFactory.Addon().V1beta1().ManagedClusterAddOns().Informer().GetStore() if err := mcaStore.Add(managedClusterAddon); err != nil { t.Fatal(err) } } if addonTemplate != nil { atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore() if err := atStore.Add(addonTemplate); err != nil { t.Fatal(err) } } kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute) agentAddon := NewCRDTemplateAgentAddon( ctx, addonName, hubKubeClient, addonClient, addonInformerFactory, kubeInformers.Rbac().V1().RoleBindings().Lister(), addonfactory.GetAddOnDeploymentConfigValues( utils.NewAddOnDeploymentConfigGetter(addonClient), addonfactory.ToAddOnCustomizedVariableValues, ToAddOnNodePlacementPrivateValues, ToAddOnRegistriesPrivateValues, ToAddOnInstallNamespacePrivateValues, ToAddOnProxyPrivateValues, ), ) objects, err := agentAddon.Manifests(ctx, tc.managedCluster, managedClusterAddon) if err != nil { assert.Equal(t, tc.expectedErr, err.Error(), tc.name) } else { tc.validateObjects(t, objects) addonOptions := agentAddon.GetAgentAddonOptions() tc.validateAgentOptions(t, addonOptions, managedClusterAddon) } }) } } func TestAgentInstallNamespace(t *testing.T) { _, ctx := ktesting.NewTestContext(t) addonName := "hello" clusterName := "cluster1" deployment := testingcommon.NewUnstructured("apps/v1", "Deployment", "test-ns", "test") deploymentRaw, _ := deployment.MarshalJSON() cases := []struct { name string addonTemplate *addonapiv1alpha1.AddOnTemplate addonDeploymentConfig *addonapiv1beta1.AddOnDeploymentConfig managedClusterAddonBuilder *testManagedClusterAddOnBuilder expected string expectedErr string }{ { name: "install namespace is set", addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AgentSpec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ Manifests: []workapiv1.Manifest{}, }, }, }, }, addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: "test-install-namespace", }, }, managedClusterAddonBuilder: newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ), expected: "test-install-namespace", }, { name: "not support addon deployment config", addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AgentSpec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ Manifests: []workapiv1.Manifest{}, }, }, }, }, managedClusterAddonBuilder: newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ), expected: "open-cluster-management-agent-addon", }, { name: "has deployment in the template", addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AgentSpec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ Manifests: []workapiv1.Manifest{ {RawExtension: runtime.RawExtension{Raw: deploymentRaw}}, }, }, }, }, }, managedClusterAddonBuilder: newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ), expected: "test-ns", }, { name: "has deployment in the template, addon status config references is not set", addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AgentSpec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ Manifests: []workapiv1.Manifest{ {RawExtension: runtime.RawExtension{Raw: deploymentRaw}}, }, }, }, }, }, managedClusterAddonBuilder: newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ).withSetStatusConfigReferences(false), expectedErr: fmt.Sprintf("addon %s template not found in status", addonName), }, { name: "override deployment in the template", addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AgentSpec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ Manifests: []workapiv1.Manifest{ {RawExtension: runtime.RawExtension{Raw: deploymentRaw}}, }, }, }, }, }, addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: "test-install-namespace", }, }, managedClusterAddonBuilder: newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ), expected: "test-install-namespace", }, { name: "empty string agentInstallNamespace should use template namespace", addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AgentSpec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ Manifests: []workapiv1.Manifest{ {RawExtension: runtime.RawExtension{Raw: deploymentRaw}}, }, }, }, }, }, addonDeploymentConfig: &addonapiv1beta1.AddOnDeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-config", Namespace: "default", }, Spec: addonapiv1beta1.AddOnDeploymentConfigSpec{ AgentInstallNamespace: "", CustomizedVariables: []addonapiv1beta1.CustomizedVariable{ { Name: "LOG_LEVEL", Value: "4", }, }, }, }, managedClusterAddonBuilder: newManagedClusterAddonBuilder( &addonapiv1beta1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: addonName, Namespace: clusterName, }, }, ), expected: "test-ns", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var managedClusterAddon *addonapiv1beta1.ManagedClusterAddOn var objs []runtime.Object if tc.managedClusterAddonBuilder != nil { if tc.addonTemplate != nil { tc.managedClusterAddonBuilder.withAddonTemplate(tc.addonTemplate) objs = append(objs, tc.addonTemplate) } if tc.addonDeploymentConfig != nil { tc.managedClusterAddonBuilder.withAddonDeploymentConfig(tc.addonDeploymentConfig) objs = append(objs, tc.addonDeploymentConfig) } managedClusterAddon = tc.managedClusterAddonBuilder.build() objs = append(objs, managedClusterAddon) } hubKubeClient := fakekube.NewSimpleClientset() addonClient := fakeaddon.NewSimpleClientset(objs...) addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute) if managedClusterAddon != nil { mcaStore := addonInformerFactory.Addon().V1beta1().ManagedClusterAddOns().Informer().GetStore() if err := mcaStore.Add(managedClusterAddon); err != nil { t.Fatal(err) } } if tc.addonTemplate != nil { atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore() if err := atStore.Add(tc.addonTemplate); err != nil { t.Fatal(err) } } kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute) agentAddon := NewCRDTemplateAgentAddon( ctx, addonName, hubKubeClient, addonClient, addonInformerFactory, kubeInformers.Rbac().V1().RoleBindings().Lister(), addonfactory.GetAddOnDeploymentConfigValues( utils.NewAddOnDeploymentConfigGetter(addonClient), addonfactory.ToAddOnCustomizedVariableValues, ToAddOnNodePlacementPrivateValues, ToAddOnRegistriesPrivateValues, ), ) ns, err := agentAddon.GetAgentAddonOptions().AgentInstallNamespace(ctx, managedClusterAddon) if err == nil { assert.Equal(t, tc.expected, ns, tc.name) } else if tc.expectedErr != err.Error() { t.Fatalf("Expected error %v, but got %v", tc.expectedErr, err) } }) } } func TestAgentManifestConfigs(t *testing.T) { _, ctx := ktesting.NewTestContext(t) addonName := "hello" s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = clusterv1apha1.Install(s) _ = addonapiv1alpha1.Install(s) _ = addonapiv1beta1.Install(s) cases := []struct { name string addonTemplate *addonapiv1alpha1.AddOnTemplate clusterManagementAddOn *addonapiv1beta1.ClusterManagementAddOn expected []workapiv1.ManifestConfigOption }{ { name: "no addontemplate in cma status", clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").Build(), expected: nil, }, { name: "no addontemplate", clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", ""). WithDefaultConfigReferences( addonapiv1beta1.DefaultConfigReference{ ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: utils.AddOnTemplateGVR.Group, Resource: utils.AddOnTemplateGVR.Resource, }, DesiredConfig: &addonapiv1beta1.ConfigSpecHash{ ConfigReferent: addonapiv1beta1.ConfigReferent{Name: "hello-template"}, SpecHash: "hash", }, }, ).Build(), expected: nil, }, { name: "addontemplate does not have manifestconfigs", clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", ""). WithDefaultConfigReferences( addonapiv1beta1.DefaultConfigReference{ ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: utils.AddOnTemplateGVR.Group, Resource: utils.AddOnTemplateGVR.Resource, }, DesiredConfig: &addonapiv1beta1.ConfigSpecHash{ ConfigReferent: addonapiv1beta1.ConfigReferent{Name: "hello-template"}, SpecHash: "hash", }, }, ).Build(), addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, }, expected: nil, }, { name: "addontemplate has manifestconfigs", clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", ""). WithDefaultConfigReferences( addonapiv1beta1.DefaultConfigReference{ ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: utils.AddOnTemplateGVR.Group, Resource: utils.AddOnTemplateGVR.Resource, }, DesiredConfig: &addonapiv1beta1.ConfigSpecHash{ ConfigReferent: addonapiv1beta1.ConfigReferent{Name: "hello-template"}, SpecHash: "hash", }, }, ).Build(), addonTemplate: &addonapiv1alpha1.AddOnTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: "hello-template", }, Spec: addonapiv1alpha1.AddOnTemplateSpec{ AddonName: "hello", AgentSpec: workapiv1.ManifestWorkSpec{ ManifestConfigs: []workapiv1.ManifestConfigOption{ { ResourceIdentifier: workapiv1.ResourceIdentifier{ Group: "apps", Resource: "deployment", Name: "hello", Namespace: "default", }, FeedbackRules: []workapiv1.FeedbackRule{ { Type: workapiv1.WellKnownStatusType, }, }, }, }, }, }, }, expected: []workapiv1.ManifestConfigOption{ { ResourceIdentifier: workapiv1.ResourceIdentifier{ Group: "apps", Resource: "deployment", Name: "hello", Namespace: "default", }, FeedbackRules: []workapiv1.FeedbackRule{ { Type: workapiv1.WellKnownStatusType, }, }, }, }, }, } for _, tc := range cases { var objs []runtime.Object if tc.clusterManagementAddOn != nil { objs = append(objs, tc.clusterManagementAddOn) } if tc.addonTemplate != nil { objs = append(objs, tc.addonTemplate) } hubKubeClient := fakekube.NewSimpleClientset() addonClient := fakeaddon.NewSimpleClientset(objs...) addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute) if tc.clusterManagementAddOn != nil { cmaStore := addonInformerFactory.Addon().V1beta1().ClusterManagementAddOns().Informer().GetStore() if err := cmaStore.Add(tc.clusterManagementAddOn); err != nil { t.Fatal(err) } } if tc.addonTemplate != nil { atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore() if err := atStore.Add(tc.addonTemplate); err != nil { t.Fatal(err) } } kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute) agentAddon := NewCRDTemplateAgentAddon( ctx, addonName, hubKubeClient, addonClient, addonInformerFactory, kubeInformers.Rbac().V1().RoleBindings().Lister(), addonfactory.GetAddOnDeploymentConfigValues( utils.NewAddOnDeploymentConfigGetter(addonClient), addonfactory.ToAddOnCustomizedVariableValues, ToAddOnNodePlacementPrivateValues, ToAddOnRegistriesPrivateValues, ), ) agentManifestConfigs := agentAddon.GetAgentAddonOptions().ManifestConfigs assert.Equal(t, tc.expected, agentManifestConfigs) } } type testManagedClusterAddOnBuilder struct { managedClusterAddOn *addonapiv1beta1.ManagedClusterAddOn addonTemplate *addonapiv1alpha1.AddOnTemplate addonDeploymentConfig *addonapiv1beta1.AddOnDeploymentConfig setStatusConfigReferences bool } func newManagedClusterAddonBuilder(mca *addonapiv1beta1.ManagedClusterAddOn) *testManagedClusterAddOnBuilder { return &testManagedClusterAddOnBuilder{ managedClusterAddOn: mca, setStatusConfigReferences: true, } } func (b *testManagedClusterAddOnBuilder) withAddonTemplate( at *addonapiv1alpha1.AddOnTemplate) *testManagedClusterAddOnBuilder { b.addonTemplate = at return b } func (b *testManagedClusterAddOnBuilder) withAddonDeploymentConfig( adc *addonapiv1beta1.AddOnDeploymentConfig) *testManagedClusterAddOnBuilder { b.addonDeploymentConfig = adc return b } func (b *testManagedClusterAddOnBuilder) withSetStatusConfigReferences(set bool) *testManagedClusterAddOnBuilder { b.setStatusConfigReferences = set return b } func (b *testManagedClusterAddOnBuilder) build() *addonapiv1beta1.ManagedClusterAddOn { if b.addonDeploymentConfig != nil { hash, _ := utils.GetAddOnDeploymentConfigSpecHash(b.addonDeploymentConfig) configReference := addonapiv1beta1.ConfigReference{ ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addondeploymentconfigs", }, DesiredConfig: &addonapiv1beta1.ConfigSpecHash{ ConfigReferent: addonapiv1beta1.ConfigReferent{ Name: b.addonDeploymentConfig.Name, Namespace: b.addonDeploymentConfig.Namespace, }, SpecHash: hash, }, } if b.setStatusConfigReferences { b.managedClusterAddOn.Status.ConfigReferences = append( b.managedClusterAddOn.Status.ConfigReferences, configReference) } b.managedClusterAddOn.Spec.Configs = append(b.managedClusterAddOn.Spec.Configs, addonapiv1beta1.AddOnConfig{ ConfigGroupResource: configReference.ConfigGroupResource, ConfigReferent: configReference.DesiredConfig.ConfigReferent, }) } if b.addonTemplate != nil { hash, _ := GetAddOnTemplateSpecHash(b.addonTemplate) configReference := addonapiv1beta1.ConfigReference{ ConfigGroupResource: addonapiv1beta1.ConfigGroupResource{ Group: "addon.open-cluster-management.io", Resource: "addontemplates", }, DesiredConfig: &addonapiv1beta1.ConfigSpecHash{ ConfigReferent: addonapiv1beta1.ConfigReferent{ Name: b.addonTemplate.Name, }, SpecHash: hash, }, } if b.setStatusConfigReferences { b.managedClusterAddOn.Status.ConfigReferences = append( b.managedClusterAddOn.Status.ConfigReferences, configReference) } b.managedClusterAddOn.Spec.Configs = append(b.managedClusterAddOn.Spec.Configs, addonapiv1beta1.AddOnConfig{ ConfigGroupResource: configReference.ConfigGroupResource, ConfigReferent: configReference.DesiredConfig.ConfigReferent, }) } return b.managedClusterAddOn }