diff --git a/pkg/spoke/resource/mapper.go b/pkg/spoke/resource/mapper.go index 938c2d82c..adb9cef1f 100644 --- a/pkg/spoke/resource/mapper.go +++ b/pkg/spoke/resource/mapper.go @@ -12,6 +12,9 @@ import ( "k8s.io/client-go/restmapper" ) +// MapperRefreshInterval is the refresh interval of mapper. It could be modified during testing +var MapperRefreshInterval = 30 * time.Second + // Mapper is a struct to define resource mapping type Mapper struct { Mapper meta.RESTMapper @@ -34,7 +37,7 @@ func (p *Mapper) Run(stopCh <-chan struct{}) { defer p.syncLock.Unlock() deferredMappd := p.Mapper.(*restmapper.DeferredDiscoveryRESTMapper) deferredMappd.Reset() - }, 30*time.Second, stopCh) + }, MapperRefreshInterval, stopCh) } // MappingForGVK returns the RESTMapping for a gvk diff --git a/test/integration/util/assertion.go b/test/integration/util/assertion.go index a851c7491..db3185e80 100644 --- a/test/integration/util/assertion.go +++ b/test/integration/util/assertion.go @@ -9,6 +9,8 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" workclientset "github.com/open-cluster-management/api/client/work/clientset/versioned" @@ -29,7 +31,7 @@ func AssertWorkCondition(namespace, name string, workClient workclientset.Interf } // check work status condition - return HaveCondition(work.Status.Conditions, string(workapiv1.WorkApplied), metav1.ConditionTrue) + return HaveCondition(work.Status.Conditions, expectedType, expectedWorkStatus) }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) } @@ -71,7 +73,7 @@ func AssertFinalizerAdded(namespace, name string, workClient workclientset.Inter } // check if all manifests are applied -func AssertManifestsApplied(manifests []workapiv1.Manifest, kubeClient kubernetes.Interface, eventuallyTimeout, eventuallyInterval int) { +func AssertExistenceOfConfigMaps(manifests []workapiv1.Manifest, kubeClient kubernetes.Interface, eventuallyTimeout, eventuallyInterval int) { gomega.Eventually(func() bool { for _, manifest := range manifests { expected := manifest.Object.(*corev1.ConfigMap) @@ -88,3 +90,20 @@ func AssertManifestsApplied(manifests []workapiv1.Manifest, kubeClient kubernete return true }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) } + +// check the existence of resource with GVR, namespace and name +func AssertExistenceOfResources(gvrs []schema.GroupVersionResource, namespaces, names []string, dynamicClient dynamic.Interface, eventuallyTimeout, eventuallyInterval int) { + gomega.Expect(gvrs).To(gomega.HaveLen(len(namespaces))) + gomega.Expect(gvrs).To(gomega.HaveLen(len(names))) + + gomega.Eventually(func() bool { + for i := range gvrs { + _, err := GetResource(namespaces[i], names[i], gvrs[i], dynamicClient) + if err != nil { + return false + } + } + + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) +} diff --git a/test/integration/util/unstructured.go b/test/integration/util/unstructured.go new file mode 100644 index 000000000..740a76f2d --- /dev/null +++ b/test/integration/util/unstructured.go @@ -0,0 +1,274 @@ +package util + +import ( + "context" + + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" +) + +const ( + guestbookCrdJson = `{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "guestbooks.my.domain" + }, + "spec": { + "conversion": { + "strategy": "None" + }, + "group": "my.domain", + "names": { + "kind": "Guestbook", + "listKind": "GuestbookList", + "plural": "guestbooks", + "singular": "guestbook" + }, + "preserveUnknownFields": true, + "scope": "Namespaced", + "validation": { + "openAPIV3Schema": { + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "foo": { + "type": "string" + } + }, + "type": "object" + }, + "status": { + "type": "object" + } + }, + "type": "object" + } + }, + "version": "v1", + "versions": [ + { + "name": "v1", + "served": true, + "storage": true + } + ] + } + }` + + guestbookCrJson = `{ + "apiVersion": "my.domain/v1", + "kind": "Guestbook", + "metadata": { + "name": "guestbook1", + "namespace": "default" + }, + "spec": { + "foo": "bar" + } + }` + + deploymentJson = `{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "namespace": "default" + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "image": "nginx:1.14.2", + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "protocol": "TCP" + } + ] + } + ] + } + } + } + }` +) + +var ( + scheme = runtime.NewScheme() + + serviceAccountGVK = schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "ServiceAccount", + } + + serviceAccountGVR = schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "serviceaccounts", + } + + roleGVK = schema.GroupVersionKind{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "Role", + } + + roleGVR = schema.GroupVersionResource{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Resource: "roles", + } + + roleBindingGVK = schema.GroupVersionKind{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "RoleBinding", + } + + roleBindingGVR = schema.GroupVersionResource{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Resource: "rolebindings", + } +) + +func init() { + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) +} + +func GuestbookCrd() (crd *unstructured.Unstructured, gvr schema.GroupVersionResource, err error) { + crd, err = loadResourceFromJSON(guestbookCrdJson) + gvr = schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1beta1", Resource: "customresourcedefinitions"} + return crd, gvr, err +} + +func GuestbookCr(namespace, name string) (cr *unstructured.Unstructured, gvr schema.GroupVersionResource, err error) { + cr, err = loadResourceFromJSON(guestbookCrJson) + if err != nil { + return cr, gvr, err + } + + cr.SetNamespace(namespace) + cr.SetName(name) + gvr = schema.GroupVersionResource{Group: "my.domain", Version: "v1", Resource: "guestbooks"} + return cr, gvr, nil +} + +func NewDeployment(namespace, name, sa string) (u *unstructured.Unstructured, gvr schema.GroupVersionResource, err error) { + u, err = loadResourceFromJSON(deploymentJson) + if err != nil { + return u, gvr, err + } + + u.SetNamespace(namespace) + u.SetName(name) + + err = unstructured.SetNestedField(u.Object, sa, "spec", "template", "spec", "serviceAccountName") + if err != nil { + return u, gvr, err + } + gvr = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + return u, gvr, nil +} + +func toUnstructured(obj runtime.Object, gvk schema.GroupVersionKind, scheme *runtime.Scheme) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + gomega.Expect(scheme.Convert(obj, u, nil)).To(gomega.Succeed()) + u.SetGroupVersionKind(gvk) + return u +} + +func NewServiceAccount(namespace, name string) (*unstructured.Unstructured, schema.GroupVersionResource) { + obj := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + + return toUnstructured(obj, serviceAccountGVK, scheme), serviceAccountGVR +} + +func NewRole(namespace, name string) (*unstructured.Unstructured, schema.GroupVersionResource) { + obj := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"create", "get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + }, + }, + } + + return toUnstructured(obj, roleGVK, scheme), roleGVR +} + +func NewRoleBinding(namespace, name, sa, role string) (*unstructured.Unstructured, schema.GroupVersionResource) { + obj := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role, + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Namespace: namespace, + Name: sa, + }, + }, + } + + return toUnstructured(obj, roleBindingGVK, scheme), roleBindingGVR +} + +func loadResourceFromJSON(json string) (*unstructured.Unstructured, error) { + obj := unstructured.Unstructured{} + err := obj.UnmarshalJSON([]byte(json)) + return &obj, err +} + +func GetResource(namespace, name string, gvr schema.GroupVersionResource, dynamicClient dynamic.Interface) (*unstructured.Unstructured, error) { + return dynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +} diff --git a/test/integration/work_test.go b/test/integration/work_test.go index fded151f5..41eba8a23 100644 --- a/test/integration/work_test.go +++ b/test/integration/work_test.go @@ -2,6 +2,7 @@ package integration import ( "context" + "time" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -9,10 +10,14 @@ import ( "github.com/openshift/library-go/pkg/controller/controllercmd" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/dynamic" workapiv1 "github.com/open-cluster-management/api/work/v1" "github.com/open-cluster-management/work/pkg/spoke" + "github.com/open-cluster-management/work/pkg/spoke/resource" "github.com/open-cluster-management/work/test/integration/util" ) @@ -43,9 +48,14 @@ var _ = ginkgo.Describe("ManifestWork", func() { _, err := spokeKubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + resource.MapperRefreshInterval = 2 * time.Second + var ctx context.Context ctx, cancel = context.WithCancel(context.Background()) go startWorkAgent(ctx, o) + + // reset manifests + manifests = nil }) ginkgo.JustBeforeEach(func() { @@ -70,24 +80,27 @@ var _ = ginkgo.Describe("ManifestWork", func() { }) ginkgo.It("should create work and then apply it successfully", func() { - util.AssertManifestsApplied(manifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + util.AssertExistenceOfConfigMaps(manifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) - /* comment this block until PR work/8 (https://github.com/open-cluster-management/work/pull/8) is merged util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionTrue, []metav1.ConditionStatus{metav1.ConditionTrue}, eventuallyTimeout, eventuallyInterval) - */ }) ginkgo.It("should update work and then apply it successfully", func() { + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionTrue, + []metav1.ConditionStatus{metav1.ConditionTrue}, eventuallyTimeout, eventuallyInterval) + newManifests := []workapiv1.Manifest{ util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm2", map[string]string{"x": "y"})), } + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Get(context.Background(), work.Name, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) work.Spec.Workload.Manifests = newManifests work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Update(context.Background(), work, metav1.UpdateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - util.AssertManifestsApplied(newManifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + util.AssertExistenceOfConfigMaps(newManifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) // TODO: check if resources created by old manifests are deleted }) @@ -111,26 +124,29 @@ var _ = ginkgo.Describe("ManifestWork", func() { }) ginkgo.It("should create work and then apply it successfully", func() { - util.AssertManifestsApplied(manifests[1:], spokeKubeClient, eventuallyTimeout, eventuallyInterval) + util.AssertExistenceOfConfigMaps(manifests[1:], spokeKubeClient, eventuallyTimeout, eventuallyInterval) - /* comment this block until PR work/8 (https://github.com/open-cluster-management/work/pull/8) is merged - util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionTrue, + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionTrue, metav1.ConditionTrue}, eventuallyTimeout, eventuallyInterval) - */ }) ginkgo.It("should update work and then apply it successfully", func() { + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionFalse, + []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionTrue, metav1.ConditionTrue}, eventuallyTimeout, eventuallyInterval) + newManifests := []workapiv1.Manifest{ util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm1", map[string]string{"a": "b"})), util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm2", map[string]string{"x": "y"})), util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm3", map[string]string{"e": "f"})), } - work.Spec.Workload.Manifests = newManifests + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Get(context.Background(), work.Name, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + work.Spec.Workload.Manifests = newManifests work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Update(context.Background(), work, metav1.UpdateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - util.AssertManifestsApplied(newManifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + util.AssertExistenceOfConfigMaps(newManifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) // TODO: check if resources created by old manifests are deleted }) @@ -144,4 +160,146 @@ var _ = ginkgo.Describe("ManifestWork", func() { }) }) + ginkgo.Context("With CRD and CR in manifests", func() { + var spokeDynamicClient dynamic.Interface + var gvrs []schema.GroupVersionResource + var objects []*unstructured.Unstructured + + ginkgo.BeforeEach(func() { + spokeDynamicClient, err = dynamic.NewForConfig(spokeRestConfig) + gvrs = nil + objects = nil + + // crd + obj, gvr, err := util.GuestbookCrd() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gvrs = append(gvrs, gvr) + objects = append(objects, obj) + + // cr + obj, gvr, err = util.GuestbookCr(o.SpokeClusterName, "guestbook1") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gvrs = append(gvrs, gvr) + objects = append(objects, obj) + + for _, obj := range objects { + manifests = append(manifests, util.ToManifest(obj)) + } + }) + + ginkgo.It("should create CRD and CR successfully", func() { + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionTrue, + []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue}, eventuallyTimeout, eventuallyInterval) + + var namespaces, names []string + for _, obj := range objects { + namespaces = append(namespaces, obj.GetNamespace()) + names = append(names, obj.GetName()) + } + + util.AssertExistenceOfResources(gvrs, namespaces, names, spokeDynamicClient, eventuallyTimeout, eventuallyInterval) + }) + }) + + ginkgo.Context("With Service Account, Role, RoleBinding and Deployment in manifests", func() { + var spokeDynamicClient dynamic.Interface + var gvrs []schema.GroupVersionResource + var objects []*unstructured.Unstructured + + ginkgo.BeforeEach(func() { + spokeDynamicClient, err = dynamic.NewForConfig(spokeRestConfig) + gvrs = nil + objects = nil + + u, gvr := util.NewServiceAccount(o.SpokeClusterName, "sa") + gvrs = append(gvrs, gvr) + objects = append(objects, u) + + u, gvr = util.NewRole(o.SpokeClusterName, "role1") + gvrs = append(gvrs, gvr) + objects = append(objects, u) + + u, gvr = util.NewRoleBinding(o.SpokeClusterName, "rolebinding1", "sa", "role1") + gvrs = append(gvrs, gvr) + objects = append(objects, u) + + u, gvr, err = util.NewDeployment(o.SpokeClusterName, "deploy1", "sa") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gvrs = append(gvrs, gvr) + objects = append(objects, u) + + for _, obj := range objects { + manifests = append(manifests, util.ToManifest(obj)) + } + }) + + ginkgo.It("should create Service Account, Role, RoleBinding and Deployment successfully", func() { + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionTrue, + []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue, metav1.ConditionTrue, metav1.ConditionTrue}, + eventuallyTimeout, eventuallyInterval) + + var namespaces, names []string + for _, obj := range objects { + namespaces = append(namespaces, obj.GetNamespace()) + names = append(names, obj.GetName()) + } + + util.AssertExistenceOfResources(gvrs, namespaces, names, spokeDynamicClient, eventuallyTimeout, eventuallyInterval) + }) + + ginkgo.It("should update Service Account and Deployment successfully", func() { + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), metav1.ConditionTrue, + []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue, metav1.ConditionTrue, metav1.ConditionTrue}, + eventuallyTimeout, eventuallyInterval) + // ensure resources are created + var namespaces, names []string + for _, obj := range objects { + namespaces = append(namespaces, obj.GetNamespace()) + names = append(names, obj.GetName()) + } + + util.AssertExistenceOfResources(gvrs, namespaces, names, spokeDynamicClient, eventuallyTimeout, eventuallyInterval) + + // update manifests in work + u, _ := util.NewServiceAccount(o.SpokeClusterName, "admin") + objects[0] = u + u, _, err = util.NewDeployment(o.SpokeClusterName, "deploy1", "admin") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + objects[3] = u + + newManifests := []workapiv1.Manifest{} + for _, obj := range objects { + newManifests = append(newManifests, util.ToManifest(obj)) + } + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Get(context.Background(), work.Name, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + work.Spec.Workload.Manifests = newManifests + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Update(context.Background(), work, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // ensure resources are created + namespaces = nil + names = nil + for _, obj := range objects { + namespaces = append(namespaces, obj.GetNamespace()) + names = append(names, obj.GetName()) + } + util.AssertExistenceOfResources(gvrs, namespaces, names, spokeDynamicClient, eventuallyTimeout, eventuallyInterval) + + // check if deployment is updated + gomega.Eventually(func() bool { + u, err := util.GetResource(o.SpokeClusterName, "deploy1", gvrs[3], spokeDynamicClient) + if err != nil { + return false + } + + sa, _, _ := unstructured.NestedString(u.Object, "spec", "template", "spec", "serviceAccountName") + if "admin" != sa { + return false + } + + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + }) + }) })