From 78c39cf7fb8974f1e993dbcc6277c2d34f3bb01c Mon Sep 17 00:00:00 2001 From: Jian Zhu <36154065+zhujian7@users.noreply.github.com> Date: Thu, 15 Sep 2022 09:22:04 +0800 Subject: [PATCH] check the executor subject permission for action apply (#152) * check work execution permission Signed-off-by: zhujian * Move requeue time in auth package Signed-off-by: zhujian * fix flaky test Signed-off-by: zhujian Signed-off-by: zhujian --- deploy/spoke/clusterrole.yaml | 5 +- pkg/spoke/auth/auth.go | 153 +++++++++++++ pkg/spoke/auth/auth_test.go | 129 +++++++++++ .../manifestwork_controller.go | 52 ++++- .../manifestwork_controller_test.go | 4 +- test/integration/executor_test.go | 212 ++++++++++++++++++ test/integration/util/assertion.go | 15 ++ test/integration/work_test.go | 22 +- 8 files changed, 575 insertions(+), 17 deletions(-) create mode 100644 pkg/spoke/auth/auth.go create mode 100644 pkg/spoke/auth/auth_test.go create mode 100644 test/integration/executor_test.go diff --git a/deploy/spoke/clusterrole.yaml b/deploy/spoke/clusterrole.yaml index 7a27934e3..a545f5918 100644 --- a/deploy/spoke/clusterrole.yaml +++ b/deploy/spoke/clusterrole.yaml @@ -15,4 +15,7 @@ rules: - apiGroups: ["work.open-cluster-management.io"] resources: ["appliedmanifestworks/finalizers"] verbs: ["update"] - \ No newline at end of file +# Allow agent to create subjectaccessreviews +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] diff --git a/pkg/spoke/auth/auth.go b/pkg/spoke/auth/auth.go new file mode 100644 index 000000000..ce3476455 --- /dev/null +++ b/pkg/spoke/auth/auth.go @@ -0,0 +1,153 @@ +package auth + +import ( + "context" + "fmt" + "strings" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + + workapiv1 "open-cluster-management.io/api/work/v1" +) + +// ExecuteAction is the action of executing the manifest work +type ExecuteAction string + +const ( + // ApplyAction represents applying(create/update) resource to the managed cluster + ApplyAction ExecuteAction = "Apply" + // DeleteAction represents deleting resource from the managed cluster + DeleteAction ExecuteAction = "Delete" +) + +// ExecutorValidator validates whether the executor has permission to perform the requests +// to the local managed cluster +type ExecutorValidator interface { + // Validate whether the work executor subject has permission to perform action on the specific manifest, + // if there is no permission will return a kubernetes forbidden error. + Validate(ctx context.Context, executor *workapiv1.ManifestWorkExecutor, + gvr schema.GroupVersionResource, namespace, name string, action ExecuteAction) error +} + +type NotAllowedError struct { + Err error + RequeueTime time.Duration +} + +func (e *NotAllowedError) Error() string { + err := e.Err.Error() + if e.RequeueTime > 0 { + err = fmt.Sprintf("%s, will try again in %s", err, e.RequeueTime.String()) + } + return err +} + +func NewExecutorValidator(kubeClient kubernetes.Interface) ExecutorValidator { + return &sarValidator{ + kubeClient: kubeClient, + } +} + +type sarValidator struct { + kubeClient kubernetes.Interface +} + +func (v *sarValidator) Validate(ctx context.Context, executor *workapiv1.ManifestWorkExecutor, + gvr schema.GroupVersionResource, namespace, name string, action ExecuteAction) error { + if executor == nil { + return nil + } + + if executor.Subject.Type != workapiv1.ExecutorSubjectTypeServiceAccount { + return fmt.Errorf("only support %s type for the executor", workapiv1.ExecutorSubjectTypeServiceAccount) + } + + sa := executor.Subject.ServiceAccount + if sa == nil { + return fmt.Errorf("the executor service account is nil") + } + + var verbs []string + switch action { + case ApplyAction: + verbs = []string{"create", "update", "patch", "get"} + case DeleteAction: + verbs = []string{"delete"} + default: + return fmt.Errorf("execute action %s is invalid", action) + } + + resource := authorizationv1.ResourceAttributes{ + Namespace: namespace, + Name: name, + Group: gvr.Group, + Version: gvr.Version, + Resource: gvr.Resource, + } + + reviews := buildSubjectAccessReviews(sa.Namespace, sa.Name, resource, verbs...) + allowed, err := validateBySubjectAccessReviews(ctx, v.kubeClient, reviews) + if err != nil { + return err + } + + if !allowed { + return &NotAllowedError{ + Err: fmt.Errorf("not allowed to %s the resource %s %s, name: %s", + strings.ToLower(string(action)), resource.Group, resource.Resource, resource.Name), + RequeueTime: 60 * time.Second, + } + } + + return nil +} + +func buildSubjectAccessReviews(saNamespace string, saName string, + resource authorizationv1.ResourceAttributes, + verbs ...string) []authorizationv1.SubjectAccessReview { + + reviews := []authorizationv1.SubjectAccessReview{} + for _, verb := range verbs { + reviews = append(reviews, authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Group: resource.Group, + Resource: resource.Resource, + Version: resource.Version, + Subresource: resource.Subresource, + Name: resource.Name, + Namespace: resource.Namespace, + Verb: verb, + }, + User: fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, saName), + Groups: []string{"system:serviceaccounts", "system:authenticated", + fmt.Sprintf("system:serviceaccounts:%s", saNamespace)}, + }, + }) + } + return reviews +} + +func validateBySubjectAccessReviews( + ctx context.Context, + kubeClient kubernetes.Interface, + subjectAccessReviews []authorizationv1.SubjectAccessReview) (bool, error) { + + for i := range subjectAccessReviews { + subjectAccessReview := subjectAccessReviews[i] + + sar, err := kubeClient.AuthorizationV1().SubjectAccessReviews().Create( + ctx, &subjectAccessReview, metav1.CreateOptions{}) + if err != nil { + return false, err + } + if !sar.Status.Allowed { + return false, nil + } + } + return true, nil +} diff --git a/pkg/spoke/auth/auth_test.go b/pkg/spoke/auth/auth_test.go new file mode 100644 index 000000000..7c193db2a --- /dev/null +++ b/pkg/spoke/auth/auth_test.go @@ -0,0 +1,129 @@ +package auth_test + +import ( + "context" + "fmt" + "testing" + + v1 "k8s.io/api/authorization/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + fakekube "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + + workapiv1 "open-cluster-management.io/api/work/v1" + "open-cluster-management.io/work/pkg/spoke/auth" +) + +func TestValidate(t *testing.T) { + + tests := map[string]struct { + executor *workapiv1.ManifestWorkExecutor + namespace string + name string + action auth.ExecuteAction + expect error + }{ + "executor nil": { + executor: nil, + expect: nil, + }, + "unsupported type": { + executor: &workapiv1.ManifestWorkExecutor{ + Subject: workapiv1.ManifestWorkExecutorSubject{ + Type: "test", + }, + }, + expect: fmt.Errorf("only support %s type for the executor", workapiv1.ExecutorSubjectTypeServiceAccount), + }, + "sa nil": { + executor: &workapiv1.ManifestWorkExecutor{ + Subject: workapiv1.ManifestWorkExecutorSubject{ + Type: workapiv1.ExecutorSubjectTypeServiceAccount, + }, + }, + expect: fmt.Errorf("the executor service account is nil"), + }, + "action invalid": { + executor: &workapiv1.ManifestWorkExecutor{ + Subject: workapiv1.ManifestWorkExecutorSubject{ + Type: workapiv1.ExecutorSubjectTypeServiceAccount, + ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{ + Namespace: "test-ns", + Name: "test-name", + }, + }, + }, + action: "test-action", + expect: fmt.Errorf("execute action test-action is invalid"), + }, + "forbideen": { + executor: &workapiv1.ManifestWorkExecutor{ + Subject: workapiv1.ManifestWorkExecutorSubject{ + Type: workapiv1.ExecutorSubjectTypeServiceAccount, + ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{ + Namespace: "test-ns", + Name: "test-name", + }, + }, + }, + namespace: "test-deny", + name: "test", + action: auth.ApplyAction, + expect: fmt.Errorf("not allowed to apply the resource secrets, name: test, will try again in 1m0s"), + }, + "allow": { + executor: &workapiv1.ManifestWorkExecutor{ + Subject: workapiv1.ManifestWorkExecutorSubject{ + Type: workapiv1.ExecutorSubjectTypeServiceAccount, + ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{ + Namespace: "test-ns", + Name: "test-name", + }, + }, + }, + namespace: "test-allow", + name: "test", + action: auth.ApplyAction, + expect: nil, + }, + } + + gvr := schema.GroupVersionResource{Version: "v1", Resource: "secrets"} + kubeClient := fakekube.NewSimpleClientset() + kubeClient.PrependReactor("create", "subjectaccessreviews", + func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + obj := action.(clienttesting.CreateActionImpl).Object.(*v1.SubjectAccessReview) + + if obj.Spec.ResourceAttributes.Namespace == "test-allow" { + return true, &v1.SubjectAccessReview{ + Status: v1.SubjectAccessReviewStatus{ + Allowed: true, + }, + }, nil + } + + if obj.Spec.ResourceAttributes.Namespace == "test-deny" { + return true, &v1.SubjectAccessReview{ + Status: v1.SubjectAccessReviewStatus{ + Denied: true, + }, + }, nil + } + return false, nil, nil + }, + ) + validator := auth.NewExecutorValidator(kubeClient) + for testName, test := range tests { + t.Run(testName, func(t *testing.T) { + err := validator.Validate(context.TODO(), test.executor, gvr, test.namespace, test.name, test.action) + if test.expect == nil { + if err != nil { + t.Errorf("expect nil but got %s", err) + } + } else if err == nil || err.Error() != test.expect.Error() { + t.Errorf("expect %s but got %s", test.expect, err) + } + }) + } +} diff --git a/pkg/spoke/controllers/manifestcontroller/manifestwork_controller.go b/pkg/spoke/controllers/manifestcontroller/manifestwork_controller.go index 9d34d7876..a9c6d45b8 100644 --- a/pkg/spoke/controllers/manifestcontroller/manifestwork_controller.go +++ b/pkg/spoke/controllers/manifestcontroller/manifestwork_controller.go @@ -30,10 +30,14 @@ import ( "open-cluster-management.io/work/pkg/helper" "open-cluster-management.io/work/pkg/spoke/apply" + "open-cluster-management.io/work/pkg/spoke/auth" "open-cluster-management.io/work/pkg/spoke/controllers" ) -var ResyncInterval = 5 * time.Minute +var ( + ResyncInterval = 5 * time.Minute + MaxRequeueDuration = 24 * time.Hour +) // ManifestWorkController is to reconcile the workload resources // fetched from hub cluster on spoke cluster. @@ -46,6 +50,7 @@ type ManifestWorkController struct { hubHash string restMapper meta.RESTMapper appliers *apply.Appliers + validator auth.ExecutorValidator } type applyResult struct { @@ -78,8 +83,8 @@ func NewManifestWorkController( spokeDynamicClient: spokeDynamicClient, hubHash: hubHash, restMapper: restMapper, - - appliers: apply.NewAppliers(spokeDynamicClient, spokeKubeClient, spokeAPIExtensionClient), + appliers: apply.NewAppliers(spokeDynamicClient, spokeKubeClient, spokeAPIExtensionClient), + validator: auth.NewExecutorValidator(spokeKubeClient), } return factory.New(). @@ -125,6 +130,7 @@ func (m *ManifestWorkController) sync(ctx context.Context, controllerContext fac if !found { return nil } + // Apply appliedManifestWork appliedManifestWorkName := fmt.Sprintf("%s-%s", m.hubHash, manifestWork.Name) appliedManifestWork, err := m.appliedManifestWorkLister.Get(appliedManifestWorkName) @@ -171,13 +177,8 @@ func (m *ManifestWorkController) sync(ctx context.Context, controllerContext fac } newManifestConditions := []workapiv1.ManifestCondition{} + var requeueTime = MaxRequeueDuration for _, result := range resourceResults { - // ignore server side apply conflict error since it cannot be resolved by error fallback. - var ssaConflict = &apply.ServerSideApplyConflictError{} - if result.Error != nil && !errors.As(result.Error, &ssaConflict) { - errs = append(errs, result.Error) - } - manifestCondition := workapiv1.ManifestCondition{ ResourceMeta: result.resourceMeta, Conditions: []metav1.Condition{}, @@ -187,18 +188,42 @@ func (m *ManifestWorkController) sync(ctx context.Context, controllerContext fac manifestCondition.Conditions = append(manifestCondition.Conditions, buildAppliedStatusCondition(result)) newManifestConditions = append(newManifestConditions, manifestCondition) + + // If it is a forbidden error, after the condition is constructed, we set the error to nil + // and requeue the item + var authError = &auth.NotAllowedError{} + if errors.As(result.Error, &authError) { + klog.V(2).Infof("apply work %s fails with err: %v", manifestWorkName, result.Error) + result.Error = nil + + if authError.RequeueTime < requeueTime { + requeueTime = authError.RequeueTime + } + } + + // ignore server side apply conflict error since it cannot be resolved by error fallback. + var ssaConflict = &apply.ServerSideApplyConflictError{} + if result.Error != nil && !errors.As(result.Error, &ssaConflict) { + errs = append(errs, result.Error) + } } // Update work status - _, _, err = helper.UpdateManifestWorkStatus( + _, updated, err := helper.UpdateManifestWorkStatus( ctx, m.manifestWorkClient, manifestWork, m.generateUpdateStatusFunc(manifestWork.Generation, newManifestConditions)) if err != nil { errs = append(errs, fmt.Errorf("failed to update work status with err %w", err)) } + + if !updated && requeueTime < MaxRequeueDuration { + controllerContext.Queue().AddAfter(manifestWorkName, requeueTime) + } + if len(errs) > 0 { err = utilerrors.NewAggregate(errs) klog.Errorf("Reconcile work %s fails with err: %v", manifestWorkName, err) } + return err } @@ -248,6 +273,13 @@ func (m *ManifestWorkController) applyOneManifest( return result } + // check the Executor subject permission before applying + err = m.validator.Validate(ctx, workSpec.Executor, gvr, resMeta.Namespace, resMeta.Name, auth.ApplyAction) + if err != nil { + result.Error = err + return result + } + // compute required ownerrefs based on delete option requiredOwner := manageOwnerRef(gvr, resMeta.Namespace, resMeta.Name, workSpec.DeleteOption, owner) diff --git a/pkg/spoke/controllers/manifestcontroller/manifestwork_controller_test.go b/pkg/spoke/controllers/manifestcontroller/manifestwork_controller_test.go index b362b3599..7a2e13b63 100644 --- a/pkg/spoke/controllers/manifestcontroller/manifestwork_controller_test.go +++ b/pkg/spoke/controllers/manifestcontroller/manifestwork_controller_test.go @@ -22,6 +22,7 @@ import ( workinformers "open-cluster-management.io/api/client/work/informers/externalversions" workapiv1 "open-cluster-management.io/api/work/v1" "open-cluster-management.io/work/pkg/spoke/apply" + "open-cluster-management.io/work/pkg/spoke/auth" "open-cluster-management.io/work/pkg/spoke/controllers" "open-cluster-management.io/work/pkg/spoke/spoketesting" ) @@ -36,13 +37,14 @@ type testController struct { func newController(t *testing.T, work *workapiv1.ManifestWork, appliedWork *workapiv1.AppliedManifestWork, mapper meta.RESTMapper) *testController { fakeWorkClient := fakeworkclient.NewSimpleClientset(work) workInformerFactory := workinformers.NewSharedInformerFactoryWithOptions(fakeWorkClient, 5*time.Minute, workinformers.WithNamespace("cluster1")) - + spokeKubeClient := fakekube.NewSimpleClientset() controller := &ManifestWorkController{ manifestWorkClient: fakeWorkClient.WorkV1().ManifestWorks("cluster1"), manifestWorkLister: workInformerFactory.Work().V1().ManifestWorks().Lister().ManifestWorks("cluster1"), appliedManifestWorkClient: fakeWorkClient.WorkV1().AppliedManifestWorks(), appliedManifestWorkLister: workInformerFactory.Work().V1().AppliedManifestWorks().Lister(), restMapper: mapper, + validator: auth.NewExecutorValidator(spokeKubeClient), } if err := workInformerFactory.Work().V1().ManifestWorks().Informer().GetStore().Add(work); err != nil { diff --git a/test/integration/executor_test.go b/test/integration/executor_test.go new file mode 100644 index 000000000..5f90324d9 --- /dev/null +++ b/test/integration/executor_test.go @@ -0,0 +1,212 @@ +package integration + +import ( + "context" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilrand "k8s.io/apimachinery/pkg/util/rand" + workapiv1 "open-cluster-management.io/api/work/v1" + "open-cluster-management.io/work/pkg/spoke" + "open-cluster-management.io/work/test/integration/util" +) + +var _ = ginkgo.Describe("ManifestWork Executor Subject", func() { + var o *spoke.WorkloadAgentOptions + var cancel context.CancelFunc + + var work *workapiv1.ManifestWork + var manifests []workapiv1.Manifest + var executor *workapiv1.ManifestWorkExecutor + + var err error + + ginkgo.BeforeEach(func() { + o = spoke.NewWorkloadAgentOptions() + o.HubKubeconfigFile = hubKubeconfigFileName + o.SpokeClusterName = utilrand.String(5) + o.StatusSyncInterval = 3 * time.Second + + ns := &corev1.Namespace{} + ns.Name = o.SpokeClusterName + _, err := spokeKubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + go startWorkAgent(ctx, o) + + // reset manifests + manifests = nil + executor = nil + }) + + ginkgo.JustBeforeEach(func() { + work = util.NewManifestWork(o.SpokeClusterName, "", manifests) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + work.Spec.Executor = executor + }) + + ginkgo.AfterEach(func() { + if cancel != nil { + cancel() + } + err := spokeKubeClient.CoreV1().Namespaces().Delete( + context.Background(), o.SpokeClusterName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.Context("Apply the resource with executor", func() { + executorName := "test-executor" + ginkgo.BeforeEach(func() { + manifests = []workapiv1.Manifest{ + util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm1", map[string]string{"a": "b"}, []string{})), + util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm2", map[string]string{"c": "d"}, []string{})), + } + executor = &workapiv1.ManifestWorkExecutor{ + Subject: workapiv1.ManifestWorkExecutorSubject{ + Type: workapiv1.ExecutorSubjectTypeServiceAccount, + ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{ + Namespace: o.SpokeClusterName, + Name: executorName, + }, + }, + } + }) + + ginkgo.It("Executor does not have permission", func() { + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Create( + context.Background(), work, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), + metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionFalse}, + eventuallyTimeout, eventuallyInterval) + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkAvailable), + metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionFalse}, + eventuallyTimeout, eventuallyInterval) + + // ensure configmaps not exist + util.AssertNonexistenceOfConfigMaps(manifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + }) + + ginkgo.It("Executor does not have permission to partial resources", func() { + roleName := "role1" + _, err = spokeKubeClient.RbacV1().Roles(o.SpokeClusterName).Create( + context.TODO(), &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: o.SpokeClusterName, + Name: roleName, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"create", "update", "patch", "get", "list", "delete"}, + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"cm1"}, + }, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = spokeKubeClient.RbacV1().RoleBindings(o.SpokeClusterName).Create( + context.TODO(), &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: o.SpokeClusterName, + Name: roleName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Namespace: o.SpokeClusterName, + Name: executorName, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: roleName, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Create( + context.Background(), work, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), + metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse}, + eventuallyTimeout, eventuallyInterval) + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkAvailable), + metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse}, + eventuallyTimeout, eventuallyInterval) + + // ensure configmap cm1 exist and cm2 not exist + util.AssertExistenceOfConfigMaps( + []workapiv1.Manifest{ + util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm1", map[string]string{"a": "b"}, []string{})), + }, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + util.AssertNonexistenceOfConfigMaps( + []workapiv1.Manifest{ + util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm2", map[string]string{"a": "b"}, []string{})), + }, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + }) + + ginkgo.It("Executor has permission for all resources", func() { + roleName := "role1" + _, err = spokeKubeClient.RbacV1().Roles(o.SpokeClusterName).Create( + context.TODO(), &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: o.SpokeClusterName, + Name: roleName, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"create", "update", "patch", "get", "list", "delete"}, + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"cm1", "cm2"}, + }, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = spokeKubeClient.RbacV1().RoleBindings(o.SpokeClusterName).Create( + context.TODO(), &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: o.SpokeClusterName, + Name: roleName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Namespace: o.SpokeClusterName, + Name: executorName, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: roleName, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Create( + context.Background(), work, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied), + metav1.ConditionTrue, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue}, + eventuallyTimeout, eventuallyInterval) + util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkAvailable), + metav1.ConditionTrue, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue}, + eventuallyTimeout, eventuallyInterval) + + // ensure configmaps all exist + util.AssertExistenceOfConfigMaps(manifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval) + }) + }) +}) diff --git a/test/integration/util/assertion.go b/test/integration/util/assertion.go index 5b4d30c19..9f6195d5d 100644 --- a/test/integration/util/assertion.go +++ b/test/integration/util/assertion.go @@ -122,6 +122,21 @@ func AssertExistenceOfConfigMaps(manifests []workapiv1.Manifest, kubeClient kube }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) } +// check if configmap does not exist +func AssertNonexistenceOfConfigMaps(manifests []workapiv1.Manifest, kubeClient kubernetes.Interface, + eventuallyTimeout, eventuallyInterval int) { + gomega.Eventually(func() bool { + for _, manifest := range manifests { + expected := manifest.Object.(*corev1.ConfigMap) + _, err := kubeClient.CoreV1().ConfigMaps(expected.Namespace).Get( + context.Background(), expected.Name, metav1.GetOptions{}) + return apierrors.IsNotFound(err) + } + + return false + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.BeFalse()) +} + // check the existence of resource with GVR, namespace and name func AssertExistenceOfResources(gvrs []schema.GroupVersionResource, namespaces, names []string, dynamicClient dynamic.Interface, eventuallyTimeout, eventuallyInterval int) { gomega.Expect(gvrs).To(gomega.HaveLen(len(namespaces))) diff --git a/test/integration/work_test.go b/test/integration/work_test.go index 553eb77a0..216c129b7 100644 --- a/test/integration/work_test.go +++ b/test/integration/work_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/dynamic" + "k8s.io/client-go/util/retry" workapiv1 "open-cluster-management.io/api/work/v1" "open-cluster-management.io/work/pkg/spoke" @@ -636,11 +637,22 @@ var _ = ginkgo.Describe("ManifestWork", func() { go func() { for _, manifest := range manifests { cm := manifest.Object.(*corev1.ConfigMap) - cm, err := spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Get(context.Background(), cm.Name, metav1.GetOptions{}) - if err == nil { - cm.Finalizers = nil - _, _ = spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Update(context.Background(), cm, metav1.UpdateOptions{}) - } + err = retry.OnError( + retry.DefaultBackoff, + func(err error) bool { + return err != nil + }, + func() error { + cm, err := spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Get(context.Background(), cm.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + cm.Finalizers = nil + _, err = spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Update(context.Background(), cm, metav1.UpdateOptions{}) + return err + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) time.Sleep(2 * time.Second) } }()