check the executor subject permission for action apply (#152)

* check work execution permission

Signed-off-by: zhujian <jiazhu@redhat.com>

* Move requeue time in auth package

Signed-off-by: zhujian <jiazhu@redhat.com>

* fix flaky test

Signed-off-by: zhujian <jiazhu@redhat.com>

Signed-off-by: zhujian <jiazhu@redhat.com>

deploy/spoke/clusterrole.yaml

@@ -15,4 +15,7 @@ rules:
 - apiGroups: ["work.open-cluster-management.io"]
   resources: ["appliedmanifestworks/finalizers"]
   verbs: ["update"]
-  
+# Allow agent to create subjectaccessreviews
+- apiGroups: ["authorization.k8s.io"]
+  resources: ["subjectaccessreviews"]
+  verbs: ["create"]

pkg/spoke/auth/auth.go

@@ -0,0 +1,153 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	authorizationv1 "k8s.io/api/authorization/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+
+	workapiv1 "open-cluster-management.io/api/work/v1"
+)
+
+// ExecuteAction is the action of executing the manifest work
+type ExecuteAction string
+
+const (
+	// ApplyAction represents applying(create/update) resource to the managed cluster
+	ApplyAction ExecuteAction = "Apply"
+	// DeleteAction represents deleting resource from the managed cluster
+	DeleteAction ExecuteAction = "Delete"
+)
+
+// ExecutorValidator validates whether the executor has permission to perform the requests
+// to the local managed cluster
+type ExecutorValidator interface {
+	// Validate whether the work executor subject has permission to perform action on the specific manifest,
+	// if there is no permission will return a kubernetes forbidden error.
+	Validate(ctx context.Context, executor *workapiv1.ManifestWorkExecutor,
+		gvr schema.GroupVersionResource, namespace, name string, action ExecuteAction) error
+}
+
+type NotAllowedError struct {
+	Err         error
+	RequeueTime time.Duration
+}
+
+func (e *NotAllowedError) Error() string {
+	err := e.Err.Error()
+	if e.RequeueTime > 0 {
+		err = fmt.Sprintf("%s, will try again in %s", err, e.RequeueTime.String())
+	}
+	return err
+}
+
+func NewExecutorValidator(kubeClient kubernetes.Interface) ExecutorValidator {
+	return &sarValidator{
+		kubeClient: kubeClient,
+	}
+}
+
+type sarValidator struct {
+	kubeClient kubernetes.Interface
+}
+
+func (v *sarValidator) Validate(ctx context.Context, executor *workapiv1.ManifestWorkExecutor,
+	gvr schema.GroupVersionResource, namespace, name string, action ExecuteAction) error {
+	if executor == nil {
+		return nil
+	}
+
+	if executor.Subject.Type != workapiv1.ExecutorSubjectTypeServiceAccount {
+		return fmt.Errorf("only support %s type for the executor", workapiv1.ExecutorSubjectTypeServiceAccount)
+	}
+
+	sa := executor.Subject.ServiceAccount
+	if sa == nil {
+		return fmt.Errorf("the executor service account is nil")
+	}
+
+	var verbs []string
+	switch action {
+	case ApplyAction:
+		verbs = []string{"create", "update", "patch", "get"}
+	case DeleteAction:
+		verbs = []string{"delete"}
+	default:
+		return fmt.Errorf("execute action %s is invalid", action)
+	}
+
+	resource := authorizationv1.ResourceAttributes{
+		Namespace: namespace,
+		Name:      name,
+		Group:     gvr.Group,
+		Version:   gvr.Version,
+		Resource:  gvr.Resource,
+	}
+
+	reviews := buildSubjectAccessReviews(sa.Namespace, sa.Name, resource, verbs...)
+	allowed, err := validateBySubjectAccessReviews(ctx, v.kubeClient, reviews)
+	if err != nil {
+		return err
+	}
+
+	if !allowed {
+		return &NotAllowedError{
+			Err: fmt.Errorf("not allowed to %s the resource %s %s, name: %s",
+				strings.ToLower(string(action)), resource.Group, resource.Resource, resource.Name),
+			RequeueTime: 60 * time.Second,
+		}
+	}
+
+	return nil
+}
+
+func buildSubjectAccessReviews(saNamespace string, saName string,
+	resource authorizationv1.ResourceAttributes,
+	verbs ...string) []authorizationv1.SubjectAccessReview {
+
+	reviews := []authorizationv1.SubjectAccessReview{}
+	for _, verb := range verbs {
+		reviews = append(reviews, authorizationv1.SubjectAccessReview{
+			Spec: authorizationv1.SubjectAccessReviewSpec{
+				ResourceAttributes: &authorizationv1.ResourceAttributes{
+					Group:       resource.Group,
+					Resource:    resource.Resource,
+					Version:     resource.Version,
+					Subresource: resource.Subresource,
+					Name:        resource.Name,
+					Namespace:   resource.Namespace,
+					Verb:        verb,
+				},
+				User: fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, saName),
+				Groups: []string{"system:serviceaccounts", "system:authenticated",
+					fmt.Sprintf("system:serviceaccounts:%s", saNamespace)},
+			},
+		})
+	}
+	return reviews
+}
+
+func validateBySubjectAccessReviews(
+	ctx context.Context,
+	kubeClient kubernetes.Interface,
+	subjectAccessReviews []authorizationv1.SubjectAccessReview) (bool, error) {
+
+	for i := range subjectAccessReviews {
+		subjectAccessReview := subjectAccessReviews[i]
+
+		sar, err := kubeClient.AuthorizationV1().SubjectAccessReviews().Create(
+			ctx, &subjectAccessReview, metav1.CreateOptions{})
+		if err != nil {
+			return false, err
+		}
+		if !sar.Status.Allowed {
+			return false, nil
+		}
+	}
+	return true, nil
+}

pkg/spoke/auth/auth_test.go

@@ -0,0 +1,129 @@
+package auth_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	v1 "k8s.io/api/authorization/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	fakekube "k8s.io/client-go/kubernetes/fake"
+	clienttesting "k8s.io/client-go/testing"
+
+	workapiv1 "open-cluster-management.io/api/work/v1"
+	"open-cluster-management.io/work/pkg/spoke/auth"
+)
+
+func TestValidate(t *testing.T) {
+
+	tests := map[string]struct {
+		executor  *workapiv1.ManifestWorkExecutor
+		namespace string
+		name      string
+		action    auth.ExecuteAction
+		expect    error
+	}{
+		"executor nil": {
+			executor: nil,
+			expect:   nil,
+		},
+		"unsupported type": {
+			executor: &workapiv1.ManifestWorkExecutor{
+				Subject: workapiv1.ManifestWorkExecutorSubject{
+					Type: "test",
+				},
+			},
+			expect: fmt.Errorf("only support %s type for the executor", workapiv1.ExecutorSubjectTypeServiceAccount),
+		},
+		"sa nil": {
+			executor: &workapiv1.ManifestWorkExecutor{
+				Subject: workapiv1.ManifestWorkExecutorSubject{
+					Type: workapiv1.ExecutorSubjectTypeServiceAccount,
+				},
+			},
+			expect: fmt.Errorf("the executor service account is nil"),
+		},
+		"action invalid": {
+			executor: &workapiv1.ManifestWorkExecutor{
+				Subject: workapiv1.ManifestWorkExecutorSubject{
+					Type: workapiv1.ExecutorSubjectTypeServiceAccount,
+					ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{
+						Namespace: "test-ns",
+						Name:      "test-name",
+					},
+				},
+			},
+			action: "test-action",
+			expect: fmt.Errorf("execute action test-action is invalid"),
+		},
+		"forbideen": {
+			executor: &workapiv1.ManifestWorkExecutor{
+				Subject: workapiv1.ManifestWorkExecutorSubject{
+					Type: workapiv1.ExecutorSubjectTypeServiceAccount,
+					ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{
+						Namespace: "test-ns",
+						Name:      "test-name",
+					},
+				},
+			},
+			namespace: "test-deny",
+			name:      "test",
+			action:    auth.ApplyAction,
+			expect:    fmt.Errorf("not allowed to apply the resource  secrets, name: test, will try again in 1m0s"),
+		},
+		"allow": {
+			executor: &workapiv1.ManifestWorkExecutor{
+				Subject: workapiv1.ManifestWorkExecutorSubject{
+					Type: workapiv1.ExecutorSubjectTypeServiceAccount,
+					ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{
+						Namespace: "test-ns",
+						Name:      "test-name",
+					},
+				},
+			},
+			namespace: "test-allow",
+			name:      "test",
+			action:    auth.ApplyAction,
+			expect:    nil,
+		},
+	}
+
+	gvr := schema.GroupVersionResource{Version: "v1", Resource: "secrets"}
+	kubeClient := fakekube.NewSimpleClientset()
+	kubeClient.PrependReactor("create", "subjectaccessreviews",
+		func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+			obj := action.(clienttesting.CreateActionImpl).Object.(*v1.SubjectAccessReview)
+
+			if obj.Spec.ResourceAttributes.Namespace == "test-allow" {
+				return true, &v1.SubjectAccessReview{
+					Status: v1.SubjectAccessReviewStatus{
+						Allowed: true,
+					},
+				}, nil
+			}
+
+			if obj.Spec.ResourceAttributes.Namespace == "test-deny" {
+				return true, &v1.SubjectAccessReview{
+					Status: v1.SubjectAccessReviewStatus{
+						Denied: true,
+					},
+				}, nil
+			}
+			return false, nil, nil
+		},
+	)
+	validator := auth.NewExecutorValidator(kubeClient)
+	for testName, test := range tests {
+		t.Run(testName, func(t *testing.T) {
+			err := validator.Validate(context.TODO(), test.executor, gvr, test.namespace, test.name, test.action)
+			if test.expect == nil {
+				if err != nil {
+					t.Errorf("expect nil but got %s", err)
+				}
+			} else if err == nil || err.Error() != test.expect.Error() {
+				t.Errorf("expect %s but got %s", test.expect, err)
+			}
+		})
+	}
+}

pkg/spoke/controllers/manifestcontroller/manifestwork_controller.go

@@ -30,10 +30,14 @@ import (
 
 	"open-cluster-management.io/work/pkg/helper"
 	"open-cluster-management.io/work/pkg/spoke/apply"
+	"open-cluster-management.io/work/pkg/spoke/auth"
 	"open-cluster-management.io/work/pkg/spoke/controllers"
 )
 
-var ResyncInterval = 5 * time.Minute
+var (
+	ResyncInterval     = 5 * time.Minute
+	MaxRequeueDuration = 24 * time.Hour
+)
 
 // ManifestWorkController is to reconcile the workload resources
 // fetched from hub cluster on spoke cluster.
@@ -46,6 +50,7 @@ type ManifestWorkController struct {
 	hubHash                   string
 	restMapper                meta.RESTMapper
 	appliers                  *apply.Appliers
+	validator                 auth.ExecutorValidator
 }
 
 type applyResult struct {
@@ -78,8 +83,8 @@ func NewManifestWorkController(
 		spokeDynamicClient:        spokeDynamicClient,
 		hubHash:                   hubHash,
 		restMapper:                restMapper,
-
-		appliers: apply.NewAppliers(spokeDynamicClient, spokeKubeClient, spokeAPIExtensionClient),
+		appliers:                  apply.NewAppliers(spokeDynamicClient, spokeKubeClient, spokeAPIExtensionClient),
+		validator:                 auth.NewExecutorValidator(spokeKubeClient),
 	}
 
 	return factory.New().
@@ -125,6 +130,7 @@ func (m *ManifestWorkController) sync(ctx context.Context, controllerContext fac
 	if !found {
 		return nil
 	}
+
 	// Apply appliedManifestWork
 	appliedManifestWorkName := fmt.Sprintf("%s-%s", m.hubHash, manifestWork.Name)
 	appliedManifestWork, err := m.appliedManifestWorkLister.Get(appliedManifestWorkName)
@@ -171,13 +177,8 @@ func (m *ManifestWorkController) sync(ctx context.Context, controllerContext fac
 	}
 
 	newManifestConditions := []workapiv1.ManifestCondition{}
+	var requeueTime = MaxRequeueDuration
 	for _, result := range resourceResults {
-		// ignore server side apply conflict error since it cannot be resolved by error fallback.
-		var ssaConflict = &apply.ServerSideApplyConflictError{}
-		if result.Error != nil && !errors.As(result.Error, &ssaConflict) {
-			errs = append(errs, result.Error)
-		}
-
 		manifestCondition := workapiv1.ManifestCondition{
 			ResourceMeta: result.resourceMeta,
 			Conditions:   []metav1.Condition{},
@@ -187,18 +188,42 @@ func (m *ManifestWorkController) sync(ctx context.Context, controllerContext fac
 		manifestCondition.Conditions = append(manifestCondition.Conditions, buildAppliedStatusCondition(result))
 
 		newManifestConditions = append(newManifestConditions, manifestCondition)
+
+		// If it is a forbidden error, after the condition is constructed, we set the error to nil
+		// and requeue the item
+		var authError = &auth.NotAllowedError{}
+		if errors.As(result.Error, &authError) {
+			klog.V(2).Infof("apply work %s fails with err: %v", manifestWorkName, result.Error)
+			result.Error = nil
+
+			if authError.RequeueTime < requeueTime {
+				requeueTime = authError.RequeueTime
+			}
+		}
+
+		// ignore server side apply conflict error since it cannot be resolved by error fallback.
+		var ssaConflict = &apply.ServerSideApplyConflictError{}
+		if result.Error != nil && !errors.As(result.Error, &ssaConflict) {
+			errs = append(errs, result.Error)
+		}
 	}
 
 	// Update work status
-	_, _, err = helper.UpdateManifestWorkStatus(
+	_, updated, err := helper.UpdateManifestWorkStatus(
 		ctx, m.manifestWorkClient, manifestWork, m.generateUpdateStatusFunc(manifestWork.Generation, newManifestConditions))
 	if err != nil {
 		errs = append(errs, fmt.Errorf("failed to update work status with err %w", err))
 	}
+
+	if !updated && requeueTime < MaxRequeueDuration {
+		controllerContext.Queue().AddAfter(manifestWorkName, requeueTime)
+	}
+
 	if len(errs) > 0 {
 		err = utilerrors.NewAggregate(errs)
 		klog.Errorf("Reconcile work %s fails with err: %v", manifestWorkName, err)
 	}
+
 	return err
 }
 
@@ -248,6 +273,13 @@ func (m *ManifestWorkController) applyOneManifest(
 		return result
 	}
 
+	// check the Executor subject permission before applying
+	err = m.validator.Validate(ctx, workSpec.Executor, gvr, resMeta.Namespace, resMeta.Name, auth.ApplyAction)
+	if err != nil {
+		result.Error = err
+		return result
+	}
+
 	// compute required ownerrefs based on delete option
 	requiredOwner := manageOwnerRef(gvr, resMeta.Namespace, resMeta.Name, workSpec.DeleteOption, owner)
 

pkg/spoke/controllers/manifestcontroller/manifestwork_controller_test.go

@@ -22,6 +22,7 @@ import (
 	workinformers "open-cluster-management.io/api/client/work/informers/externalversions"
 	workapiv1 "open-cluster-management.io/api/work/v1"
 	"open-cluster-management.io/work/pkg/spoke/apply"
+	"open-cluster-management.io/work/pkg/spoke/auth"
 	"open-cluster-management.io/work/pkg/spoke/controllers"
 	"open-cluster-management.io/work/pkg/spoke/spoketesting"
 )
@@ -36,13 +37,14 @@ type testController struct {
 func newController(t *testing.T, work *workapiv1.ManifestWork, appliedWork *workapiv1.AppliedManifestWork, mapper meta.RESTMapper) *testController {
 	fakeWorkClient := fakeworkclient.NewSimpleClientset(work)
 	workInformerFactory := workinformers.NewSharedInformerFactoryWithOptions(fakeWorkClient, 5*time.Minute, workinformers.WithNamespace("cluster1"))
-
+	spokeKubeClient := fakekube.NewSimpleClientset()
 	controller := &ManifestWorkController{
 		manifestWorkClient:        fakeWorkClient.WorkV1().ManifestWorks("cluster1"),
 		manifestWorkLister:        workInformerFactory.Work().V1().ManifestWorks().Lister().ManifestWorks("cluster1"),
 		appliedManifestWorkClient: fakeWorkClient.WorkV1().AppliedManifestWorks(),
 		appliedManifestWorkLister: workInformerFactory.Work().V1().AppliedManifestWorks().Lister(),
 		restMapper:                mapper,
+		validator:                 auth.NewExecutorValidator(spokeKubeClient),
 	}
 
 	if err := workInformerFactory.Work().V1().ManifestWorks().Informer().GetStore().Add(work); err != nil {

test/integration/executor_test.go

@@ -0,0 +1,212 @@
+package integration
+
+import (
+	"context"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilrand "k8s.io/apimachinery/pkg/util/rand"
+	workapiv1 "open-cluster-management.io/api/work/v1"
+	"open-cluster-management.io/work/pkg/spoke"
+	"open-cluster-management.io/work/test/integration/util"
+)
+
+var _ = ginkgo.Describe("ManifestWork Executor Subject", func() {
+	var o *spoke.WorkloadAgentOptions
+	var cancel context.CancelFunc
+
+	var work *workapiv1.ManifestWork
+	var manifests []workapiv1.Manifest
+	var executor *workapiv1.ManifestWorkExecutor
+
+	var err error
+
+	ginkgo.BeforeEach(func() {
+		o = spoke.NewWorkloadAgentOptions()
+		o.HubKubeconfigFile = hubKubeconfigFileName
+		o.SpokeClusterName = utilrand.String(5)
+		o.StatusSyncInterval = 3 * time.Second
+
+		ns := &corev1.Namespace{}
+		ns.Name = o.SpokeClusterName
+		_, err := spokeKubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		var ctx context.Context
+		ctx, cancel = context.WithCancel(context.Background())
+		go startWorkAgent(ctx, o)
+
+		// reset manifests
+		manifests = nil
+		executor = nil
+	})
+
+	ginkgo.JustBeforeEach(func() {
+		work = util.NewManifestWork(o.SpokeClusterName, "", manifests)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		work.Spec.Executor = executor
+	})
+
+	ginkgo.AfterEach(func() {
+		if cancel != nil {
+			cancel()
+		}
+		err := spokeKubeClient.CoreV1().Namespaces().Delete(
+			context.Background(), o.SpokeClusterName, metav1.DeleteOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	})
+
+	ginkgo.Context("Apply the resource with executor", func() {
+		executorName := "test-executor"
+		ginkgo.BeforeEach(func() {
+			manifests = []workapiv1.Manifest{
+				util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm1", map[string]string{"a": "b"}, []string{})),
+				util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm2", map[string]string{"c": "d"}, []string{})),
+			}
+			executor = &workapiv1.ManifestWorkExecutor{
+				Subject: workapiv1.ManifestWorkExecutorSubject{
+					Type: workapiv1.ExecutorSubjectTypeServiceAccount,
+					ServiceAccount: &workapiv1.ManifestWorkSubjectServiceAccount{
+						Namespace: o.SpokeClusterName,
+						Name:      executorName,
+					},
+				},
+			}
+		})
+
+		ginkgo.It("Executor does not have permission", func() {
+			work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Create(
+				context.Background(), work, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied),
+				metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionFalse},
+				eventuallyTimeout, eventuallyInterval)
+			util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkAvailable),
+				metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionFalse},
+				eventuallyTimeout, eventuallyInterval)
+
+			// ensure configmaps not exist
+			util.AssertNonexistenceOfConfigMaps(manifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval)
+		})
+
+		ginkgo.It("Executor does not have permission to partial resources", func() {
+			roleName := "role1"
+			_, err = spokeKubeClient.RbacV1().Roles(o.SpokeClusterName).Create(
+				context.TODO(), &rbacv1.Role{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: o.SpokeClusterName,
+						Name:      roleName,
+					},
+					Rules: []rbacv1.PolicyRule{
+						{
+							Verbs:         []string{"create", "update", "patch", "get", "list", "delete"},
+							APIGroups:     []string{""},
+							Resources:     []string{"configmaps"},
+							ResourceNames: []string{"cm1"},
+						},
+					},
+				}, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+			_, err = spokeKubeClient.RbacV1().RoleBindings(o.SpokeClusterName).Create(
+				context.TODO(), &rbacv1.RoleBinding{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: o.SpokeClusterName,
+						Name:      roleName,
+					},
+					Subjects: []rbacv1.Subject{
+						{
+							Kind:      "ServiceAccount",
+							Namespace: o.SpokeClusterName,
+							Name:      executorName,
+						},
+					},
+					RoleRef: rbacv1.RoleRef{
+						APIGroup: "rbac.authorization.k8s.io",
+						Kind:     "Role",
+						Name:     roleName,
+					},
+				}, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Create(
+				context.Background(), work, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied),
+				metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse},
+				eventuallyTimeout, eventuallyInterval)
+			util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkAvailable),
+				metav1.ConditionFalse, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse},
+				eventuallyTimeout, eventuallyInterval)
+
+			// ensure configmap cm1 exist and cm2 not exist
+			util.AssertExistenceOfConfigMaps(
+				[]workapiv1.Manifest{
+					util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm1", map[string]string{"a": "b"}, []string{})),
+				}, spokeKubeClient, eventuallyTimeout, eventuallyInterval)
+			util.AssertNonexistenceOfConfigMaps(
+				[]workapiv1.Manifest{
+					util.ToManifest(util.NewConfigmap(o.SpokeClusterName, "cm2", map[string]string{"a": "b"}, []string{})),
+				}, spokeKubeClient, eventuallyTimeout, eventuallyInterval)
+		})
+
+		ginkgo.It("Executor has permission for all resources", func() {
+			roleName := "role1"
+			_, err = spokeKubeClient.RbacV1().Roles(o.SpokeClusterName).Create(
+				context.TODO(), &rbacv1.Role{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: o.SpokeClusterName,
+						Name:      roleName,
+					},
+					Rules: []rbacv1.PolicyRule{
+						{
+							Verbs:         []string{"create", "update", "patch", "get", "list", "delete"},
+							APIGroups:     []string{""},
+							Resources:     []string{"configmaps"},
+							ResourceNames: []string{"cm1", "cm2"},
+						},
+					},
+				}, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+			_, err = spokeKubeClient.RbacV1().RoleBindings(o.SpokeClusterName).Create(
+				context.TODO(), &rbacv1.RoleBinding{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: o.SpokeClusterName,
+						Name:      roleName,
+					},
+					Subjects: []rbacv1.Subject{
+						{
+							Kind:      "ServiceAccount",
+							Namespace: o.SpokeClusterName,
+							Name:      executorName,
+						},
+					},
+					RoleRef: rbacv1.RoleRef{
+						APIGroup: "rbac.authorization.k8s.io",
+						Kind:     "Role",
+						Name:     roleName,
+					},
+				}, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			work, err = hubWorkClient.WorkV1().ManifestWorks(o.SpokeClusterName).Create(
+				context.Background(), work, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+			util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkApplied),
+				metav1.ConditionTrue, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue},
+				eventuallyTimeout, eventuallyInterval)
+			util.AssertWorkCondition(work.Namespace, work.Name, hubWorkClient, string(workapiv1.WorkAvailable),
+				metav1.ConditionTrue, []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionTrue},
+				eventuallyTimeout, eventuallyInterval)
+
+			// ensure configmaps all exist
+			util.AssertExistenceOfConfigMaps(manifests, spokeKubeClient, eventuallyTimeout, eventuallyInterval)
+		})
+	})
+})

test/integration/util/assertion.go

@@ -122,6 +122,21 @@ func AssertExistenceOfConfigMaps(manifests []workapiv1.Manifest, kubeClient kube
 	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
 }
 
+// check if configmap does not exist
+func AssertNonexistenceOfConfigMaps(manifests []workapiv1.Manifest, kubeClient kubernetes.Interface,
+	eventuallyTimeout, eventuallyInterval int) {
+	gomega.Eventually(func() bool {
+		for _, manifest := range manifests {
+			expected := manifest.Object.(*corev1.ConfigMap)
+			_, err := kubeClient.CoreV1().ConfigMaps(expected.Namespace).Get(
+				context.Background(), expected.Name, metav1.GetOptions{})
+			return apierrors.IsNotFound(err)
+		}
+
+		return false
+	}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.BeFalse())
+}
+
 // check the existence of resource with GVR, namespace and name
 func AssertExistenceOfResources(gvrs []schema.GroupVersionResource, namespaces, names []string, dynamicClient dynamic.Interface, eventuallyTimeout, eventuallyInterval int) {
 	gomega.Expect(gvrs).To(gomega.HaveLen(len(namespaces)))

test/integration/work_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilrand "k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/util/retry"
 
 	workapiv1 "open-cluster-management.io/api/work/v1"
 	"open-cluster-management.io/work/pkg/spoke"
@@ -636,11 +637,22 @@ var _ = ginkgo.Describe("ManifestWork", func() {
 			go func() {
 				for _, manifest := range manifests {
 					cm := manifest.Object.(*corev1.ConfigMap)
-					cm, err := spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Get(context.Background(), cm.Name, metav1.GetOptions{})
-					if err == nil {
-						cm.Finalizers = nil
-						_, _ = spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Update(context.Background(), cm, metav1.UpdateOptions{})
-					}
+					err = retry.OnError(
+						retry.DefaultBackoff,
+						func(err error) bool {
+							return err != nil
+						},
+						func() error {
+							cm, err := spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Get(context.Background(), cm.Name, metav1.GetOptions{})
+							if err != nil {
+								return err
+							}
+
+							cm.Finalizers = nil
+							_, err = spokeKubeClient.CoreV1().ConfigMaps(cm.Namespace).Update(context.Background(), cm, metav1.UpdateOptions{})
+							return err
+						})
+					gomega.Expect(err).ToNot(gomega.HaveOccurred())
 					time.Sleep(2 * time.Second)
 				}
 			}()